From 6f68c11814dc9c76deb088931bae56ba40df2a27 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 15 Apr 2021 19:54:08 +0200 Subject: [PATCH 001/239] Load broker queue configuration using MinosConfig (minos_microservice_common lib) Broker database configuration must be loaded with MinosConfig #5 --- minos/networks/broker.py | 81 ++++++++++++++++++++++++++++++++++++++++ requirements_dev.txt | 77 +++++++++++++++++++++++++++++++++----- tests/test_broker.py | 63 +++++++++++++++++++++++++++++++ tests/test_config.yaml | 60 +++++++++++++++++++++++++++++ 4 files changed, 271 insertions(+), 10 deletions(-) create mode 100644 minos/networks/broker.py create mode 100644 tests/test_broker.py create mode 100644 tests/test_config.yaml diff --git a/minos/networks/broker.py b/minos/networks/broker.py new file mode 100644 index 00000000..d8852a87 --- /dev/null +++ b/minos/networks/broker.py @@ -0,0 +1,81 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from minos.common.logs import log +from minos.common.configuration.config import MinosConfig +import typing as t +from peewee import * +import abc + + +database_proxy = DatabaseProxy() + + +class BrokerQueueBaseModel(Model): + class Meta: + database = database_proxy + + +class Queue(BrokerQueueBaseModel): + queue_id = AutoField() + topic = CharField() + model = BinaryUUIDField() + retry = IntegerField() + creation_date = DateTimeField() + update_date = DateTimeField() + + +def create_event_tables(config: MinosConfig): + database = PostgresqlDatabase( + config.events.queue.database, + user=config.events.queue.user, + password=config.events.queue.password, + host=config.events.queue.host, + port=config.events.queue.port) + + database_proxy.initialize(database) + with database: + database.create_tables([Queue]) + + +def create_command_tables(config: MinosConfig): + database = PostgresqlDatabase( + config.commands.queue.database, + user=config.commands.queue.user, + password=config.commands.queue.password, + host=config.commands.queue.host, + port=config.commands.queue.port) + + database_proxy.initialize(database) + with database: + database.create_tables([Queue]) + + +def drop_event_tables(config: MinosConfig): + database = PostgresqlDatabase( + config.events.queue.database, + user=config.events.queue.user, + password=config.events.queue.password, + host=config.events.queue.host, + port=config.events.queue.port) + + database_proxy.initialize(database) + with database: + database.drop_tables([Queue]) + + +def drop_commands_tables(config: MinosConfig): + database = PostgresqlDatabase( + config.commands.queue.database, + user=config.commands.queue.user, + password=config.commands.queue.password, + host=config.commands.queue.host, + port=config.commands.queue.port) + + database_proxy.initialize(database) + with database: + database.drop_tables([Queue]) diff --git a/requirements_dev.txt b/requirements_dev.txt index aae4484e..13b8812a 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,12 +1,69 @@ -pip -bump2version -wheel -watchdog -flake8 -tox -coverage -Sphinx -twine - +aiokafka==0.7.0 +aiomisc==12.1.0 +alabaster==0.7.12 +appdirs==1.4.4 +atomicwrites==1.4.0 +attrs==20.3.0 +avro==1.10.2 +Babel==2.9.0 +bleach==3.3.0 +bump2version==1.0.1 +certifi==2020.12.5 +chardet==4.0.0 +colorama==0.4.4 +colorlog==5.0.1 +coverage==5.5 +distlib==0.3.1 +docutils==0.16 +filelock==3.0.12 +flake8==3.9.0 +idna==2.10 +imagesize==1.2.0 +importlib-metadata==3.10.1 +Jinja2==2.11.3 +kafka-python==2.0.2 +keyring==23.0.1 +lmdb==1.1.1 +MarkupSafe==1.1.1 +mccabe==0.6.1 +minos-microservice-common==0.0.1.7 +more-itertools==8.7.0 +orjson==3.5.1 +packaging==20.9 +peewee==3.14.4 +pkginfo==1.7.0 +pluggy==0.13.1 +psycopg2==2.8.6 +py==1.10.0 +pycodestyle==2.7.0 +pyflakes==2.3.1 +Pygments==2.8.1 +pyparsing==2.4.7 pytest==4.6.5 pytest-runner==5.1 +pytz==2021.1 +PyYAML==5.4.1 +readme-renderer==29.0 +requests==2.25.1 +requests-toolbelt==0.9.1 +rfc3986==1.4.0 +six==1.15.0 +snowballstemmer==2.1.0 +Sphinx==3.5.4 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +toml==0.10.2 +tox==3.23.0 +tqdm==4.60.0 +twine==3.4.1 +typing-extensions==3.7.4.3 +urllib3==1.26.4 +virtualenv==20.4.3 +watchdog==2.0.2 +wcwidth==0.2.5 +webencodings==0.5.1 +zipp==3.4.1 diff --git a/tests/test_broker.py b/tests/test_broker.py new file mode 100644 index 00000000..92538ebe --- /dev/null +++ b/tests/test_broker.py @@ -0,0 +1,63 @@ +import pytest + +from minos.common.logs import log +from minos.networks.broker import create_event_tables, create_command_tables, drop_event_tables, drop_commands_tables, Queue +from minos.common.configuration.config import MinosConfig +from peewee import * + +# create role broker with createdb login password 'br0k3r'; +# CREATE DATABASE broker_db OWNER broker; + +@pytest.fixture() +def config(): + return MinosConfig(path='./tests/test_config.yaml') + +@pytest.fixture() +def events_database(config): + return PostgresqlDatabase( + config.events.queue.database, + user=config.events.queue.user, + password=config.events.queue.password, + host=config.events.queue.host, + port=config.events.queue.port) + + +@pytest.fixture() +def commands_database(config): + return PostgresqlDatabase( + config.events.queue.database, + user=config.events.queue.user, + password=config.events.queue.password, + host=config.events.queue.host, + port=config.events.queue.port) + + +def test_broker_events_tables_creation(config, events_database): + create_event_tables(config) + assert events_database.table_exists(table_name="queue") is True + + +def test_broker_events_database_connection(events_database): + assert events_database.connect() is True + + +def test_broker_event_tables_deletion(config, events_database): + drop_event_tables(config) + assert events_database.table_exists(table_name="queue") is False + + +def test_broker_commands_tables_creation(config, commands_database): + create_command_tables(config) + assert commands_database.table_exists(table_name="queue") is True + + +def test_broker_commands_database_connection(commands_database): + assert commands_database.connect() is True + + +def test_broker_commands_tables_deletion(config, commands_database): + drop_commands_tables(config) + assert commands_database.table_exists(table_name="queue") is False + + + diff --git a/tests/test_config.yaml b/tests/test_config.yaml new file mode 100644 index 00000000..f0523ced --- /dev/null +++ b/tests/test_config.yaml @@ -0,0 +1,60 @@ +service: + name: Order +rest: + host: localhost + port: 8900 + endpoints: + - name: AddOrder + route: /order + method: POST + controller: minos.services.OrderService + action: add_order +events: + broker: localhost + port: 9092 + database: + path: ./tests/local_db.lmdb + name: database_events_test + items: + - name: TicketAdded + controller: minos.services.CQRSService + action: ticket_added + - name: TicketDeleted + controller: minos.services.CQRSService + action: ticket_deleted + - name: OrderAdded + controller: minos.services.CQRSService + action: order_added + queue: + database: broker_db + user: broker + password: br0k3r + host: localhost + port: 5432 + records: 10 +commands: + broker: localhost + port: 9092 + database: + path: ./tests/local_db.lmdb + name: database_commands_test + items: + - name: AddOrder + controller: minos.services.OrderService + action: add_order + - name: DeleteOrder + controller: minos.services.OrderService + action: delete_order + - name: UpdateOrder + controller: minos.services.OrderService + action: update_order + - name: GetOrder + controller: minos.service.OrderService + action: get_order + queue: + database: broker_db + user: broker + password: br0k3r + host: localhost + port: 5432 + records: 10 From f80761fc04b38dbcaa4da5e490248a44385584d3 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Mon, 19 Apr 2021 13:08:28 +0200 Subject: [PATCH 002/239] Store Command/Events into queue The broker must have PostgreSQL DB to save the queue #4 --- minos/networks/broker.py | 149 +++++++++++++++++++++++++++++---------- tests/test_broker.py | 31 ++++++-- 2 files changed, 140 insertions(+), 40 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index d8852a87..472fd174 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -10,11 +10,27 @@ import typing as t from peewee import * import abc +import datetime database_proxy = DatabaseProxy() +class MinosBrokerDatabase: + def __init__(self, conf): + self.database = PostgresqlDatabase( + conf.queue.database, + user=conf.queue.user, + password=conf.queue.password, + host=conf.queue.host, + port=conf.queue.port, + ) + + def get_connection(self): + database_proxy.initialize(self.database) + return self.database + + class BrokerQueueBaseModel(Model): class Meta: database = database_proxy @@ -23,59 +39,120 @@ class Meta: class Queue(BrokerQueueBaseModel): queue_id = AutoField() topic = CharField() - model = BinaryUUIDField() - retry = IntegerField() - creation_date = DateTimeField() - update_date = DateTimeField() + model = BlobField() + retry = IntegerField(default=0) + creation_date = DateTimeField(default=datetime.datetime.now) + update_date = DateTimeField(default=datetime.datetime.now) def create_event_tables(config: MinosConfig): - database = PostgresqlDatabase( - config.events.queue.database, - user=config.events.queue.user, - password=config.events.queue.password, - host=config.events.queue.host, - port=config.events.queue.port) - - database_proxy.initialize(database) + database = MinosBrokerDatabase(config.events).get_connection() + with database: database.create_tables([Queue]) def create_command_tables(config: MinosConfig): - database = PostgresqlDatabase( - config.commands.queue.database, - user=config.commands.queue.user, - password=config.commands.queue.password, - host=config.commands.queue.host, - port=config.commands.queue.port) - - database_proxy.initialize(database) + database = MinosBrokerDatabase(config.commands).get_connection() + with database: database.create_tables([Queue]) def drop_event_tables(config: MinosConfig): - database = PostgresqlDatabase( - config.events.queue.database, - user=config.events.queue.user, - password=config.events.queue.password, - host=config.events.queue.host, - port=config.events.queue.port) - - database_proxy.initialize(database) + database = MinosBrokerDatabase(config.events).get_connection() with database: database.drop_tables([Queue]) def drop_commands_tables(config: MinosConfig): - database = PostgresqlDatabase( - config.commands.queue.database, - user=config.commands.queue.user, - password=config.commands.queue.password, - host=config.commands.queue.host, - port=config.commands.queue.port) - - database_proxy.initialize(database) + database = MinosBrokerDatabase(config.commands).get_connection() with database: database.drop_tables([Queue]) + + +class AggregateModel: + name: str + pass + + +class ModelBase: + pass + + +class EventModel(ModelBase): + topic: str + model: str + items: AggregateModel + + +class CommandModel(ModelBase): + topic: str + model: str + items: AggregateModel + + +class BrokerBase(abc.ABC): + @abc.abstractmethod + def _database(self): + raise NotImplementedError + + @abc.abstractmethod + def send(self): + raise NotImplementedError + + +class MinosEventBroker(BrokerBase): + def __init__(self, topic: str, model: AggregateModel, config: MinosConfig): + self.config = config + self.topic = topic + self.model = model + self._database() + + def _database(self): + self.database = MinosBrokerDatabase(self.config.events).get_connection() + + def send(self): + self.database.connect() + + # TODO: Change + event_instance = EventModel() + event_instance.topic = self.topic + event_instance.name = self.model.name + event_instance.items = self.model + + query = Queue.insert(topic=self.topic, model=hex(10)) + result = query.execute() + log.debug(result) + + self.database.close() + + return result + + +class MinosCommandBroker(BrokerBase): + def __init__(self, topic: str, model: AggregateModel, config: MinosConfig): + self.config = config + self.topic = topic + self.model = model + self._database() + + def _database(self): + self.database = MinosBrokerDatabase(self.config.events).get_connection() + + def send(self): + self.database.connect() + + # TODO: Change + event_instance = CommandModel() + event_instance.topic = self.topic + event_instance.name = self.model.name + event_instance.items = self.model + + query = Queue.insert(topic=self.topic, model=hex(10)) + result = query.execute() + log.debug(result) + + self.database.close() + + return result diff --git a/tests/test_broker.py b/tests/test_broker.py index 92538ebe..7b4f79c7 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -1,7 +1,8 @@ import pytest from minos.common.logs import log -from minos.networks.broker import create_event_tables, create_command_tables, drop_event_tables, drop_commands_tables, Queue +from minos.networks.broker import create_event_tables, create_command_tables,\ + drop_event_tables, drop_commands_tables, Queue, MinosEventBroker, MinosCommandBroker, AggregateModel from minos.common.configuration.config import MinosConfig from peewee import * @@ -55,9 +56,31 @@ def test_broker_commands_database_connection(commands_database): assert commands_database.connect() is True -def test_broker_commands_tables_deletion(config, commands_database): - drop_commands_tables(config) - assert commands_database.table_exists(table_name="queue") is False +def test_events_broker_insertion(config, events_database): + a = AggregateModel() + a.name = "EventBroker" + + result = MinosEventBroker("EventBroker", a, config).send() + + query = Queue.select().where(Queue.topic == "EventBroker") + + assert result is not None + assert query.get() is not None + +def test_commands_broker_insertion(config, commands_database): + a = AggregateModel() + a.name = "CommandBroker" + result = MinosCommandBroker("CommandBroker", a, config).send() + query = Queue.select().where(Queue.topic == "CommandBroker") + + assert result is not None + assert query.get() is not None + + +# Olways leave on end +def test_broker_commands_tables_deletion(config, commands_database): + drop_commands_tables(config) + assert commands_database.table_exists(table_name="queue") is False From 761db6e1e4cb7dceda11452473ab7ff5b303efb3 Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Mon, 19 Apr 2021 18:28:50 +0200 Subject: [PATCH 003/239] modified format of rst files to md --- .github/workflows/python-tests.yml | 6 +----- .restyled.yaml | 28 ++++++++++++++++++++++++++++ HISTORY.rst => HISTORY.md | 1 - README.md | 16 ++++++++++++++++ README.rst | 18 ------------------ setup.py | 5 +++-- 6 files changed, 48 insertions(+), 26 deletions(-) create mode 100644 .restyled.yaml rename HISTORY.rst => HISTORY.md (91%) create mode 100644 README.md delete mode 100644 README.rst diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index f4a78da6..95a2e1d5 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -1,8 +1,4 @@ -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] +on: push jobs: build: diff --git a/.restyled.yaml b/.restyled.yaml new file mode 100644 index 00000000..e8e0a9d3 --- /dev/null +++ b/.restyled.yaml @@ -0,0 +1,28 @@ +--- +enabled: true +exclude: + - "**/*.md" + - ".idea/**/*" + - "docs/**/*" + - "**/*.in" + - "Makefile" + - ".github/workflows/**/*" +restylers: + - name: black + image: restyled/restyler-black:v19.10b0 + command: + - black + arguments: ["--line-length", "120"] + include: + - "**/*.py" + interpreters: + - python + - name: isort + image: restyled/restyler-isort:v4.3.21 + command: + - isort + arguments: [] + include: + - "**/*.py" + interpreters: + - python diff --git a/HISTORY.rst b/HISTORY.md similarity index 91% rename from HISTORY.rst rename to HISTORY.md index d32b71d6..529ee21e 100644 --- a/HISTORY.rst +++ b/HISTORY.md @@ -1,4 +1,3 @@ -======= History ======= diff --git a/README.md b/README.md new file mode 100644 index 00000000..bc770c2c --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +Minos Microservice Networks +=========================== + +[![codecov](https://codecov.io/gh/Clariteia/minos_microservice_networks/branch/main/graph/badge.svg)](https://codecov.io/gh/Clariteia/minos_microservice_common) + +![Tests](https://github.com/Clariteia/minos_microservice_networks/actions/workflows/python-tests.yml/badge.svg) + +Python Package with the common network classes and utlities used in Minos Microservice + + + +Credits +------- + +This package was created with ![Cookiecutter](https://github.com/audreyr/cookiecutter) +and the ![Minos Package](https://github.com/Clariteia/minos-pypackage) project template. diff --git a/README.rst b/README.rst deleted file mode 100644 index 15b60fc3..00000000 --- a/README.rst +++ /dev/null @@ -1,18 +0,0 @@ -=========================== -Minos Microservice Networks -=========================== - -Python Package with the common network classes and utlities used in Minos Microservice - -Features --------- - -* TODO - -Credits -------- - -This package was created with Cookiecutter_ and the `clariteia/minos-pypackage`_ project template. - -.. _Cookiecutter: https://github.com/audreyr/cookiecutter -.. _`clariteia/minos-pypackage`: https://bitbucket.org/clariteia-devs/minos-pypackage/src/master/ diff --git a/setup.py b/setup.py index 600da3b6..2fa661b8 100644 --- a/setup.py +++ b/setup.py @@ -4,10 +4,10 @@ from setuptools import setup, find_namespace_packages -with open('README.rst') as readme_file: +with open('README.md') as readme_file: readme = readme_file.read() -with open('HISTORY.rst') as history_file: +with open('HISTORY.md') as history_file: history = history_file.read() requirements = ['minos-microservice-common', 'aiokafka', 'aiomisc'] @@ -32,6 +32,7 @@ ], description="Python Package with the common network classes and utlities used in Minos Microservice", install_requires=requirements, + long_description_content_type='text/markdown', long_description=readme + '\n\n' + history, include_package_data=True, keywords='minos_microservice_networks', From f3d3a67b7f6d2068111d2ef2c69bb8948c5e9610 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 10:22:48 +0200 Subject: [PATCH 004/239] Migrate to aiopg Migrate the concept from PostgreSQL ORM to python driver #11 --- minos/networks/broker.py | 139 +++++++++++++++++++-------------------- tests/test_broker.py | 125 +++++++++++++++++++++-------------- 2 files changed, 141 insertions(+), 123 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 472fd174..99ff09be 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -8,67 +8,25 @@ from minos.common.logs import log from minos.common.configuration.config import MinosConfig import typing as t -from peewee import * import abc import datetime - -database_proxy = DatabaseProxy() +import aiomisc +import aiopg +import asyncio +from aiomisc.service.periodic import Service, PeriodicService class MinosBrokerDatabase: - def __init__(self, conf): - self.database = PostgresqlDatabase( - conf.queue.database, - user=conf.queue.user, - password=conf.queue.password, - host=conf.queue.host, - port=conf.queue.port, + async def get_connection(self, conf): + conn = await aiopg.connect( + database=conf.events.queue.database, + user=conf.events.queue.user, + password=conf.events.queue.password, + host=conf.events.queue.host, + port=conf.events.queue.port, ) - - def get_connection(self): - database_proxy.initialize(self.database) - return self.database - - -class BrokerQueueBaseModel(Model): - class Meta: - database = database_proxy - - -class Queue(BrokerQueueBaseModel): - queue_id = AutoField() - topic = CharField() - model = BlobField() - retry = IntegerField(default=0) - creation_date = DateTimeField(default=datetime.datetime.now) - update_date = DateTimeField(default=datetime.datetime.now) - - -def create_event_tables(config: MinosConfig): - database = MinosBrokerDatabase(config.events).get_connection() - - with database: - database.create_tables([Queue]) - - -def create_command_tables(config: MinosConfig): - database = MinosBrokerDatabase(config.commands).get_connection() - - with database: - database.create_tables([Queue]) - - -def drop_event_tables(config: MinosConfig): - database = MinosBrokerDatabase(config.events).get_connection() - with database: - database.drop_tables([Queue]) - - -def drop_commands_tables(config: MinosConfig): - database = MinosBrokerDatabase(config.commands).get_connection() - with database: - database.drop_tables([Queue]) + return conn class AggregateModel: @@ -94,11 +52,11 @@ class CommandModel(ModelBase): class BrokerBase(abc.ABC): @abc.abstractmethod - def _database(self): + def _database(self): # pragma: no cover raise NotImplementedError @abc.abstractmethod - def send(self): + def send(self): # pragma: no cover raise NotImplementedError @@ -110,10 +68,9 @@ def __init__(self, topic: str, model: AggregateModel, config: MinosConfig): self._database() def _database(self): - self.database = MinosBrokerDatabase(self.config.events).get_connection() + pass - def send(self): - self.database.connect() + async def send(self): # TODO: Change event_instance = EventModel() @@ -121,13 +78,23 @@ def send(self): event_instance.name = self.model.name event_instance.items = self.model - query = Queue.insert(topic=self.topic, model=hex(10)) - result = query.execute() - log.debug(result) + bin_data = b"bytes object" - self.database.close() + conn = await MinosBrokerDatabase().get_connection(self.config) - return result + cur = await conn.cursor() + await cur.execute( + "INSERT INTO queue (topic, model, retry, creation_date, update_date) VALUES (%s, %s, %s, %s, %s) RETURNING queue_id;", + ( + event_instance.topic, + bin_data, + 0, + datetime.datetime.now(), + datetime.datetime.now(), + ), + ) + + conn.close() class MinosCommandBroker(BrokerBase): @@ -138,10 +105,9 @@ def __init__(self, topic: str, model: AggregateModel, config: MinosConfig): self._database() def _database(self): - self.database = MinosBrokerDatabase(self.config.events).get_connection() + pass - def send(self): - self.database.connect() + async def send(self): # TODO: Change event_instance = CommandModel() @@ -149,10 +115,39 @@ def send(self): event_instance.name = self.model.name event_instance.items = self.model - query = Queue.insert(topic=self.topic, model=hex(10)) - result = query.execute() - log.debug(result) + bin_data = b"bytes object" + + conn = await MinosBrokerDatabase().get_connection(self.config) + + cur = await conn.cursor() + await cur.execute( + "INSERT INTO queue (topic, model, retry, creation_date, update_date) VALUES (%s, %s, %s, %s, %s) RETURNING queue_id;", + ( + event_instance.topic, + bin_data, + 0, + datetime.datetime.now(), + datetime.datetime.now(), + ), + ) + + conn.close() + + +class BrokerDatabaseInitializer(Service): + async def start(self): + # Send signal to entrypoint for continue running + self.start_event.set() + + conn = await MinosBrokerDatabase().get_connection(self.config) + + cur = await conn.cursor() + await cur.execute( + 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, ' + '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' + '"creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' + ) - self.database.close() + conn.close() - return result + return diff --git a/tests/test_broker.py b/tests/test_broker.py index 7b4f79c7..70bbc0cf 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -1,86 +1,109 @@ import pytest from minos.common.logs import log -from minos.networks.broker import create_event_tables, create_command_tables,\ - drop_event_tables, drop_commands_tables, Queue, MinosEventBroker, MinosCommandBroker, AggregateModel +from minos.networks.broker import MinosEventBroker, MinosCommandBroker, AggregateModel, BrokerDatabaseInitializer, MinosBrokerDatabase from minos.common.configuration.config import MinosConfig -from peewee import * +import aiopg +import asyncio +import aiomisc +from aiomisc.service.periodic import Service -# create role broker with createdb login password 'br0k3r'; -# CREATE DATABASE broker_db OWNER broker; -@pytest.fixture() +@pytest.fixture(scope='session') def config(): return MinosConfig(path='./tests/test_config.yaml') -@pytest.fixture() -def events_database(config): - return PostgresqlDatabase( - config.events.queue.database, - user=config.events.queue.user, - password=config.events.queue.password, - host=config.events.queue.host, - port=config.events.queue.port) +@pytest.fixture(scope='session') +def services(config): + return ( + BrokerDatabaseInitializer( + config=config + ), + ) -@pytest.fixture() -def commands_database(config): - return PostgresqlDatabase( - config.events.queue.database, - user=config.events.queue.user, - password=config.events.queue.password, - host=config.events.queue.host, - port=config.events.queue.port) +@pytest.fixture(scope='session') +def run_services(services, config): + with aiomisc.entrypoint(*services) as loop: + loop.run_forever() -def test_broker_events_tables_creation(config, events_database): - create_event_tables(config) - assert events_database.table_exists(table_name="queue") is True - - -def test_broker_events_database_connection(events_database): - assert events_database.connect() is True +@pytest.fixture() +async def database(config): + return await MinosBrokerDatabase().get_connection(config) -def test_broker_event_tables_deletion(config, events_database): - drop_event_tables(config) - assert events_database.table_exists(table_name="queue") is False +""" +async def test_create_database(database): + cur = await database.cursor() + await cur.execute( + 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, ' + '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' + '"creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' + ) + database.close() +""" -def test_broker_commands_tables_creation(config, commands_database): - create_command_tables(config) - assert commands_database.table_exists(table_name="queue") is True +async def test_if_queue_table_exists(database): + cur = await database.cursor() + await cur.execute("SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'queue';") + ret = [] + async for row in cur: + ret.append(row) -def test_broker_commands_database_connection(commands_database): - assert commands_database.connect() is True + database.close() + assert ret == [(1,)] -def test_events_broker_insertion(config, events_database): +async def test_events_broker_insertion(config, database): a = AggregateModel() a.name = "EventBroker" - result = MinosEventBroker("EventBroker", a, config).send() + m = MinosEventBroker("EventBroker", a, config) + await m.send() + + cur = await database.cursor() - query = Queue.select().where(Queue.topic == "EventBroker") + await cur.execute("SELECT 1 FROM queue WHERE topic = 'EventBroker' LIMIT 1;") + ret = [] + async for row in cur: + ret.append(row) - assert result is not None - assert query.get() is not None + database.close() + assert ret == [(1,)] -def test_commands_broker_insertion(config, commands_database): +async def test_commands_broker_insertion(config, database): a = AggregateModel() a.name = "CommandBroker" - result = MinosCommandBroker("CommandBroker", a, config).send() + m = MinosCommandBroker("CommandBroker", a, config) + await m.send() - query = Queue.select().where(Queue.topic == "CommandBroker") + cur = await database.cursor() - assert result is not None - assert query.get() is not None + await cur.execute("SELECT 1 FROM queue WHERE topic = 'CommandBroker' LIMIT 1;") + ret = [] + async for row in cur: + ret.append(row) + database.close() + assert ret == [(1,)] -# Olways leave on end -def test_broker_commands_tables_deletion(config, commands_database): - drop_commands_tables(config) - assert commands_database.table_exists(table_name="queue") is False + + +async def test_drop_database(database): + cur = await database.cursor() + await cur.execute("DROP TABLE IF EXISTS queue;") + database.close() + + + + +# create role broker with createdb login password 'br0k3r'; +# CREATE DATABASE broker_db OWNER broker; +# 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, "topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL)', [] +# 'SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = %s ORDER BY tablename', ('public',) +# 'DROP TABLE IF EXISTS "queue"' From 4552dee59dcf2c7097a27514c715c6bc86d677ea Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 10:29:09 +0200 Subject: [PATCH 005/239] Rename test_config.yaml --- tests/test_broker.py | 2 +- tests/{test_config.yaml => test_config_.yaml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/{test_config.yaml => test_config_.yaml} (100%) diff --git a/tests/test_broker.py b/tests/test_broker.py index 70bbc0cf..dc95669e 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -11,7 +11,7 @@ @pytest.fixture(scope='session') def config(): - return MinosConfig(path='./tests/test_config.yaml') + return MinosConfig(path='./tests/test_config_.yaml') @pytest.fixture(scope='session') diff --git a/tests/test_config.yaml b/tests/test_config_.yaml similarity index 100% rename from tests/test_config.yaml rename to tests/test_config_.yaml From 3be14acebf445637cc23b5a2d1ab87ecb7adb40c Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 20 Apr 2021 08:29:53 +0000 Subject: [PATCH 006/239] Restyled by autopep8 --- tests/test_broker.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_broker.py b/tests/test_broker.py index dc95669e..2d1ad16d 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -45,6 +45,7 @@ async def test_create_database(database): database.close() """ + async def test_if_queue_table_exists(database): cur = await database.cursor() @@ -93,15 +94,12 @@ async def test_commands_broker_insertion(config, database): assert ret == [(1,)] - async def test_drop_database(database): cur = await database.cursor() await cur.execute("DROP TABLE IF EXISTS queue;") database.close() - - # create role broker with createdb login password 'br0k3r'; # CREATE DATABASE broker_db OWNER broker; # 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, "topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL)', [] From 36ae95e90e709c766466247303af1038f75cc609 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 20 Apr 2021 08:29:54 +0000 Subject: [PATCH 007/239] Restyled by black --- tests/test_broker.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/test_broker.py b/tests/test_broker.py index 2d1ad16d..498db7bf 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -1,7 +1,13 @@ import pytest from minos.common.logs import log -from minos.networks.broker import MinosEventBroker, MinosCommandBroker, AggregateModel, BrokerDatabaseInitializer, MinosBrokerDatabase +from minos.networks.broker import ( + MinosEventBroker, + MinosCommandBroker, + AggregateModel, + BrokerDatabaseInitializer, + MinosBrokerDatabase, +) from minos.common.configuration.config import MinosConfig import aiopg import asyncio @@ -9,21 +15,17 @@ from aiomisc.service.periodic import Service -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def config(): - return MinosConfig(path='./tests/test_config_.yaml') + return MinosConfig(path="./tests/test_config_.yaml") -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def services(config): - return ( - BrokerDatabaseInitializer( - config=config - ), - ) + return (BrokerDatabaseInitializer(config=config),) -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def run_services(services, config): with aiomisc.entrypoint(*services) as loop: loop.run_forever() @@ -33,6 +35,7 @@ def run_services(services, config): async def database(config): return await MinosBrokerDatabase().get_connection(config) + """ async def test_create_database(database): cur = await database.cursor() @@ -49,7 +52,9 @@ async def test_create_database(database): async def test_if_queue_table_exists(database): cur = await database.cursor() - await cur.execute("SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'queue';") + await cur.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'queue';" + ) ret = [] async for row in cur: ret.append(row) From 64889d86ba645fa1255210b321f3b10436522b8e Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 20 Apr 2021 08:29:56 +0000 Subject: [PATCH 008/239] Restyled by isort --- minos/networks/broker.py | 10 +++++----- tests/test_broker.py | 21 +++++++++------------ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 99ff09be..5a1dc944 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -5,16 +5,16 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from minos.common.logs import log -from minos.common.configuration.config import MinosConfig -import typing as t import abc +import asyncio import datetime +import typing as t import aiomisc import aiopg -import asyncio -from aiomisc.service.periodic import Service, PeriodicService +from aiomisc.service.periodic import PeriodicService, Service +from minos.common.configuration.config import MinosConfig +from minos.common.logs import log class MinosBrokerDatabase: diff --git a/tests/test_broker.py b/tests/test_broker.py index 498db7bf..7f730015 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -1,18 +1,15 @@ -import pytest - -from minos.common.logs import log -from minos.networks.broker import ( - MinosEventBroker, - MinosCommandBroker, - AggregateModel, - BrokerDatabaseInitializer, - MinosBrokerDatabase, -) -from minos.common.configuration.config import MinosConfig -import aiopg import asyncio + import aiomisc +import aiopg +import pytest from aiomisc.service.periodic import Service +from minos.common.configuration.config import MinosConfig +from minos.common.logs import log + +from minos.networks.broker import (AggregateModel, BrokerDatabaseInitializer, + MinosBrokerDatabase, MinosCommandBroker, + MinosEventBroker) @pytest.fixture(scope="session") From e40c410d97c18ff2659ecfdb2ea4479a9061aa65 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 20 Apr 2021 08:30:02 +0000 Subject: [PATCH 009/239] Restyled by reorder-python-imports --- minos/networks/broker.py | 5 +++-- tests/test_broker.py | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 5a1dc944..b7db82ee 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -4,7 +4,6 @@ # # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. - import abc import asyncio import datetime @@ -12,7 +11,9 @@ import aiomisc import aiopg -from aiomisc.service.periodic import PeriodicService, Service +from aiomisc.service.periodic import PeriodicService +from aiomisc.service.periodic import Service + from minos.common.configuration.config import MinosConfig from minos.common.logs import log diff --git a/tests/test_broker.py b/tests/test_broker.py index 7f730015..a662ba8d 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -4,12 +4,14 @@ import aiopg import pytest from aiomisc.service.periodic import Service + from minos.common.configuration.config import MinosConfig from minos.common.logs import log - -from minos.networks.broker import (AggregateModel, BrokerDatabaseInitializer, - MinosBrokerDatabase, MinosCommandBroker, - MinosEventBroker) +from minos.networks.broker import AggregateModel +from minos.networks.broker import BrokerDatabaseInitializer +from minos.networks.broker import MinosBrokerDatabase +from minos.networks.broker import MinosCommandBroker +from minos.networks.broker import MinosEventBroker @pytest.fixture(scope="session") From 13262071fa7fbfb92e616190f7803547c67a4323 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 20 Apr 2021 08:30:07 +0000 Subject: [PATCH 010/239] Restyled by yapf --- tests/test_broker.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_broker.py b/tests/test_broker.py index a662ba8d..d359bb06 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -21,7 +21,7 @@ def config(): @pytest.fixture(scope="session") def services(config): - return (BrokerDatabaseInitializer(config=config),) + return (BrokerDatabaseInitializer(config=config), ) @pytest.fixture(scope="session") @@ -59,7 +59,7 @@ async def test_if_queue_table_exists(database): ret.append(row) database.close() - assert ret == [(1,)] + assert ret == [(1, )] async def test_events_broker_insertion(config, database): @@ -71,13 +71,14 @@ async def test_events_broker_insertion(config, database): cur = await database.cursor() - await cur.execute("SELECT 1 FROM queue WHERE topic = 'EventBroker' LIMIT 1;") + await cur.execute( + "SELECT 1 FROM queue WHERE topic = 'EventBroker' LIMIT 1;") ret = [] async for row in cur: ret.append(row) database.close() - assert ret == [(1,)] + assert ret == [(1, )] async def test_commands_broker_insertion(config, database): @@ -89,13 +90,14 @@ async def test_commands_broker_insertion(config, database): cur = await database.cursor() - await cur.execute("SELECT 1 FROM queue WHERE topic = 'CommandBroker' LIMIT 1;") + await cur.execute( + "SELECT 1 FROM queue WHERE topic = 'CommandBroker' LIMIT 1;") ret = [] async for row in cur: ret.append(row) database.close() - assert ret == [(1,)] + assert ret == [(1, )] async def test_drop_database(database): From 4c3d9368f62ec205e36b70a417ba6b8082e17a1a Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 12:13:36 +0200 Subject: [PATCH 011/239] Update test_broker.py #19 --- tests/test_broker.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/tests/test_broker.py b/tests/test_broker.py index d359bb06..f815b6f9 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -3,6 +3,7 @@ import aiomisc import aiopg import pytest +import time from aiomisc.service.periodic import Service from minos.common.configuration.config import MinosConfig @@ -19,15 +20,13 @@ def config(): return MinosConfig(path="./tests/test_config_.yaml") -@pytest.fixture(scope="session") +@pytest.fixture def services(config): - return (BrokerDatabaseInitializer(config=config), ) - - -@pytest.fixture(scope="session") -def run_services(services, config): - with aiomisc.entrypoint(*services) as loop: - loop.run_forever() + return [ + BrokerDatabaseInitializer( + config=config + ) + ] @pytest.fixture() @@ -35,20 +34,8 @@ async def database(config): return await MinosBrokerDatabase().get_connection(config) -""" -async def test_create_database(database): - cur = await database.cursor() - await cur.execute( - 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, ' - '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' - '"creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' - ) - - database.close() -""" - - async def test_if_queue_table_exists(database): + time.sleep(1) cur = await database.cursor() await cur.execute( From a42c1e74c9dbaeac7447b243ea75bd5b3ced50cf Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 12:21:14 +0200 Subject: [PATCH 012/239] Update python-tests.yml --- .github/workflows/python-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 95a2e1d5..bf095d17 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -29,7 +29,6 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - apt-get install -y libc6 build-essential python -m pip install --upgrade pip if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi python setup.py install From f0c2cf7f098b6ba4b61f05dfed16425d8679e3b5 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 12:52:17 +0200 Subject: [PATCH 013/239] Update python-tests.yml --- .github/workflows/python-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index bf095d17..95a2e1d5 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -29,6 +29,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | + apt-get install -y libc6 build-essential python -m pip install --upgrade pip if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi python setup.py install From ce684bd1997ab33f6240b2d76116fddac6c9a152 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 13:01:33 +0200 Subject: [PATCH 014/239] Update python-tests.yml --- .github/workflows/python-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 95a2e1d5..924bbad5 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -29,6 +29,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | + apt update apt-get install -y libc6 build-essential python -m pip install --upgrade pip if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi From db664682d72daf7266eb631808da57a1373c4405 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 20 Apr 2021 11:01:41 +0000 Subject: [PATCH 015/239] Restyled by black --- tests/test_broker.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/tests/test_broker.py b/tests/test_broker.py index f815b6f9..6d79ac62 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -22,11 +22,7 @@ def config(): @pytest.fixture def services(config): - return [ - BrokerDatabaseInitializer( - config=config - ) - ] + return [BrokerDatabaseInitializer(config=config)] @pytest.fixture() @@ -38,15 +34,13 @@ async def test_if_queue_table_exists(database): time.sleep(1) cur = await database.cursor() - await cur.execute( - "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'queue';" - ) + await cur.execute("SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'queue';") ret = [] async for row in cur: ret.append(row) database.close() - assert ret == [(1, )] + assert ret == [(1,)] async def test_events_broker_insertion(config, database): @@ -58,14 +52,13 @@ async def test_events_broker_insertion(config, database): cur = await database.cursor() - await cur.execute( - "SELECT 1 FROM queue WHERE topic = 'EventBroker' LIMIT 1;") + await cur.execute("SELECT 1 FROM queue WHERE topic = 'EventBroker' LIMIT 1;") ret = [] async for row in cur: ret.append(row) database.close() - assert ret == [(1, )] + assert ret == [(1,)] async def test_commands_broker_insertion(config, database): @@ -77,14 +70,13 @@ async def test_commands_broker_insertion(config, database): cur = await database.cursor() - await cur.execute( - "SELECT 1 FROM queue WHERE topic = 'CommandBroker' LIMIT 1;") + await cur.execute("SELECT 1 FROM queue WHERE topic = 'CommandBroker' LIMIT 1;") ret = [] async for row in cur: ret.append(row) database.close() - assert ret == [(1, )] + assert ret == [(1,)] async def test_drop_database(database): From a2ad8c8419886c04d5194f2d2fd6f20afc2959d8 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 20 Apr 2021 11:01:43 +0000 Subject: [PATCH 016/239] Restyled by isort --- tests/test_broker.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/test_broker.py b/tests/test_broker.py index 6d79ac62..c5376474 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -1,18 +1,15 @@ import asyncio +import time import aiomisc import aiopg import pytest -import time from aiomisc.service.periodic import Service - from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import AggregateModel -from minos.networks.broker import BrokerDatabaseInitializer -from minos.networks.broker import MinosBrokerDatabase -from minos.networks.broker import MinosCommandBroker -from minos.networks.broker import MinosEventBroker +from minos.networks.broker import (AggregateModel, BrokerDatabaseInitializer, + MinosBrokerDatabase, MinosCommandBroker, + MinosEventBroker) @pytest.fixture(scope="session") From 1ba3ed16cb68179d4eae46c48bfb9fe392ed6721 Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Tue, 20 Apr 2021 13:32:24 +0200 Subject: [PATCH 017/239] removed apt-get install build essential --- .github/workflows/python-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 95a2e1d5..bf095d17 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -29,7 +29,6 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - apt-get install -y libc6 build-essential python -m pip install --upgrade pip if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi python setup.py install From d65ef22a09c616890fae7dc5f02916e8048ef14a Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Tue, 20 Apr 2021 13:36:58 +0200 Subject: [PATCH 018/239] added actions test whitiut kafka server test --- .github/workflows/python-tests.yml | 24 ++------ tests/test_broker.py | 13 ----- tests/test_event_manager.py | 88 +++++++++++++++--------------- 3 files changed, 50 insertions(+), 75 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index bf095d17..8904d7a0 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -2,25 +2,12 @@ on: push jobs: build: + runs-on: ubuntu-latest strategy: matrix: python-version: [3.9] - container: - image: python:latest - services: - zookeeper: - image: bitnami/zookeeper:latest - ports: - - "2181:2181" - kafka: - image: bitnami/kafka:latest - ports: - - "9092:9092" - env: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_HOST_NAME: localhost + steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -29,9 +16,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi - python setup.py install + python -m pip install --upgrade pip + if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi + python setup.py install - name: Test with pytest run: | make test @@ -44,3 +31,4 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml fail_ci_if_error: true + diff --git a/tests/test_broker.py b/tests/test_broker.py index d359bb06..6ec838a7 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -35,19 +35,6 @@ async def database(config): return await MinosBrokerDatabase().get_connection(config) -""" -async def test_create_database(database): - cur = await database.cursor() - await cur.execute( - 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, ' - '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' - '"creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' - ) - - database.close() -""" - - async def test_if_queue_table_exists(database): cur = await database.cursor() diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 58ca9b8d..4df37c27 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -1,44 +1,44 @@ -import asyncio -import logging - -import pytest -import string -from aiokafka import AIOKafkaProducer -import random - -from aiomisc.log import basic_config -from minos.common.configuration.config import MinosConfig - -from minos.networks.event import MinosEventServer - - -@pytest.fixture() -def config(): - return MinosConfig(path='./tests/test_config.yaml') - - -@pytest.fixture() -def services(config): - return [MinosEventServer(conf=config)] - - -async def test_producer_kafka(loop): - basic_config( - level=logging.INFO, - buffered=True, - log_format='color', - flush_interval=2 - ) - - producer = AIOKafkaProducer(loop=loop, bootstrap_servers='localhost:9092') - # Get cluster layout and topic/partition allocation - await producer.start() - # Produce messages - string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=20)) - await producer.send_and_wait("TicketAdded", string_to_send.encode()) - await asyncio.sleep(1) - - other_string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=40)) - await producer.send_and_wait("TicketDeleted", other_string_to_send.encode()) - await asyncio.sleep(1) - await producer.stop() +# import asyncio +# import logging +# +# import pytest +# import string +# from aiokafka import AIOKafkaProducer +# import random +# +# from aiomisc.log import basic_config +# from minos.common.configuration.config import MinosConfig +# +# from minos.networks.event import MinosEventServer +# +# +# @pytest.fixture() +# def config(): +# return MinosConfig(path='./tests/test_config.yaml') +# +# +# @pytest.fixture() +# def services(config): +# return [MinosEventServer(conf=config)] +# +# +# async def test_producer_kafka(loop): +# basic_config( +# level=logging.INFO, +# buffered=True, +# log_format='color', +# flush_interval=2 +# ) +# +# producer = AIOKafkaProducer(loop=loop, bootstrap_servers='localhost:9092') +# # Get cluster layout and topic/partition allocation +# await producer.start() +# # Produce messages +# string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=20)) +# await producer.send_and_wait("TicketAdded", string_to_send.encode()) +# await asyncio.sleep(1) +# +# other_string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=40)) +# await producer.send_and_wait("TicketDeleted", other_string_to_send.encode()) +# await asyncio.sleep(1) +# await producer.stop() From cf7da77305417a3509938a33da0cb2e149c5320b Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Tue, 20 Apr 2021 13:41:28 +0200 Subject: [PATCH 019/239] added aiopg dependecy --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2fa661b8..3c166562 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ with open('HISTORY.md') as history_file: history = history_file.read() -requirements = ['minos-microservice-common', 'aiokafka', 'aiomisc'] +requirements = ['minos-microservice-common', 'aiokafka', 'aiomisc', 'aiopg'] setup_requirements = ['pytest-runner',] From 5f2e746cbff235c0b1a5aa0551d5aa674a4926db Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 14:00:38 +0200 Subject: [PATCH 020/239] Update python-tests.yml --- .github/workflows/python-tests.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index ca16c667..7a5b9e04 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -8,6 +8,22 @@ jobs: matrix: python-version: [3.9] + # Service containers to run with `container-job` + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -32,3 +48,8 @@ jobs: files: ./coverage.xml fail_ci_if_error: true + - name: Create database + run: | + PGPASSWORD=postgres psql -U postgres -tc "CREATE ROLE broker with createdb login password 'br0k3r';" + PGPASSWORD=postgres psql -U postgres -tc "CREATE DATABASE broker_db OWNER broker;" + From ae4a717f33855aa278ebb80c347914026363e3c8 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 14:10:22 +0200 Subject: [PATCH 021/239] Update python-tests.yml --- .github/workflows/python-tests.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 7a5b9e04..2840928a 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -35,6 +35,10 @@ jobs: python -m pip install --upgrade pip if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi python setup.py install + - name: Create database + run: | + PGPASSWORD=postgres psql -U postgres -tc "CREATE ROLE broker with createdb login password 'br0k3r';" + PGPASSWORD=postgres psql -U postgres -tc "CREATE DATABASE broker_db OWNER broker;" - name: Test with pytest run: | make test @@ -48,8 +52,3 @@ jobs: files: ./coverage.xml fail_ci_if_error: true - - name: Create database - run: | - PGPASSWORD=postgres psql -U postgres -tc "CREATE ROLE broker with createdb login password 'br0k3r';" - PGPASSWORD=postgres psql -U postgres -tc "CREATE DATABASE broker_db OWNER broker;" - From cff311e59dbc5b447a0a74a902f93e260513e836 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 14:32:50 +0200 Subject: [PATCH 022/239] Update python-tests.yml --- .github/workflows/python-tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 2840928a..bf8ed64b 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -37,8 +37,10 @@ jobs: python setup.py install - name: Create database run: | - PGPASSWORD=postgres psql -U postgres -tc "CREATE ROLE broker with createdb login password 'br0k3r';" - PGPASSWORD=postgres psql -U postgres -tc "CREATE DATABASE broker_db OWNER broker;" + psql -h postgres -U postgres -c "CREATE ROLE broker with createdb login password 'br0k3r';" + psql -h postgres -U postgres -c "CREATE DATABASE broker_db OWNER broker;" + env: + PGPASSWORD: postgres - name: Test with pytest run: | make test From 6c76627bd233d8341b52c323c2c14ac3157bc329 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 14:36:13 +0200 Subject: [PATCH 023/239] Update python-tests.yml --- .github/workflows/python-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index bf8ed64b..bd9891b2 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -37,8 +37,8 @@ jobs: python setup.py install - name: Create database run: | - psql -h postgres -U postgres -c "CREATE ROLE broker with createdb login password 'br0k3r';" - psql -h postgres -U postgres -c "CREATE DATABASE broker_db OWNER broker;" + psql -h localhost -U postgres -c "CREATE ROLE broker with createdb login password 'br0k3r';" + psql -h localhost -U postgres -c "CREATE DATABASE broker_db OWNER broker;" env: PGPASSWORD: postgres - name: Test with pytest From 49c2a659fcef5a73174ea2587a5cd538b8d391cd Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 14:51:24 +0200 Subject: [PATCH 024/239] Update python-tests.yml --- .github/workflows/python-tests.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index bd9891b2..73fcc289 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -16,6 +16,7 @@ jobs: image: postgres # Provide the password for postgres env: + POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres # Set health checks to wait until postgres has started options: >- @@ -23,6 +24,9 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 steps: - uses: actions/checkout@v2 @@ -37,10 +41,8 @@ jobs: python setup.py install - name: Create database run: | - psql -h localhost -U postgres -c "CREATE ROLE broker with createdb login password 'br0k3r';" - psql -h localhost -U postgres -c "CREATE DATABASE broker_db OWNER broker;" - env: - PGPASSWORD: postgres + PGPASSWORD=postgres psql -U postgres -tc "CREATE ROLE broker with createdb login password 'br0k3r';" + PGPASSWORD=postgres psql -U postgres -tc "CREATE DATABASE broker_db OWNER broker;" - name: Test with pytest run: | make test From fb545afac4a931b2d2fdf719b47b45250911962d Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 17:46:46 +0200 Subject: [PATCH 025/239] Update python-tests.yml --- .github/workflows/python-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 73fcc289..e390134c 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -13,7 +13,7 @@ jobs: # Label used to access the service container postgres: # Docker Hub image - image: postgres + image: postgres:11 # Provide the password for postgres env: POSTGRES_USER: postgres @@ -41,8 +41,8 @@ jobs: python setup.py install - name: Create database run: | - PGPASSWORD=postgres psql -U postgres -tc "CREATE ROLE broker with createdb login password 'br0k3r';" - PGPASSWORD=postgres psql -U postgres -tc "CREATE DATABASE broker_db OWNER broker;" + PGPASSWORD=postgres psql -U postgres -h localhost -p 5432 -tc "CREATE ROLE broker with createdb login password 'br0k3r';" + PGPASSWORD=postgres psql -U postgres -h localhost -p 5432 -tc "CREATE DATABASE broker_db OWNER broker;" - name: Test with pytest run: | make test From 55973ab8d4aecfc0969d944d69137a505dbc2d9f Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 17:53:17 +0200 Subject: [PATCH 026/239] Update requirements_dev.txt --- requirements_dev.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 13b8812a..ba0dc533 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,15 +1,19 @@ aiokafka==0.7.0 aiomisc==12.1.0 +aiopg==1.2.1 alabaster==0.7.12 appdirs==1.4.4 +async-timeout==3.0.1 atomicwrites==1.4.0 attrs==20.3.0 avro==1.10.2 Babel==2.9.0 +black==20.8b1 bleach==3.3.0 bump2version==1.0.1 certifi==2020.12.5 chardet==4.0.0 +click==7.1.2 colorama==0.4.4 colorlog==5.0.1 coverage==5.5 @@ -26,14 +30,17 @@ keyring==23.0.1 lmdb==1.1.1 MarkupSafe==1.1.1 mccabe==0.6.1 -minos-microservice-common==0.0.1.7 +minos-microservice-common==0.0.2 more-itertools==8.7.0 +mypy-extensions==0.4.3 orjson==3.5.1 packaging==20.9 +pathspec==0.8.1 peewee==3.14.4 pkginfo==1.7.0 pluggy==0.13.1 psycopg2==2.8.6 +psycopg2-binary==2.8.6 py==1.10.0 pycodestyle==2.7.0 pyflakes==2.3.1 @@ -44,6 +51,7 @@ pytest-runner==5.1 pytz==2021.1 PyYAML==5.4.1 readme-renderer==29.0 +regex==2021.4.4 requests==2.25.1 requests-toolbelt==0.9.1 rfc3986==1.4.0 @@ -60,6 +68,7 @@ toml==0.10.2 tox==3.23.0 tqdm==4.60.0 twine==3.4.1 +typed-ast==1.4.3 typing-extensions==3.7.4.3 urllib3==1.26.4 virtualenv==20.4.3 From c974f02d9c34c9ef605aea4e7a730056677c3c05 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 20 Apr 2021 18:13:40 +0200 Subject: [PATCH 027/239] Update test_broker.py --- tests/test_broker.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_broker.py b/tests/test_broker.py index c5376474..1d079d8d 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -1,10 +1,5 @@ -import asyncio import time - -import aiomisc -import aiopg import pytest -from aiomisc.service.periodic import Service from minos.common.configuration.config import MinosConfig from minos.common.logs import log from minos.networks.broker import (AggregateModel, BrokerDatabaseInitializer, @@ -12,7 +7,7 @@ MinosEventBroker) -@pytest.fixture(scope="session") +@pytest.fixture() def config(): return MinosConfig(path="./tests/test_config_.yaml") From 4a2eae7548dd14b8e712caea50e9e295fb0b8d3d Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 21 Apr 2021 10:28:18 +0200 Subject: [PATCH 028/239] Update test_config_.yaml --- tests/test_config_.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_config_.yaml b/tests/test_config_.yaml index f0523ced..52807e60 100644 --- a/tests/test_config_.yaml +++ b/tests/test_config_.yaml @@ -29,7 +29,7 @@ events: database: broker_db user: broker password: br0k3r - host: localhost + host: postgres port: 5432 records: 10 commands: From a1f3d9127fd735450fc98337135687b50af80c21 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 21 Apr 2021 11:06:53 +0200 Subject: [PATCH 029/239] Configure DB --- .github/workflows/python-tests.yml | 1 + tests/test_config_.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index e390134c..3c542fb2 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -18,6 +18,7 @@ jobs: env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: trust # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready diff --git a/tests/test_config_.yaml b/tests/test_config_.yaml index 52807e60..f0523ced 100644 --- a/tests/test_config_.yaml +++ b/tests/test_config_.yaml @@ -29,7 +29,7 @@ events: database: broker_db user: broker password: br0k3r - host: postgres + host: localhost port: 5432 records: 10 commands: From f487213f6a64b7c9ba0809cc8de6cfed64529c59 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 21 Apr 2021 11:29:50 +0200 Subject: [PATCH 030/239] Initial structure of Periodic Service Periodic service check every half of second the queue status #32 --- minos/networks/broker.py | 57 ++++++++++++++++++++++++++++++++++++++++ tests/test_broker.py | 6 ++++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index b7db82ee..77cdc40c 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -13,6 +13,7 @@ import aiopg from aiomisc.service.periodic import PeriodicService from aiomisc.service.periodic import Service +from aiokafka import AIOKafkaProducer from minos.common.configuration.config import MinosConfig from minos.common.logs import log @@ -152,3 +153,59 @@ async def start(self): conn.close() return + + + +class Dispatcher: + def __init__(self, config): + self.config = config + + async def run(self): + conn = await MinosBrokerDatabase().get_connection(self.config) + + async with conn.cursor() as cur: + await cur.execute("SELECT * FROM queue WHERE retry <= 2 ORDER BY creation_date ASC LIMIT 10;") + async for row in cur: + + log.debug(row) + log.debug("id = %s", row[0]) + log.debug("topic = %s", row[1]) + log.debug("model = %s", row[2]) + log.debug("retry = %s", row[3]) + log.debug("creation_date = %s", row[4]) + log.debug("update_date = %s", row[5]) + + sent_to_kafka = await self._send_to_kafka() + if sent_to_kafka: + # Delete from database If the event was sent successfully to Kafka. + async with conn.cursor() as cur2: + await cur2.execute("DELETE FROM queue WHERE queue_id=%d;" % row[0]) + else: + # Update queue retry column. Increase by 1. + async with conn.cursor() as cur3: + await cur3.execute("UPDATE queue SET retry = retry + 1 WHERE queue_id=%d;" % row[0]) + + conn.commit() + conn.close() + + async def _send_to_kafka(self): + flag = False + producer = AIOKafkaProducer(bootstrap_servers='localhost:9092') + # Get cluster layout and initial topic/partition leadership information + await producer.start() + try: + # Produce message + await producer.send_and_wait("my_topic", b"Super message") + flag = True + except Exception as e: + flag = False + finally: + # Wait for all pending messages to be delivered or expire. + await producer.stop() + + return flag + + +class EventBrokerQueueDispatcher(PeriodicService): + async def callback(self): + pass diff --git a/tests/test_broker.py b/tests/test_broker.py index 1d079d8d..2feb20af 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -4,7 +4,7 @@ from minos.common.logs import log from minos.networks.broker import (AggregateModel, BrokerDatabaseInitializer, MinosBrokerDatabase, MinosCommandBroker, - MinosEventBroker) + MinosEventBroker, Dispatcher) @pytest.fixture() @@ -71,6 +71,10 @@ async def test_commands_broker_insertion(config, database): assert ret == [(1,)] +async def test_queue_dispatcher(config): + d = Dispatcher(config) + await d.run() + async def test_drop_database(database): cur = await database.cursor() await cur.execute("DROP TABLE IF EXISTS queue;") From 1726a3ab7c4d001fc07fe4c3d680141246fc1d85 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 21 Apr 2021 11:43:49 +0200 Subject: [PATCH 031/239] Update python-tests.yml --- .github/workflows/python-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 3c542fb2..7efb381d 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -13,11 +13,12 @@ jobs: # Label used to access the service container postgres: # Docker Hub image - image: postgres:11 + image: postgres # Provide the password for postgres env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres + POSTGRES_HOST: localhost POSTGRES_HOST_AUTH_METHOD: trust # Set health checks to wait until postgres has started options: >- From 75cdc3b29bc3a34dd255af0d94022217ad15a4de Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 21 Apr 2021 12:08:07 +0200 Subject: [PATCH 032/239] Update broker.py --- minos/networks/broker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 77cdc40c..578c5e87 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -152,7 +152,7 @@ async def start(self): conn.close() - return + await self.stop(self) From 3b50fcd63f3c2eb624234ec6b4be81027846f5cf Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Wed, 21 Apr 2021 14:24:39 +0200 Subject: [PATCH 033/239] modified the actions added container for postgresql actions and recognition --- .github/workflows/python-tests.yml | 37 ++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 8904d7a0..a1ea76de 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -4,9 +4,29 @@ jobs: build: runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.9] + container: python:3.9-buster + + # Service containers to run with `container-job` + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres:latest + # Provide the password for postgres + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST: localhost + POSTGRES_HOST_AUTH_METHOD: trust + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 steps: - uses: actions/checkout@v2 @@ -16,9 +36,13 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi - python setup.py install + python -m pip install --upgrade pip + if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi + python setup.py install + - name: Create database + run: | + PGPASSWORD=postgres psql -U postgres -h postgres -p 5432 -tc "CREATE ROLE broker with createdb login password 'br0k3r';" + PGPASSWORD=postgres psql -U postgres -h postgres -p 5432 -tc "CREATE DATABASE broker_db OWNER broker;" - name: Test with pytest run: | make test @@ -31,4 +55,3 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml fail_ci_if_error: true - From 0a1c2478e4531f7a545699c0791586973798cca4 Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Wed, 21 Apr 2021 14:29:33 +0200 Subject: [PATCH 034/239] added OS dependencies for postgresql --- .github/workflows/python-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index a1ea76de..4710a2d7 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -39,6 +39,10 @@ jobs: python -m pip install --upgrade pip if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi python setup.py install + - name: Install Postgresql client + run: | + apt-get update + apt-get install --yes postgresql-client - name: Create database run: | PGPASSWORD=postgres psql -U postgres -h postgres -p 5432 -tc "CREATE ROLE broker with createdb login password 'br0k3r';" From 906df561a7371f55be860620096b4a01ef693e04 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 21 Apr 2021 14:56:54 +0200 Subject: [PATCH 035/239] Model correction --- minos/networks/broker.py | 52 +++++++++++----------------------------- tests/test_broker.py | 23 +++++++++++------- 2 files changed, 28 insertions(+), 47 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 578c5e87..55534fbe 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -16,6 +16,8 @@ from aiokafka import AIOKafkaProducer from minos.common.configuration.config import MinosConfig +from minos.common import MinosModel, ModelRef +from minos.common.broker import MinosBaseBroker from minos.common.logs import log @@ -30,57 +32,31 @@ async def get_connection(self, conf): ) return conn +class Aggregate(MinosModel): + test_id: int -class AggregateModel: - name: str - pass - -class ModelBase: - pass - - -class EventModel(ModelBase): - topic: str - model: str - items: AggregateModel - - -class CommandModel(ModelBase): +class EventModel(MinosModel): topic: str model: str - items: AggregateModel - - -class BrokerBase(abc.ABC): - @abc.abstractmethod - def _database(self): # pragma: no cover - raise NotImplementedError - - @abc.abstractmethod - def send(self): # pragma: no cover - raise NotImplementedError + #items: list[ModelRef[Aggregate]] + items: list[str] -class MinosEventBroker(BrokerBase): - def __init__(self, topic: str, model: AggregateModel, config: MinosConfig): +class MinosEventBroker(MinosBaseBroker): + def __init__(self, topic: str, config: MinosConfig): self.config = config self.topic = topic - self.model = model self._database() def _database(self): pass - async def send(self): + async def send(self, model: Aggregate): # TODO: Change - event_instance = EventModel() - event_instance.topic = self.topic - event_instance.name = self.model.name - event_instance.items = self.model - - bin_data = b"bytes object" + event_instance = EventModel(topic=self.topic, model="Change", items=[str(model)]) + bin_data = event_instance.avro_bytes conn = await MinosBrokerDatabase().get_connection(self.config) @@ -98,7 +74,7 @@ async def send(self): conn.close() - +""" class MinosCommandBroker(BrokerBase): def __init__(self, topic: str, model: AggregateModel, config: MinosConfig): self.config = config @@ -134,7 +110,7 @@ async def send(self): ) conn.close() - +""" class BrokerDatabaseInitializer(Service): async def start(self): diff --git a/tests/test_broker.py b/tests/test_broker.py index 2feb20af..417c0f9b 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -2,8 +2,9 @@ import pytest from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import (AggregateModel, BrokerDatabaseInitializer, - MinosBrokerDatabase, MinosCommandBroker, +from minos.common.model.model import MinosModel +from minos.networks.broker import (BrokerDatabaseInitializer, + MinosBrokerDatabase, Aggregate, MinosEventBroker, Dispatcher) @@ -35,12 +36,15 @@ async def test_if_queue_table_exists(database): assert ret == [(1,)] +class AgregateTest(Aggregate): + test: int + + async def test_events_broker_insertion(config, database): - a = AggregateModel() - a.name = "EventBroker" + a = AgregateTest(test_id=1, test=2) - m = MinosEventBroker("EventBroker", a, config) - await m.send() + m = MinosEventBroker("EventBroker", config) + await m.send(a) cur = await database.cursor() @@ -52,7 +56,7 @@ async def test_events_broker_insertion(config, database): database.close() assert ret == [(1,)] - +""" async def test_commands_broker_insertion(config, database): a = AggregateModel() a.name = "CommandBroker" @@ -69,17 +73,18 @@ async def test_commands_broker_insertion(config, database): database.close() assert ret == [(1,)] - +""" async def test_queue_dispatcher(config): d = Dispatcher(config) await d.run() +""" async def test_drop_database(database): cur = await database.cursor() await cur.execute("DROP TABLE IF EXISTS queue;") database.close() - +""" # create role broker with createdb login password 'br0k3r'; # CREATE DATABASE broker_db OWNER broker; From d10bcead49841b273450f3acfadda8f2ed173ed0 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 21 Apr 2021 12:57:04 +0000 Subject: [PATCH 036/239] Restyled by black --- minos/networks/broker.py | 16 ++++++---------- tests/test_broker.py | 13 ++++++++++--- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 55534fbe..71ab8236 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -32,6 +32,7 @@ async def get_connection(self, conf): ) return conn + class Aggregate(MinosModel): test_id: int @@ -39,7 +40,7 @@ class Aggregate(MinosModel): class EventModel(MinosModel): topic: str model: str - #items: list[ModelRef[Aggregate]] + # items: list[ModelRef[Aggregate]] items: list[str] @@ -63,17 +64,12 @@ async def send(self, model: Aggregate): cur = await conn.cursor() await cur.execute( "INSERT INTO queue (topic, model, retry, creation_date, update_date) VALUES (%s, %s, %s, %s, %s) RETURNING queue_id;", - ( - event_instance.topic, - bin_data, - 0, - datetime.datetime.now(), - datetime.datetime.now(), - ), + (event_instance.topic, bin_data, 0, datetime.datetime.now(), datetime.datetime.now(),), ) conn.close() + """ class MinosCommandBroker(BrokerBase): def __init__(self, topic: str, model: AggregateModel, config: MinosConfig): @@ -112,6 +108,7 @@ async def send(self): conn.close() """ + class BrokerDatabaseInitializer(Service): async def start(self): # Send signal to entrypoint for continue running @@ -131,7 +128,6 @@ async def start(self): await self.stop(self) - class Dispatcher: def __init__(self, config): self.config = config @@ -166,7 +162,7 @@ async def run(self): async def _send_to_kafka(self): flag = False - producer = AIOKafkaProducer(bootstrap_servers='localhost:9092') + producer = AIOKafkaProducer(bootstrap_servers="localhost:9092") # Get cluster layout and initial topic/partition leadership information await producer.start() try: diff --git a/tests/test_broker.py b/tests/test_broker.py index 417c0f9b..b4ccd732 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -3,9 +3,13 @@ from minos.common.configuration.config import MinosConfig from minos.common.logs import log from minos.common.model.model import MinosModel -from minos.networks.broker import (BrokerDatabaseInitializer, - MinosBrokerDatabase, Aggregate, - MinosEventBroker, Dispatcher) +from minos.networks.broker import ( + BrokerDatabaseInitializer, + MinosBrokerDatabase, + Aggregate, + MinosEventBroker, + Dispatcher, +) @pytest.fixture() @@ -56,6 +60,7 @@ async def test_events_broker_insertion(config, database): database.close() assert ret == [(1,)] + """ async def test_commands_broker_insertion(config, database): a = AggregateModel() @@ -75,10 +80,12 @@ async def test_commands_broker_insertion(config, database): assert ret == [(1,)] """ + async def test_queue_dispatcher(config): d = Dispatcher(config) await d.run() + """ async def test_drop_database(database): cur = await database.cursor() From fd7f471f53d0e6fe3f4b8875cb71ab0e4bf352ac Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 21 Apr 2021 12:57:05 +0000 Subject: [PATCH 037/239] Restyled by isort --- minos/networks/broker.py | 6 ++---- tests/test_broker.py | 11 ++++------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 71ab8236..3f5777a3 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -11,13 +11,11 @@ import aiomisc import aiopg -from aiomisc.service.periodic import PeriodicService -from aiomisc.service.periodic import Service from aiokafka import AIOKafkaProducer - -from minos.common.configuration.config import MinosConfig +from aiomisc.service.periodic import PeriodicService, Service from minos.common import MinosModel, ModelRef from minos.common.broker import MinosBaseBroker +from minos.common.configuration.config import MinosConfig from minos.common.logs import log diff --git a/tests/test_broker.py b/tests/test_broker.py index b4ccd732..967c25dd 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -1,15 +1,12 @@ import time + import pytest from minos.common.configuration.config import MinosConfig from minos.common.logs import log from minos.common.model.model import MinosModel -from minos.networks.broker import ( - BrokerDatabaseInitializer, - MinosBrokerDatabase, - Aggregate, - MinosEventBroker, - Dispatcher, -) +from minos.networks.broker import (Aggregate, BrokerDatabaseInitializer, + Dispatcher, MinosBrokerDatabase, + MinosEventBroker) @pytest.fixture() From 1f1bb09632ee904c40a5f397c513876842971acb Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 21 Apr 2021 15:01:31 +0200 Subject: [PATCH 038/239] Update python-tests.yml --- .github/workflows/python-tests.yml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index d43ba546..30779198 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -6,28 +6,6 @@ jobs: runs-on: ubuntu-latest container: python:3.9-buster - # Service containers to run with `container-job` - services: - # Label used to access the service container - postgres: - # Docker Hub image - image: postgres:latest - # Provide the password for postgres - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_HOST: localhost - POSTGRES_HOST_AUTH_METHOD: trust - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - # Maps tcp port 5432 on service container to the host - - 5432:5432 - # Service containers to run with `container-job` services: # Label used to access the service container From c9a82452ba5dbb90d90323f51b0c50f8c6d835d2 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 21 Apr 2021 15:05:03 +0200 Subject: [PATCH 039/239] Update test_config_.yaml --- tests/test_config_.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_config_.yaml b/tests/test_config_.yaml index f0523ced..7a4fd137 100644 --- a/tests/test_config_.yaml +++ b/tests/test_config_.yaml @@ -29,7 +29,7 @@ events: database: broker_db user: broker password: br0k3r - host: localhost + host: postgres port: 5432 records: 10 commands: @@ -55,6 +55,6 @@ commands: database: broker_db user: broker password: br0k3r - host: localhost + host: postgres port: 5432 records: 10 From 8460e704092ab511bdce2460aa56137ced74f066 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 21 Apr 2021 15:29:44 +0200 Subject: [PATCH 040/239] Uncomment MinosCommandBroker --- minos/networks/broker.py | 35 +++++++++++++++-------------------- tests/test_broker.py | 15 ++++++--------- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 3f5777a3..1fb6d72d 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -42,6 +42,14 @@ class EventModel(MinosModel): items: list[str] +class CommandModel(MinosModel): + topic: str + model: str + # items: list[ModelRef[Aggregate]] + items: list[str] + callback: str + + class MinosEventBroker(MinosBaseBroker): def __init__(self, topic: str, config: MinosConfig): self.config = config @@ -68,43 +76,30 @@ async def send(self, model: Aggregate): conn.close() -""" -class MinosCommandBroker(BrokerBase): - def __init__(self, topic: str, model: AggregateModel, config: MinosConfig): +class MinosCommandBroker(MinosBaseBroker): + def __init__(self, topic: str, config: MinosConfig): self.config = config self.topic = topic - self.model = model self._database() def _database(self): pass - async def send(self): - + async def send(self, model: Aggregate, callback): # TODO: Change - event_instance = CommandModel() - event_instance.topic = self.topic - event_instance.name = self.model.name - event_instance.items = self.model - - bin_data = b"bytes object" + event_instance = CommandModel(topic=self.topic, model="Change", items=[str(model)], callback=callback) + bin_data = event_instance.avro_bytes conn = await MinosBrokerDatabase().get_connection(self.config) cur = await conn.cursor() await cur.execute( "INSERT INTO queue (topic, model, retry, creation_date, update_date) VALUES (%s, %s, %s, %s, %s) RETURNING queue_id;", - ( - event_instance.topic, - bin_data, - 0, - datetime.datetime.now(), - datetime.datetime.now(), - ), + (event_instance.topic, bin_data, 0, datetime.datetime.now(), datetime.datetime.now(),), ) conn.close() -""" + class BrokerDatabaseInitializer(Service): diff --git a/tests/test_broker.py b/tests/test_broker.py index 967c25dd..bf68f351 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -3,10 +3,9 @@ import pytest from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.common.model.model import MinosModel from minos.networks.broker import (Aggregate, BrokerDatabaseInitializer, Dispatcher, MinosBrokerDatabase, - MinosEventBroker) + MinosEventBroker, MinosCommandBroker) @pytest.fixture() @@ -58,13 +57,11 @@ async def test_events_broker_insertion(config, database): assert ret == [(1,)] -""" async def test_commands_broker_insertion(config, database): - a = AggregateModel() - a.name = "CommandBroker" + a = AgregateTest(test_id=1, test=2) - m = MinosCommandBroker("CommandBroker", a, config) - await m.send() + m = MinosCommandBroker("CommandBroker", config) + await m.send(model=a, callback="test") cur = await database.cursor() @@ -75,15 +72,15 @@ async def test_commands_broker_insertion(config, database): database.close() assert ret == [(1,)] -""" +""" async def test_queue_dispatcher(config): d = Dispatcher(config) await d.run() -""" + async def test_drop_database(database): cur = await database.cursor() await cur.execute("DROP TABLE IF EXISTS queue;") From 1d97b07f12c2767724aef11a5b4fd2b4ed74d46c Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 21 Apr 2021 15:39:39 +0200 Subject: [PATCH 041/239] Fix coverage --- Makefile | 4 ++-- minos/networks/broker.py | 4 +++- tests/test_broker.py | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index bc9d0a2d..f28a0731 100644 --- a/Makefile +++ b/Makefile @@ -59,8 +59,8 @@ test-all: ## run tests on every Python version with tox coverage: ## check code coverage quickly with the default Python coverage run --source minos -m pytest coverage report -m - coverage html - $(BROWSER) htmlcov/index.html + coverage xml + ## $(BROWSER) htmlcov/index.html docs: ## generate Sphinx HTML documentation, including API docs rm -f docs/minos_microservice_networks.rst diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 1fb6d72d..882df91b 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -120,7 +120,7 @@ async def start(self): await self.stop(self) - +""" class Dispatcher: def __init__(self, config): self.config = config @@ -174,3 +174,5 @@ async def _send_to_kafka(self): class EventBrokerQueueDispatcher(PeriodicService): async def callback(self): pass + +""" diff --git a/tests/test_broker.py b/tests/test_broker.py index bf68f351..fdc21ee9 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -4,8 +4,8 @@ from minos.common.configuration.config import MinosConfig from minos.common.logs import log from minos.networks.broker import (Aggregate, BrokerDatabaseInitializer, - Dispatcher, MinosBrokerDatabase, - MinosEventBroker, MinosCommandBroker) + MinosBrokerDatabase, MinosEventBroker, + MinosCommandBroker) @pytest.fixture() From 4cc539b26a40eacafefdc8342f523a7bc6f14ef3 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 21 Apr 2021 13:39:49 +0000 Subject: [PATCH 042/239] Restyled by black --- minos/networks/broker.py | 2 +- tests/test_broker.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 882df91b..c380d835 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -101,7 +101,6 @@ async def send(self, model: Aggregate, callback): conn.close() - class BrokerDatabaseInitializer(Service): async def start(self): # Send signal to entrypoint for continue running @@ -120,6 +119,7 @@ async def start(self): await self.stop(self) + """ class Dispatcher: def __init__(self, config): diff --git a/tests/test_broker.py b/tests/test_broker.py index fdc21ee9..147dc45c 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -3,9 +3,13 @@ import pytest from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import (Aggregate, BrokerDatabaseInitializer, - MinosBrokerDatabase, MinosEventBroker, - MinosCommandBroker) +from minos.networks.broker import ( + Aggregate, + BrokerDatabaseInitializer, + MinosBrokerDatabase, + MinosEventBroker, + MinosCommandBroker, +) @pytest.fixture() From 23bca6c387eed1a12bbc8e0696e216b91db73b5a Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 21 Apr 2021 13:39:50 +0000 Subject: [PATCH 043/239] Restyled by isort --- tests/test_broker.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/test_broker.py b/tests/test_broker.py index 147dc45c..da6650b3 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -3,13 +3,9 @@ import pytest from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import ( - Aggregate, - BrokerDatabaseInitializer, - MinosBrokerDatabase, - MinosEventBroker, - MinosCommandBroker, -) +from minos.networks.broker import (Aggregate, BrokerDatabaseInitializer, + MinosBrokerDatabase, MinosCommandBroker, + MinosEventBroker) @pytest.fixture() From 092f5a256630f65365612dcbf31f1737b84d436c Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 22 Apr 2021 10:55:33 +0200 Subject: [PATCH 044/239] Kafka services + Broker PeriodicService Add Kafka Service in Github Actions for unit tests #3 Periodic service check every half of second the queue status #32 --- .github/workflows/python-tests.yml | 13 ++++++- minos/networks/broker.py | 28 +++++++------- tests/test_broker.py | 55 +++++++++++++-------------- tests/test_config.yaml | 14 +++++++ tests/test_config_.yaml | 60 ------------------------------ 5 files changed, 68 insertions(+), 102 deletions(-) delete mode 100644 tests/test_config_.yaml diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 30779198..7323ee30 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -27,7 +27,18 @@ jobs: ports: # Maps tcp port 5432 on service container to the host - 5432:5432 - + zookeeper: + image: bitnami/zookeeper:latest + ports: + - "2181:2181" + kafka: + image: bitnami/kafka:latest + ports: + - "9092:9092" + env: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_HOST_NAME: localhost steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/minos/networks/broker.py b/minos/networks/broker.py index c380d835..a9dfc2a0 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -73,8 +73,12 @@ async def send(self, model: Aggregate): (event_instance.topic, bin_data, 0, datetime.datetime.now(), datetime.datetime.now(),), ) + queue_id = await cur.fetchone() + affected_rows = cur.rowcount conn.close() + return affected_rows, queue_id[0] + class MinosCommandBroker(MinosBaseBroker): def __init__(self, topic: str, config: MinosConfig): @@ -85,7 +89,7 @@ def __init__(self, topic: str, config: MinosConfig): def _database(self): pass - async def send(self, model: Aggregate, callback): + async def send(self, model: Aggregate, callback: t.Callable): # TODO: Change event_instance = CommandModel(topic=self.topic, model="Change", items=[str(model)], callback=callback) bin_data = event_instance.avro_bytes @@ -98,8 +102,12 @@ async def send(self, model: Aggregate, callback): (event_instance.topic, bin_data, 0, datetime.datetime.now(), datetime.datetime.now(),), ) + queue_id = await cur.fetchone() + affected_rows = cur.rowcount conn.close() + return affected_rows, queue_id[0] + class BrokerDatabaseInitializer(Service): async def start(self): @@ -120,12 +128,8 @@ async def start(self): await self.stop(self) -""" -class Dispatcher: - def __init__(self, config): - self.config = config - - async def run(self): +class EventBrokerQueueDispatcher(PeriodicService): + async def callback(self): conn = await MinosBrokerDatabase().get_connection(self.config) async with conn.cursor() as cur: @@ -140,7 +144,7 @@ async def run(self): log.debug("creation_date = %s", row[4]) log.debug("update_date = %s", row[5]) - sent_to_kafka = await self._send_to_kafka() + sent_to_kafka = await self._send_to_kafka(topic=row[1], message=row[2]) if sent_to_kafka: # Delete from database If the event was sent successfully to Kafka. async with conn.cursor() as cur2: @@ -153,14 +157,14 @@ async def run(self): conn.commit() conn.close() - async def _send_to_kafka(self): + async def _send_to_kafka(self, topic: str, message: bytes): flag = False producer = AIOKafkaProducer(bootstrap_servers="localhost:9092") # Get cluster layout and initial topic/partition leadership information await producer.start() try: # Produce message - await producer.send_and_wait("my_topic", b"Super message") + await producer.send_and_wait(topic, message) flag = True except Exception as e: flag = False @@ -171,8 +175,4 @@ async def _send_to_kafka(self): return flag -class EventBrokerQueueDispatcher(PeriodicService): - async def callback(self): - pass -""" diff --git a/tests/test_broker.py b/tests/test_broker.py index da6650b3..2309ff44 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -1,28 +1,37 @@ import time import pytest +import asyncio from minos.common.configuration.config import MinosConfig from minos.common.logs import log from minos.networks.broker import (Aggregate, BrokerDatabaseInitializer, MinosBrokerDatabase, MinosCommandBroker, - MinosEventBroker) + MinosEventBroker, EventBrokerQueueDispatcher) -@pytest.fixture() +@pytest.fixture def config(): - return MinosConfig(path="./tests/test_config_.yaml") + return MinosConfig(path="./tests/test_config.yaml") @pytest.fixture def services(config): - return [BrokerDatabaseInitializer(config=config)] + return [ + BrokerDatabaseInitializer(config=config), + EventBrokerQueueDispatcher(interval=0.5, delay=0, config=config) + ] -@pytest.fixture() +@pytest.fixture async def database(config): return await MinosBrokerDatabase().get_connection(config) +async def test_database_connection(database): + assert database.closed == 0 + database.close() + + async def test_if_queue_table_exists(database): time.sleep(1) cur = await database.cursor() @@ -36,16 +45,20 @@ async def test_if_queue_table_exists(database): assert ret == [(1,)] -class AgregateTest(Aggregate): +class AggregateTest(Aggregate): test: int -async def test_events_broker_insertion(config, database): - a = AgregateTest(test_id=1, test=2) +async def test_events_broker_insertion(config): + a = AggregateTest(test_id=1, test=2) m = MinosEventBroker("EventBroker", config) - await m.send(a) + affected_rows, queue_id = await m.send(a) + + assert affected_rows == 1 + assert queue_id > 0 + """ cur = await database.cursor() await cur.execute("SELECT 1 FROM queue WHERE topic = 'EventBroker' LIMIT 1;") @@ -55,32 +68,20 @@ async def test_events_broker_insertion(config, database): database.close() assert ret == [(1,)] + """ -async def test_commands_broker_insertion(config, database): - a = AgregateTest(test_id=1, test=2) +async def test_commands_broker_insertion(config): + a = AggregateTest(test_id=1, test=2) m = MinosCommandBroker("CommandBroker", config) - await m.send(model=a, callback="test") - cur = await database.cursor() - - await cur.execute("SELECT 1 FROM queue WHERE topic = 'CommandBroker' LIMIT 1;") - ret = [] - async for row in cur: - ret.append(row) - - database.close() - assert ret == [(1,)] + affected_rows, queue_id = await m.send(model=a, callback="test") + assert affected_rows == 1 + assert queue_id > 0 """ -async def test_queue_dispatcher(config): - d = Dispatcher(config) - await d.run() - - - async def test_drop_database(database): cur = await database.cursor() await cur.execute("DROP TABLE IF EXISTS queue;") diff --git a/tests/test_config.yaml b/tests/test_config.yaml index 01e79614..f04cdc0f 100644 --- a/tests/test_config.yaml +++ b/tests/test_config.yaml @@ -22,6 +22,13 @@ events: - name: TicketDeleted controller: tests.services.CqrsTestService.CqrsService action: ticket_deleted + queue: + database: broker_db + user: broker + password: br0k3r + host: postgres + port: 5432 + records: 10 commands: broker: localhost port: 9092 @@ -38,3 +45,10 @@ commands: - name: GetOrder controller: minos.service.OrderService action: get_order + queue: + database: broker_db + user: broker + password: br0k3r + host: postgres + port: 5432 + records: 10 diff --git a/tests/test_config_.yaml b/tests/test_config_.yaml deleted file mode 100644 index 7a4fd137..00000000 --- a/tests/test_config_.yaml +++ /dev/null @@ -1,60 +0,0 @@ -service: - name: Order -rest: - host: localhost - port: 8900 - endpoints: - - name: AddOrder - route: /order - method: POST - controller: minos.services.OrderService - action: add_order -events: - broker: localhost - port: 9092 - database: - path: ./tests/local_db.lmdb - name: database_events_test - items: - - name: TicketAdded - controller: minos.services.CQRSService - action: ticket_added - - name: TicketDeleted - controller: minos.services.CQRSService - action: ticket_deleted - - name: OrderAdded - controller: minos.services.CQRSService - action: order_added - queue: - database: broker_db - user: broker - password: br0k3r - host: postgres - port: 5432 - records: 10 -commands: - broker: localhost - port: 9092 - database: - path: ./tests/local_db.lmdb - name: database_commands_test - items: - - name: AddOrder - controller: minos.services.OrderService - action: add_order - - name: DeleteOrder - controller: minos.services.OrderService - action: delete_order - - name: UpdateOrder - controller: minos.services.OrderService - action: update_order - - name: GetOrder - controller: minos.service.OrderService - action: get_order - queue: - database: broker_db - user: broker - password: br0k3r - host: postgres - port: 5432 - records: 10 From 1328bee89550cb4c3b2bc6cb4cba25cd35b0b1e9 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 22 Apr 2021 09:15:24 +0000 Subject: [PATCH 045/239] Restyled by black --- minos/networks/broker.py | 3 --- tests/test_broker.py | 16 +++++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index a9dfc2a0..4c9d0be4 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -173,6 +173,3 @@ async def _send_to_kafka(self, topic: str, message: bytes): await producer.stop() return flag - - - diff --git a/tests/test_broker.py b/tests/test_broker.py index 2309ff44..fd0b0829 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -4,9 +4,14 @@ import asyncio from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import (Aggregate, BrokerDatabaseInitializer, - MinosBrokerDatabase, MinosCommandBroker, - MinosEventBroker, EventBrokerQueueDispatcher) +from minos.networks.broker import ( + Aggregate, + BrokerDatabaseInitializer, + MinosBrokerDatabase, + MinosCommandBroker, + MinosEventBroker, + EventBrokerQueueDispatcher, +) @pytest.fixture @@ -16,10 +21,7 @@ def config(): @pytest.fixture def services(config): - return [ - BrokerDatabaseInitializer(config=config), - EventBrokerQueueDispatcher(interval=0.5, delay=0, config=config) - ] + return [BrokerDatabaseInitializer(config=config), EventBrokerQueueDispatcher(interval=0.5, delay=0, config=config)] @pytest.fixture From 6907c544186874c79850adcb0728aec75e346b59 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 22 Apr 2021 09:15:25 +0000 Subject: [PATCH 046/239] Restyled by isort --- tests/test_broker.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/test_broker.py b/tests/test_broker.py index fd0b0829..c8478dea 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -1,17 +1,13 @@ +import asyncio import time import pytest -import asyncio from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import ( - Aggregate, - BrokerDatabaseInitializer, - MinosBrokerDatabase, - MinosCommandBroker, - MinosEventBroker, - EventBrokerQueueDispatcher, -) +from minos.networks.broker import (Aggregate, BrokerDatabaseInitializer, + EventBrokerQueueDispatcher, + MinosBrokerDatabase, MinosCommandBroker, + MinosEventBroker) @pytest.fixture From 5adabaaec5805b941a14816d8bf44a5838be5a60 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 22 Apr 2021 12:48:44 +0200 Subject: [PATCH 047/239] Add pytest-asyncio Minos Broker Tests #38 --- minos/networks/broker.py | 107 ++++++++++++++++++--------------------- requirements_dev.txt | 4 +- tests/test_broker.py | 4 ++ 3 files changed, 56 insertions(+), 59 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 4c9d0be4..2b98a60e 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -20,8 +20,8 @@ class MinosBrokerDatabase: - async def get_connection(self, conf): - conn = await aiopg.connect( + def get_connection(self, conf): + conn = aiopg.connect( database=conf.events.queue.database, user=conf.events.queue.user, password=conf.events.queue.password, @@ -65,17 +65,15 @@ async def send(self, model: Aggregate): event_instance = EventModel(topic=self.topic, model="Change", items=[str(model)]) bin_data = event_instance.avro_bytes - conn = await MinosBrokerDatabase().get_connection(self.config) + async with MinosBrokerDatabase().get_connection(self.config) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO queue (topic, model, retry, creation_date, update_date) VALUES (%s, %s, %s, %s, %s) RETURNING queue_id;", + (event_instance.topic, bin_data, 0, datetime.datetime.now(), datetime.datetime.now(),), + ) - cur = await conn.cursor() - await cur.execute( - "INSERT INTO queue (topic, model, retry, creation_date, update_date) VALUES (%s, %s, %s, %s, %s) RETURNING queue_id;", - (event_instance.topic, bin_data, 0, datetime.datetime.now(), datetime.datetime.now(),), - ) - - queue_id = await cur.fetchone() - affected_rows = cur.rowcount - conn.close() + queue_id = await cur.fetchone() + affected_rows = cur.rowcount return affected_rows, queue_id[0] @@ -94,17 +92,15 @@ async def send(self, model: Aggregate, callback: t.Callable): event_instance = CommandModel(topic=self.topic, model="Change", items=[str(model)], callback=callback) bin_data = event_instance.avro_bytes - conn = await MinosBrokerDatabase().get_connection(self.config) - - cur = await conn.cursor() - await cur.execute( - "INSERT INTO queue (topic, model, retry, creation_date, update_date) VALUES (%s, %s, %s, %s, %s) RETURNING queue_id;", - (event_instance.topic, bin_data, 0, datetime.datetime.now(), datetime.datetime.now(),), - ) + async with MinosBrokerDatabase().get_connection(self.config) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO queue (topic, model, retry, creation_date, update_date) VALUES (%s, %s, %s, %s, %s) RETURNING queue_id;", + (event_instance.topic, bin_data, 0, datetime.datetime.now(), datetime.datetime.now(),), + ) - queue_id = await cur.fetchone() - affected_rows = cur.rowcount - conn.close() + queue_id = await cur.fetchone() + affected_rows = cur.rowcount return affected_rows, queue_id[0] @@ -114,48 +110,43 @@ async def start(self): # Send signal to entrypoint for continue running self.start_event.set() - conn = await MinosBrokerDatabase().get_connection(self.config) - - cur = await conn.cursor() - await cur.execute( - 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, ' - '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' - '"creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' - ) - - conn.close() + async with MinosBrokerDatabase().get_connection(self.config) as connect: + async with connect.cursor() as cur: + await cur.execute( + 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, ' + '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' + '"creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' + ) await self.stop(self) class EventBrokerQueueDispatcher(PeriodicService): async def callback(self): - conn = await MinosBrokerDatabase().get_connection(self.config) - - async with conn.cursor() as cur: - await cur.execute("SELECT * FROM queue WHERE retry <= 2 ORDER BY creation_date ASC LIMIT 10;") - async for row in cur: - - log.debug(row) - log.debug("id = %s", row[0]) - log.debug("topic = %s", row[1]) - log.debug("model = %s", row[2]) - log.debug("retry = %s", row[3]) - log.debug("creation_date = %s", row[4]) - log.debug("update_date = %s", row[5]) - - sent_to_kafka = await self._send_to_kafka(topic=row[1], message=row[2]) - if sent_to_kafka: - # Delete from database If the event was sent successfully to Kafka. - async with conn.cursor() as cur2: - await cur2.execute("DELETE FROM queue WHERE queue_id=%d;" % row[0]) - else: - # Update queue retry column. Increase by 1. - async with conn.cursor() as cur3: - await cur3.execute("UPDATE queue SET retry = retry + 1 WHERE queue_id=%d;" % row[0]) - - conn.commit() - conn.close() + async with MinosBrokerDatabase().get_connection(self.config) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT * FROM queue WHERE retry <= 2 ORDER BY creation_date ASC LIMIT 10;") + async for row in cur: + + log.debug(row) + log.debug("id = %s", row[0]) + log.debug("topic = %s", row[1]) + log.debug("model = %s", row[2]) + log.debug("retry = %s", row[3]) + log.debug("creation_date = %s", row[4]) + log.debug("update_date = %s", row[5]) + + sent_to_kafka = await self._send_to_kafka(topic=row[1], message=row[2]) + if sent_to_kafka: + # Delete from database If the event was sent successfully to Kafka. + async with connect.cursor() as cur2: + await cur2.execute("DELETE FROM queue WHERE queue_id=%d;" % row[0]) + else: + # Update queue retry column. Increase by 1. + async with connect.cursor() as cur3: + await cur3.execute("UPDATE queue SET retry = retry + 1 WHERE queue_id=%d;" % row[0]) + + connect.commit() async def _send_to_kafka(self, topic: str, message: bytes): flag = False diff --git a/requirements_dev.txt b/requirements_dev.txt index ba0dc533..c2cf2c7e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -24,6 +24,7 @@ flake8==3.9.0 idna==2.10 imagesize==1.2.0 importlib-metadata==3.10.1 +iniconfig==1.1.1 Jinja2==2.11.3 kafka-python==2.0.2 keyring==23.0.1 @@ -46,7 +47,8 @@ pycodestyle==2.7.0 pyflakes==2.3.1 Pygments==2.8.1 pyparsing==2.4.7 -pytest==4.6.5 +pytest==6.2.3 +pytest-asyncio==0.15.1 pytest-runner==5.1 pytz==2021.1 PyYAML==5.4.1 diff --git a/tests/test_broker.py b/tests/test_broker.py index c8478dea..75e89306 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -25,11 +25,13 @@ async def database(config): return await MinosBrokerDatabase().get_connection(config) +@pytest.mark.asyncio async def test_database_connection(database): assert database.closed == 0 database.close() +@pytest.mark.asyncio async def test_if_queue_table_exists(database): time.sleep(1) cur = await database.cursor() @@ -47,6 +49,7 @@ class AggregateTest(Aggregate): test: int +@pytest.mark.asyncio async def test_events_broker_insertion(config): a = AggregateTest(test_id=1, test=2) @@ -69,6 +72,7 @@ async def test_events_broker_insertion(config): """ +@pytest.mark.asyncio async def test_commands_broker_insertion(config): a = AggregateTest(test_id=1, test=2) From dca0cf17abf3aa3e31d0e4e7dec1238ef1013e3c Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 22 Apr 2021 17:18:02 +0200 Subject: [PATCH 048/239] Restructure tests Minos Broker Tests #38 --- minos/networks/broker.py | 20 ++++--- tests/broker/__init__.py | 0 tests/broker/conftest.py | 19 ++++++ tests/broker/database_testcase.py | 66 +++++++++++++++++++++ tests/broker/test_broker.py | 60 +++++++++++++++++++ tests/test_broker.py | 97 ------------------------------- 6 files changed, 156 insertions(+), 106 deletions(-) create mode 100644 tests/broker/__init__.py create mode 100644 tests/broker/conftest.py create mode 100644 tests/broker/database_testcase.py create mode 100644 tests/broker/test_broker.py delete mode 100644 tests/test_broker.py diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 2b98a60e..805e82cc 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -105,18 +105,22 @@ async def send(self, model: Aggregate, callback: t.Callable): return affected_rows, queue_id[0] +async def broker_table_creation(config: MinosConfig): + async with MinosBrokerDatabase().get_connection(config) as connect: + async with connect.cursor() as cur: + await cur.execute( + 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, ' + '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' + '"creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' + ) + + class BrokerDatabaseInitializer(Service): async def start(self): # Send signal to entrypoint for continue running self.start_event.set() - async with MinosBrokerDatabase().get_connection(self.config) as connect: - async with connect.cursor() as cur: - await cur.execute( - 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, ' - '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' - '"creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' - ) + await broker_table_creation(self.config) await self.stop(self) @@ -146,8 +150,6 @@ async def callback(self): async with connect.cursor() as cur3: await cur3.execute("UPDATE queue SET retry = retry + 1 WHERE queue_id=%d;" % row[0]) - connect.commit() - async def _send_to_kafka(self, topic: str, message: bytes): flag = False producer = AIOKafkaProducer(bootstrap_servers="localhost:9092") diff --git a/tests/broker/__init__.py b/tests/broker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/broker/conftest.py b/tests/broker/conftest.py new file mode 100644 index 00000000..b70f3065 --- /dev/null +++ b/tests/broker/conftest.py @@ -0,0 +1,19 @@ +import pytest + +from minos.common.configuration.config import MinosConfig +from minos.networks.broker import MinosBrokerDatabase, BrokerDatabaseInitializer, EventBrokerQueueDispatcher + + +@pytest.fixture +def broker_config(): + return MinosConfig(path="./tests/test_config.yaml") + + +@pytest.fixture +async def database(broker_config): + return await MinosBrokerDatabase().get_connection(broker_config) + + +@pytest.fixture +def services(broker_config): + return [BrokerDatabaseInitializer(config=broker_config), EventBrokerQueueDispatcher(interval=0.5, delay=0, config=broker_config)] diff --git a/tests/broker/database_testcase.py b/tests/broker/database_testcase.py new file mode 100644 index 00000000..7dce924e --- /dev/null +++ b/tests/broker/database_testcase.py @@ -0,0 +1,66 @@ +""" +Copyright (C) 2021 Clariteia SL +This file is part of minos framework. +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +import os +import unittest + +import aiopg + +from minos.common.configuration.config import MinosConfig +from minos.networks.broker import MinosBrokerDatabase, broker_table_creation + + +class PostgresAsyncTestCase(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self._meta_kwargs = { + "host": os.getenv("POSTGRES_HOST", "localhost"), + "port": os.getenv("POSTGRES_PORT", 5432), + "database": os.getenv("POSTGRES_DATABASE", "postgres"), + "user": os.getenv("POSTGRES_USER", "postgres"), + "password": os.getenv("POSTGRES_PASSWORD", ""), + } + + self.kwargs = self._meta_kwargs | { + "database": "broker_db", + "user": "broker", + "password": "br0k3r", + } + + async def asyncSetUp(self): + await broker_table_creation(self._broker_config()) + """ + async with aiopg.connect(**self._meta_kwargs) as connection: + async with connection.cursor() as cursor: + + template = "DROP DATABASE IF EXISTS {database};" + await cursor.execute(template.format(**self.kwargs)) + + template = "DROP ROLE IF EXISTS {user};" + await cursor.execute(template.format(**self.kwargs)) + + template = "CREATE ROLE {user} WITH CREATEDB LOGIN ENCRYPTED PASSWORD {password!r};" + await cursor.execute(template.format(**self.kwargs)) + + template = "CREATE DATABASE {database} WITH OWNER = {user};" + await cursor.execute(template.format(**self.kwargs)) + """ + + @staticmethod + def _broker_config(): + return MinosConfig(path="./tests/test_config.yaml") + + async def _database(self): + return await MinosBrokerDatabase().get_connection(self._broker_config()) + + async def asyncTearDown(self): + pass + """ + database = await self._database() + async with database as connection: + async with connection.cursor() as cursor: + template = "DROP TABLE IF EXISTS queue" + await cursor.execute(template) + """ + diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py new file mode 100644 index 00000000..71b80f07 --- /dev/null +++ b/tests/broker/test_broker.py @@ -0,0 +1,60 @@ +import asyncio +import time + +import pytest + +from minos.common.logs import log +from minos.networks.broker import (Aggregate, MinosCommandBroker, MinosEventBroker) +from tests.broker.database_testcase import ( + PostgresAsyncTestCase, +) + + +class AggregateTest(Aggregate): + test: int + + +class TestPostgreSqlMinosBroker(PostgresAsyncTestCase): + async def test_database_connection(self): + database = await self._database() + async with database as connect: + assert database.closed == 0 + + async def test_if_queue_table_exists(self): + database = await self._database() + async with database as connect: + async with connect.cursor() as cur: + + await cur.execute("SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'queue';") + ret = [] + async for row in cur: + ret.append(row) + + assert ret == [(1,)] + + async def test_events_broker_insertion(self): + a = AggregateTest(test_id=1, test=2) + + m = MinosEventBroker("EventBroker", self._broker_config()) + + affected_rows, queue_id = await m.send(a) + + assert affected_rows == 1 + assert queue_id > 0 + + async def test_commands_broker_insertion(self): + a = AggregateTest(test_id=1, test=2) + + m = MinosCommandBroker("CommandBroker", self._broker_config()) + + affected_rows, queue_id = await m.send(model=a, callback="test") + assert affected_rows == 1 + assert queue_id > 0 + + + +# create role broker with createdb login password 'br0k3r'; +# CREATE DATABASE broker_db OWNER broker; +# 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, "topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL)', [] +# 'SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = %s ORDER BY tablename', ('public',) +# 'DROP TABLE IF EXISTS "queue"' diff --git a/tests/test_broker.py b/tests/test_broker.py deleted file mode 100644 index 75e89306..00000000 --- a/tests/test_broker.py +++ /dev/null @@ -1,97 +0,0 @@ -import asyncio -import time - -import pytest -from minos.common.configuration.config import MinosConfig -from minos.common.logs import log -from minos.networks.broker import (Aggregate, BrokerDatabaseInitializer, - EventBrokerQueueDispatcher, - MinosBrokerDatabase, MinosCommandBroker, - MinosEventBroker) - - -@pytest.fixture -def config(): - return MinosConfig(path="./tests/test_config.yaml") - - -@pytest.fixture -def services(config): - return [BrokerDatabaseInitializer(config=config), EventBrokerQueueDispatcher(interval=0.5, delay=0, config=config)] - - -@pytest.fixture -async def database(config): - return await MinosBrokerDatabase().get_connection(config) - - -@pytest.mark.asyncio -async def test_database_connection(database): - assert database.closed == 0 - database.close() - - -@pytest.mark.asyncio -async def test_if_queue_table_exists(database): - time.sleep(1) - cur = await database.cursor() - - await cur.execute("SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'queue';") - ret = [] - async for row in cur: - ret.append(row) - - database.close() - assert ret == [(1,)] - - -class AggregateTest(Aggregate): - test: int - - -@pytest.mark.asyncio -async def test_events_broker_insertion(config): - a = AggregateTest(test_id=1, test=2) - - m = MinosEventBroker("EventBroker", config) - - affected_rows, queue_id = await m.send(a) - - assert affected_rows == 1 - assert queue_id > 0 - """ - cur = await database.cursor() - - await cur.execute("SELECT 1 FROM queue WHERE topic = 'EventBroker' LIMIT 1;") - ret = [] - async for row in cur: - ret.append(row) - - database.close() - assert ret == [(1,)] - """ - - -@pytest.mark.asyncio -async def test_commands_broker_insertion(config): - a = AggregateTest(test_id=1, test=2) - - m = MinosCommandBroker("CommandBroker", config) - - affected_rows, queue_id = await m.send(model=a, callback="test") - assert affected_rows == 1 - assert queue_id > 0 - - -""" -async def test_drop_database(database): - cur = await database.cursor() - await cur.execute("DROP TABLE IF EXISTS queue;") - database.close() -""" - -# create role broker with createdb login password 'br0k3r'; -# CREATE DATABASE broker_db OWNER broker; -# 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, "topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL)', [] -# 'SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = %s ORDER BY tablename', ('public',) -# 'DROP TABLE IF EXISTS "queue"' From 1b5c046beb2e92ee46625e1f44f47eed225df849 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 22 Apr 2021 21:36:43 +0200 Subject: [PATCH 049/239] Kafka Consumer Service Test --- tests/broker/conftest.py | 33 ++++++++++++++++++++++++++++++++- tests/broker/test_broker.py | 3 +++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/broker/conftest.py b/tests/broker/conftest.py index b70f3065..1b13895c 100644 --- a/tests/broker/conftest.py +++ b/tests/broker/conftest.py @@ -2,6 +2,9 @@ from minos.common.configuration.config import MinosConfig from minos.networks.broker import MinosBrokerDatabase, BrokerDatabaseInitializer, EventBrokerQueueDispatcher +from aiomisc.service.periodic import Service +from aiokafka import AIOKafkaConsumer +from minos.common.logs import log @pytest.fixture @@ -16,4 +19,32 @@ async def database(broker_config): @pytest.fixture def services(broker_config): - return [BrokerDatabaseInitializer(config=broker_config), EventBrokerQueueDispatcher(interval=0.5, delay=0, config=broker_config)] + return [KafkaConsumer(), BrokerDatabaseInitializer(config=broker_config), EventBrokerQueueDispatcher(interval=0.5, delay=0, config=broker_config)] + + +class KafkaConsumer(Service): + async def start(self): + self.start_event.set() + + consumer = AIOKafkaConsumer( + 'EventBroker', "CommandBroker", + loop=self.loop, + bootstrap_servers='localhost:9092') + # Get cluster layout and join group `my-group` + + await consumer.start() + try: + # Consume messages + async for msg in consumer: + log.debug("+++++++++++++++++++++++++++++++++++++++++++++++") + log.debug(msg.topic) + log.debug(msg.key) + log.debug(msg.value) + log.debug("++++++++++++++++++++++++++++++++++++++++++++++++") + #log.debug("consumed: ", msg.topic, msg.partition, msg.offset, + # msg.key, msg.value, msg.timestamp) + finally: + # Will leave consumer group; perform autocommit if enabled. + await consumer.stop() + + await self.stop(self) diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index 71b80f07..3523e949 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -3,6 +3,7 @@ import pytest +from aiokafka import AIOKafkaConsumer from minos.common.logs import log from minos.networks.broker import (Aggregate, MinosCommandBroker, MinosEventBroker) from tests.broker.database_testcase import ( @@ -15,6 +16,7 @@ class AggregateTest(Aggregate): class TestPostgreSqlMinosBroker(PostgresAsyncTestCase): + async def test_database_connection(self): database = await self._database() async with database as connect: @@ -53,6 +55,7 @@ async def test_commands_broker_insertion(self): + # create role broker with createdb login password 'br0k3r'; # CREATE DATABASE broker_db OWNER broker; # 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, "topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL)', [] From e6e352c167d473b0f3a56b88a748fbd26885f2ec Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 23 Apr 2021 10:43:20 +0200 Subject: [PATCH 050/239] Remove Kafka Consumer tests and refactor code --- minos/networks/broker.py | 74 +++++++++++++++++++--------------------- tests/broker/conftest.py | 4 +-- 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 805e82cc..e2fb8b62 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -125,44 +125,40 @@ async def start(self): await self.stop(self) +async def send_to_kafka(topic: str, message: bytes, config: MinosConfig): + flag = False + log.debug(config.events.broker) + producer = AIOKafkaProducer(bootstrap_servers="{host}:{port}".format(host=config.events.broker.host, port=config.events.broker.port)) + # Get cluster layout and initial topic/partition leadership information + await producer.start() + try: + # Produce message + await producer.send_and_wait(topic, message) + flag = True + finally: + # Wait for all pending messages to be delivered or expire. + await producer.stop() + + return flag + + +async def broker_queue_dispatcher(config: MinosConfig): + async with MinosBrokerDatabase().get_connection(config) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT * FROM queue WHERE retry <= 2 ORDER BY creation_date ASC LIMIT 10;") + async for row in cur: + sent_to_kafka = await send_to_kafka(topic=row[1], message=row[2], config=config) + if sent_to_kafka: + # Delete from database If the event was sent successfully to Kafka. + async with connect.cursor() as cur2: + await cur2.execute("DELETE FROM queue WHERE queue_id=%d;" % row[0]) + else: + # Update queue retry column. Increase by 1. + async with connect.cursor() as cur3: + await cur3.execute("UPDATE queue SET retry = retry + 1 WHERE queue_id=%d;" % row[0]) + + class EventBrokerQueueDispatcher(PeriodicService): async def callback(self): - async with MinosBrokerDatabase().get_connection(self.config) as connect: - async with connect.cursor() as cur: - await cur.execute("SELECT * FROM queue WHERE retry <= 2 ORDER BY creation_date ASC LIMIT 10;") - async for row in cur: - - log.debug(row) - log.debug("id = %s", row[0]) - log.debug("topic = %s", row[1]) - log.debug("model = %s", row[2]) - log.debug("retry = %s", row[3]) - log.debug("creation_date = %s", row[4]) - log.debug("update_date = %s", row[5]) - - sent_to_kafka = await self._send_to_kafka(topic=row[1], message=row[2]) - if sent_to_kafka: - # Delete from database If the event was sent successfully to Kafka. - async with connect.cursor() as cur2: - await cur2.execute("DELETE FROM queue WHERE queue_id=%d;" % row[0]) - else: - # Update queue retry column. Increase by 1. - async with connect.cursor() as cur3: - await cur3.execute("UPDATE queue SET retry = retry + 1 WHERE queue_id=%d;" % row[0]) - - async def _send_to_kafka(self, topic: str, message: bytes): - flag = False - producer = AIOKafkaProducer(bootstrap_servers="localhost:9092") - # Get cluster layout and initial topic/partition leadership information - await producer.start() - try: - # Produce message - await producer.send_and_wait(topic, message) - flag = True - except Exception as e: - flag = False - finally: - # Wait for all pending messages to be delivered or expire. - await producer.stop() - - return flag + await broker_queue_dispatcher(self.config) + diff --git a/tests/broker/conftest.py b/tests/broker/conftest.py index 1b13895c..1aca94ab 100644 --- a/tests/broker/conftest.py +++ b/tests/broker/conftest.py @@ -19,7 +19,7 @@ async def database(broker_config): @pytest.fixture def services(broker_config): - return [KafkaConsumer(), BrokerDatabaseInitializer(config=broker_config), EventBrokerQueueDispatcher(interval=0.5, delay=0, config=broker_config)] + return [BrokerDatabaseInitializer(config=broker_config), EventBrokerQueueDispatcher(interval=0.5, delay=0, config=broker_config)] class KafkaConsumer(Service): @@ -46,5 +46,5 @@ async def start(self): finally: # Will leave consumer group; perform autocommit if enabled. await consumer.stop() - + await self.stop(self) From ea11913301a53f4d8a57030ec1b368dc200ee7dc Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 23 Apr 2021 13:43:56 +0200 Subject: [PATCH 051/239] Tests Kafka Exceptions handling --- minos/networks/broker.py | 27 ++++----- tests/broker/test_broker.py | 110 ++++++++++++++++++++++++++++++++++- tests/wrong_test_config.yaml | 54 +++++++++++++++++ 3 files changed, 176 insertions(+), 15 deletions(-) create mode 100644 tests/wrong_test_config.yaml diff --git a/minos/networks/broker.py b/minos/networks/broker.py index e2fb8b62..f0aaa6a7 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -60,8 +60,6 @@ def _database(self): pass async def send(self, model: Aggregate): - - # TODO: Change event_instance = EventModel(topic=self.topic, model="Change", items=[str(model)]) bin_data = event_instance.avro_bytes @@ -88,7 +86,6 @@ def _database(self): pass async def send(self, model: Aggregate, callback: t.Callable): - # TODO: Change event_instance = CommandModel(topic=self.topic, model="Change", items=[str(model)], callback=callback) bin_data = event_instance.avro_bytes @@ -127,7 +124,6 @@ async def start(self): async def send_to_kafka(topic: str, message: bytes, config: MinosConfig): flag = False - log.debug(config.events.broker) producer = AIOKafkaProducer(bootstrap_servers="{host}:{port}".format(host=config.events.broker.host, port=config.events.broker.port)) # Get cluster layout and initial topic/partition leadership information await producer.start() @@ -147,15 +143,20 @@ async def broker_queue_dispatcher(config: MinosConfig): async with connect.cursor() as cur: await cur.execute("SELECT * FROM queue WHERE retry <= 2 ORDER BY creation_date ASC LIMIT 10;") async for row in cur: - sent_to_kafka = await send_to_kafka(topic=row[1], message=row[2], config=config) - if sent_to_kafka: - # Delete from database If the event was sent successfully to Kafka. - async with connect.cursor() as cur2: - await cur2.execute("DELETE FROM queue WHERE queue_id=%d;" % row[0]) - else: - # Update queue retry column. Increase by 1. - async with connect.cursor() as cur3: - await cur3.execute("UPDATE queue SET retry = retry + 1 WHERE queue_id=%d;" % row[0]) + sent_to_kafka = False + try: + sent_to_kafka = await send_to_kafka(topic=row[1], message=row[2], config=config) + if sent_to_kafka: + # Delete from database If the event was sent successfully to Kafka. + async with connect.cursor() as cur2: + await cur2.execute("DELETE FROM queue WHERE queue_id=%d;" % row[0]) + except: + sent_to_kafka = False + finally: + if not sent_to_kafka: + # Update queue retry column. Increase by 1. + async with connect.cursor() as cur3: + await cur3.execute("UPDATE queue SET retry = retry + 1 WHERE queue_id=%d;" % row[0]) class EventBrokerQueueDispatcher(PeriodicService): diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index 3523e949..d3d5cbcc 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -3,9 +3,9 @@ import pytest -from aiokafka import AIOKafkaConsumer from minos.common.logs import log -from minos.networks.broker import (Aggregate, MinosCommandBroker, MinosEventBroker) +from minos.common.configuration.config import MinosConfig +from minos.networks.broker import (Aggregate, MinosCommandBroker, MinosEventBroker, broker_queue_dispatcher, send_to_kafka) from tests.broker.database_testcase import ( PostgresAsyncTestCase, ) @@ -44,6 +44,58 @@ async def test_events_broker_insertion(self): assert affected_rows == 1 assert queue_id > 0 + async def test_if_events_was_deleted(self): + a = AggregateTest(test_id=1, test=2) + m = MinosEventBroker("EventBroker-Delete", self._broker_config()) + affected_rows_1, queue_id_1 = await m.send(a) + affected_rows_2, queue_id_2 = await m.send(a) + + await broker_queue_dispatcher(self._broker_config()) + + database = await self._database() + async with database as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "EventBroker-Delete") + records = await cur.fetchone() + + assert affected_rows_1 == 1 + assert queue_id_1 > 0 + assert affected_rows_2 == 1 + assert queue_id_2 > 0 + assert records[0] == 0 + + async def test_if_events_retry_was_incremented(self): + a = AggregateTest(test_id=1, test=2) + m = MinosEventBroker("EventBroker-Delete", self._broker_config()) + affected_rows_1, queue_id_1 = await m.send(a) + affected_rows_2, queue_id_2 = await m.send(a) + + config = MinosConfig(path="./tests/wrong_test_config.yaml") + + await broker_queue_dispatcher(config) + + database = await self._database() + async with database as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "EventBroker-Delete") + records = await cur.fetchone() + + await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_1) + retry_1 = await cur.fetchone() + + await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_2) + retry_2 = await cur.fetchone() + + assert affected_rows_1 == 1 + assert queue_id_1 > 0 + assert affected_rows_2 == 1 + assert queue_id_2 > 0 + assert records[0] == 2 + assert retry_1[0] > 0 + assert retry_2[0] > 0 + async def test_commands_broker_insertion(self): a = AggregateTest(test_id=1, test=2) @@ -53,7 +105,61 @@ async def test_commands_broker_insertion(self): assert affected_rows == 1 assert queue_id > 0 + async def test_if_commands_was_deleted(self): + a = AggregateTest(test_id=1, test=2) + m = MinosCommandBroker("CommandBroker-Delete", self._broker_config()) + affected_rows_1, queue_id_1 = await m.send(a, callback="test1") + affected_rows_2, queue_id_2 = await m.send(a, callback="test2") + + await broker_queue_dispatcher(self._broker_config()) + + database = await self._database() + async with database as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "CommandBroker-Delete") + records = await cur.fetchone() + + assert affected_rows_1 == 1 + assert queue_id_1 > 0 + assert affected_rows_2 == 1 + assert queue_id_2 > 0 + assert records[0] == 0 + + async def test_if_commands_retry_was_incremented(self): + a = AggregateTest(test_id=1, test=2) + m = MinosCommandBroker("CommandBroker-Delete", self._broker_config()) + affected_rows_1, queue_id_1 = await m.send(a, callback="test1") + affected_rows_2, queue_id_2 = await m.send(a, callback="test2") + + config = MinosConfig(path="./tests/wrong_test_config.yaml") + await broker_queue_dispatcher(config) + + database = await self._database() + async with database as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "CommandBroker-Delete") + records = await cur.fetchone() + + await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_1) + retry_1 = await cur.fetchone() + + await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_2) + retry_2 = await cur.fetchone() + + assert affected_rows_1 == 1 + assert queue_id_1 > 0 + assert affected_rows_2 == 1 + assert queue_id_2 > 0 + assert records[0] == 2 + assert retry_1[0] > 0 + assert retry_2[0] > 0 + + async def test_send_to_kafka_ok(self): + response = await send_to_kafka(topic="TestKafkaSend", message=bytes(), config=self._broker_config()) + assert response is True # create role broker with createdb login password 'br0k3r'; diff --git a/tests/wrong_test_config.yaml b/tests/wrong_test_config.yaml new file mode 100644 index 00000000..f9943353 --- /dev/null +++ b/tests/wrong_test_config.yaml @@ -0,0 +1,54 @@ +service: + name: Order +rest: + host: localhost + port: 8900 + endpoints: + - name: AddOrder + route: /order + method: POST + controller: minos.services.OrderService + action: add_order +events: + broker: localhost + port: 4092 + database: + path: ./tests/local_db.lmdb + name: database_events_test + items: + - name: TicketAdded + controller: tests.services.CqrsTestService.CqrsService + action: ticket_added + - name: TicketDeleted + controller: tests.services.CqrsTestService.CqrsService + action: ticket_deleted + queue: + database: broker_db + user: broker + password: br0k3r + host: localhost + port: 5432 + records: 10 +commands: + broker: localhost + port: 4092 + items: + - name: AddOrder + controller: minos.services.OrderService + action: add_order + - name: DeleteOrder + controller: minos.services.OrderService + action: delete_order + - name: UpdateOrder + controller: minos.services.OrderService + action: update_order + - name: GetOrder + controller: minos.service.OrderService + action: get_order + queue: + database: broker_db + user: broker + password: br0k3r + host: localhost + port: 5432 + records: 10 From 019924b1534df9d301a2e3cf73bb4793d95463db Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 23 Apr 2021 13:53:31 +0200 Subject: [PATCH 052/239] Update wrong_test_config.yaml --- tests/wrong_test_config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/wrong_test_config.yaml b/tests/wrong_test_config.yaml index f9943353..6013672e 100644 --- a/tests/wrong_test_config.yaml +++ b/tests/wrong_test_config.yaml @@ -26,7 +26,7 @@ events: database: broker_db user: broker password: br0k3r - host: localhost + host: postgres port: 5432 records: 10 commands: @@ -49,6 +49,6 @@ commands: database: broker_db user: broker password: br0k3r - host: localhost + host: postgres port: 5432 records: 10 From 51f2e10adadd7950be6fcfae3e14d4bf9d1d50b6 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 23 Apr 2021 14:33:30 +0200 Subject: [PATCH 053/239] Update test_broker.py --- tests/broker/test_broker.py | 68 ++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index d3d5cbcc..959b9c62 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -34,6 +34,10 @@ async def test_if_queue_table_exists(self): assert ret == [(1,)] + async def test_send_to_kafka_ok(self): + response = await send_to_kafka(topic="TestKafkaSend", message=bytes(), config=self._broker_config()) + assert response is True + async def test_events_broker_insertion(self): a = AggregateTest(test_id=1, test=2) @@ -65,37 +69,6 @@ async def test_if_events_was_deleted(self): assert queue_id_2 > 0 assert records[0] == 0 - async def test_if_events_retry_was_incremented(self): - a = AggregateTest(test_id=1, test=2) - m = MinosEventBroker("EventBroker-Delete", self._broker_config()) - affected_rows_1, queue_id_1 = await m.send(a) - affected_rows_2, queue_id_2 = await m.send(a) - - config = MinosConfig(path="./tests/wrong_test_config.yaml") - - await broker_queue_dispatcher(config) - - database = await self._database() - async with database as connect: - async with connect.cursor() as cur: - await cur.execute( - "SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "EventBroker-Delete") - records = await cur.fetchone() - - await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_1) - retry_1 = await cur.fetchone() - - await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_2) - retry_2 = await cur.fetchone() - - assert affected_rows_1 == 1 - assert queue_id_1 > 0 - assert affected_rows_2 == 1 - assert queue_id_2 > 0 - assert records[0] == 2 - assert retry_1[0] > 0 - assert retry_2[0] > 0 - async def test_commands_broker_insertion(self): a = AggregateTest(test_id=1, test=2) @@ -157,9 +130,36 @@ async def test_if_commands_retry_was_incremented(self): assert retry_1[0] > 0 assert retry_2[0] > 0 - async def test_send_to_kafka_ok(self): - response = await send_to_kafka(topic="TestKafkaSend", message=bytes(), config=self._broker_config()) - assert response is True + async def test_if_events_retry_was_incremented(self): + a = AggregateTest(test_id=1, test=2) + m = MinosEventBroker("EventBroker-Delete", self._broker_config()) + affected_rows_1, queue_id_1 = await m.send(a) + affected_rows_2, queue_id_2 = await m.send(a) + + config = MinosConfig(path="./tests/wrong_test_config.yaml") + + await broker_queue_dispatcher(config) + + database = await self._database() + async with database as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "EventBroker-Delete") + records = await cur.fetchone() + + await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_1) + retry_1 = await cur.fetchone() + + await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_2) + retry_2 = await cur.fetchone() + + assert affected_rows_1 == 1 + assert queue_id_1 > 0 + assert affected_rows_2 == 1 + assert queue_id_2 > 0 + assert records[0] == 2 + assert retry_1[0] > 0 + assert retry_2[0] > 0 # create role broker with createdb login password 'br0k3r'; From dc8921db948164cc8a7dbef47490861d88a7e263 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 23 Apr 2021 12:33:41 +0000 Subject: [PATCH 054/239] Restyled by black --- minos/networks/broker.py | 5 +++-- tests/broker/conftest.py | 12 ++++++------ tests/broker/database_testcase.py | 1 - tests/broker/test_broker.py | 27 ++++++++++++++------------- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index f0aaa6a7..45e4205c 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -124,7 +124,9 @@ async def start(self): async def send_to_kafka(topic: str, message: bytes, config: MinosConfig): flag = False - producer = AIOKafkaProducer(bootstrap_servers="{host}:{port}".format(host=config.events.broker.host, port=config.events.broker.port)) + producer = AIOKafkaProducer( + bootstrap_servers="{host}:{port}".format(host=config.events.broker.host, port=config.events.broker.port) + ) # Get cluster layout and initial topic/partition leadership information await producer.start() try: @@ -162,4 +164,3 @@ async def broker_queue_dispatcher(config: MinosConfig): class EventBrokerQueueDispatcher(PeriodicService): async def callback(self): await broker_queue_dispatcher(self.config) - diff --git a/tests/broker/conftest.py b/tests/broker/conftest.py index 1aca94ab..d7be4342 100644 --- a/tests/broker/conftest.py +++ b/tests/broker/conftest.py @@ -19,17 +19,17 @@ async def database(broker_config): @pytest.fixture def services(broker_config): - return [BrokerDatabaseInitializer(config=broker_config), EventBrokerQueueDispatcher(interval=0.5, delay=0, config=broker_config)] + return [ + BrokerDatabaseInitializer(config=broker_config), + EventBrokerQueueDispatcher(interval=0.5, delay=0, config=broker_config), + ] class KafkaConsumer(Service): async def start(self): self.start_event.set() - consumer = AIOKafkaConsumer( - 'EventBroker', "CommandBroker", - loop=self.loop, - bootstrap_servers='localhost:9092') + consumer = AIOKafkaConsumer("EventBroker", "CommandBroker", loop=self.loop, bootstrap_servers="localhost:9092") # Get cluster layout and join group `my-group` await consumer.start() @@ -41,7 +41,7 @@ async def start(self): log.debug(msg.key) log.debug(msg.value) log.debug("++++++++++++++++++++++++++++++++++++++++++++++++") - #log.debug("consumed: ", msg.topic, msg.partition, msg.offset, + # log.debug("consumed: ", msg.topic, msg.partition, msg.offset, # msg.key, msg.value, msg.timestamp) finally: # Will leave consumer group; perform autocommit if enabled. diff --git a/tests/broker/database_testcase.py b/tests/broker/database_testcase.py index 7dce924e..2e672ad6 100644 --- a/tests/broker/database_testcase.py +++ b/tests/broker/database_testcase.py @@ -63,4 +63,3 @@ async def asyncTearDown(self): template = "DROP TABLE IF EXISTS queue" await cursor.execute(template) """ - diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index 959b9c62..cd828715 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -5,10 +5,14 @@ from minos.common.logs import log from minos.common.configuration.config import MinosConfig -from minos.networks.broker import (Aggregate, MinosCommandBroker, MinosEventBroker, broker_queue_dispatcher, send_to_kafka) -from tests.broker.database_testcase import ( - PostgresAsyncTestCase, +from minos.networks.broker import ( + Aggregate, + MinosCommandBroker, + MinosEventBroker, + broker_queue_dispatcher, + send_to_kafka, ) +from tests.broker.database_testcase import PostgresAsyncTestCase class AggregateTest(Aggregate): @@ -16,7 +20,6 @@ class AggregateTest(Aggregate): class TestPostgreSqlMinosBroker(PostgresAsyncTestCase): - async def test_database_connection(self): database = await self._database() async with database as connect: @@ -27,7 +30,9 @@ async def test_if_queue_table_exists(self): async with database as connect: async with connect.cursor() as cur: - await cur.execute("SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'queue';") + await cur.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'queue';" + ) ret = [] async for row in cur: ret.append(row) @@ -59,8 +64,7 @@ async def test_if_events_was_deleted(self): database = await self._database() async with database as connect: async with connect.cursor() as cur: - await cur.execute( - "SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "EventBroker-Delete") + await cur.execute("SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "EventBroker-Delete") records = await cur.fetchone() assert affected_rows_1 == 1 @@ -89,8 +93,7 @@ async def test_if_commands_was_deleted(self): database = await self._database() async with database as connect: async with connect.cursor() as cur: - await cur.execute( - "SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "CommandBroker-Delete") + await cur.execute("SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "CommandBroker-Delete") records = await cur.fetchone() assert affected_rows_1 == 1 @@ -112,8 +115,7 @@ async def test_if_commands_retry_was_incremented(self): database = await self._database() async with database as connect: async with connect.cursor() as cur: - await cur.execute( - "SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "CommandBroker-Delete") + await cur.execute("SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "CommandBroker-Delete") records = await cur.fetchone() await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_1) @@ -143,8 +145,7 @@ async def test_if_events_retry_was_incremented(self): database = await self._database() async with database as connect: async with connect.cursor() as cur: - await cur.execute( - "SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "EventBroker-Delete") + await cur.execute("SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "EventBroker-Delete") records = await cur.fetchone() await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_1) From 40c59e3e507b45ede5440077f8584a1b4abcfeab Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 23 Apr 2021 12:33:42 +0000 Subject: [PATCH 055/239] Restyled by isort --- tests/broker/conftest.py | 9 +++++---- tests/broker/database_testcase.py | 1 - tests/broker/test_broker.py | 13 ++++--------- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/tests/broker/conftest.py b/tests/broker/conftest.py index d7be4342..c6bc320e 100644 --- a/tests/broker/conftest.py +++ b/tests/broker/conftest.py @@ -1,10 +1,11 @@ import pytest - -from minos.common.configuration.config import MinosConfig -from minos.networks.broker import MinosBrokerDatabase, BrokerDatabaseInitializer, EventBrokerQueueDispatcher -from aiomisc.service.periodic import Service from aiokafka import AIOKafkaConsumer +from aiomisc.service.periodic import Service +from minos.common.configuration.config import MinosConfig from minos.common.logs import log +from minos.networks.broker import (BrokerDatabaseInitializer, + EventBrokerQueueDispatcher, + MinosBrokerDatabase) @pytest.fixture diff --git a/tests/broker/database_testcase.py b/tests/broker/database_testcase.py index 2e672ad6..559556b4 100644 --- a/tests/broker/database_testcase.py +++ b/tests/broker/database_testcase.py @@ -7,7 +7,6 @@ import unittest import aiopg - from minos.common.configuration.config import MinosConfig from minos.networks.broker import MinosBrokerDatabase, broker_table_creation diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index cd828715..0ae08f22 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -2,16 +2,11 @@ import time import pytest - -from minos.common.logs import log from minos.common.configuration.config import MinosConfig -from minos.networks.broker import ( - Aggregate, - MinosCommandBroker, - MinosEventBroker, - broker_queue_dispatcher, - send_to_kafka, -) +from minos.common.logs import log +from minos.networks.broker import (Aggregate, MinosCommandBroker, + MinosEventBroker, broker_queue_dispatcher, + send_to_kafka) from tests.broker.database_testcase import PostgresAsyncTestCase From 2a23406afc11eb3ccba699705b31dec67e6155be Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 23 Apr 2021 14:46:16 +0200 Subject: [PATCH 056/239] Update test_broker.py --- tests/broker/test_broker.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index 0ae08f22..6f675fda 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -157,9 +157,10 @@ async def test_if_events_retry_was_incremented(self): assert retry_1[0] > 0 assert retry_2[0] > 0 - -# create role broker with createdb login password 'br0k3r'; -# CREATE DATABASE broker_db OWNER broker; -# 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, "topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL)', [] -# 'SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = %s ORDER BY tablename', ('public',) -# 'DROP TABLE IF EXISTS "queue"' +""" +create role broker with createdb login password 'br0k3r'; +CREATE DATABASE broker_db OWNER broker; +'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, "topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL)', [] +'SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = %s ORDER BY tablename', ('public',) +'DROP TABLE IF EXISTS "queue"' +""" From 46c59fe3827e7482291ffeb065862b358caf19de Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 23 Apr 2021 14:51:23 +0200 Subject: [PATCH 057/239] Update test_config.yaml --- tests/test_config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_config.yaml b/tests/test_config.yaml index f04cdc0f..03b3dc3a 100644 --- a/tests/test_config.yaml +++ b/tests/test_config.yaml @@ -10,7 +10,7 @@ rest: controller: minos.services.OrderService action: add_order events: - broker: localhost + broker: kafka port: 9092 database: path: ./tests/local_db.lmdb @@ -30,7 +30,7 @@ events: port: 5432 records: 10 commands: - broker: localhost + broker: kafka port: 9092 items: - name: AddOrder From 26b064719b7907976b5e1e30df865208ff0ba2a1 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 23 Apr 2021 14:59:46 +0200 Subject: [PATCH 058/239] Update python-tests.yml --- .github/workflows/python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 7323ee30..65b2f724 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -38,7 +38,7 @@ jobs: env: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_HOST_NAME: localhost + KAFKA_ADVERTISED_HOST_NAME: kafka steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From e71f7c60ec03244a24577882ee463f8f051266be Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 23 Apr 2021 13:00:07 +0000 Subject: [PATCH 059/239] Restyled by black --- tests/broker/conftest.py | 4 +--- tests/broker/test_broker.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/broker/conftest.py b/tests/broker/conftest.py index c6bc320e..15068def 100644 --- a/tests/broker/conftest.py +++ b/tests/broker/conftest.py @@ -3,9 +3,7 @@ from aiomisc.service.periodic import Service from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import (BrokerDatabaseInitializer, - EventBrokerQueueDispatcher, - MinosBrokerDatabase) +from minos.networks.broker import BrokerDatabaseInitializer, EventBrokerQueueDispatcher, MinosBrokerDatabase @pytest.fixture diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index 6f675fda..210e2cd8 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -4,9 +4,13 @@ import pytest from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import (Aggregate, MinosCommandBroker, - MinosEventBroker, broker_queue_dispatcher, - send_to_kafka) +from minos.networks.broker import ( + Aggregate, + MinosCommandBroker, + MinosEventBroker, + broker_queue_dispatcher, + send_to_kafka, +) from tests.broker.database_testcase import PostgresAsyncTestCase @@ -157,6 +161,7 @@ async def test_if_events_retry_was_incremented(self): assert retry_1[0] > 0 assert retry_2[0] > 0 + """ create role broker with createdb login password 'br0k3r'; CREATE DATABASE broker_db OWNER broker; From 7a31be762a2d50adaca7b7440e39189493594e6c Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 23 Apr 2021 13:00:12 +0000 Subject: [PATCH 060/239] Restyled by isort --- tests/broker/conftest.py | 4 +++- tests/broker/test_broker.py | 10 +++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/broker/conftest.py b/tests/broker/conftest.py index 15068def..c6bc320e 100644 --- a/tests/broker/conftest.py +++ b/tests/broker/conftest.py @@ -3,7 +3,9 @@ from aiomisc.service.periodic import Service from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import BrokerDatabaseInitializer, EventBrokerQueueDispatcher, MinosBrokerDatabase +from minos.networks.broker import (BrokerDatabaseInitializer, + EventBrokerQueueDispatcher, + MinosBrokerDatabase) @pytest.fixture diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index 210e2cd8..dc7b81c0 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -4,13 +4,9 @@ import pytest from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import ( - Aggregate, - MinosCommandBroker, - MinosEventBroker, - broker_queue_dispatcher, - send_to_kafka, -) +from minos.networks.broker import (Aggregate, MinosCommandBroker, + MinosEventBroker, broker_queue_dispatcher, + send_to_kafka) from tests.broker.database_testcase import PostgresAsyncTestCase From 87d0cb594c228d0ac50173de8bb4c2a3cf8c82c9 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 23 Apr 2021 15:22:21 +0200 Subject: [PATCH 061/239] Kafka configuration --- .github/workflows/python-tests.yml | 1 + tests/test_config.yaml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 65b2f724..4208b076 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -39,6 +39,7 @@ jobs: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ADVERTISED_HOST_NAME: kafka + KAFKA_LISTENERS: INTERNAL://:9092,EXTERNAL://:9093 steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/tests/test_config.yaml b/tests/test_config.yaml index 03b3dc3a..0fae30f4 100644 --- a/tests/test_config.yaml +++ b/tests/test_config.yaml @@ -11,7 +11,7 @@ rest: action: add_order events: broker: kafka - port: 9092 + port: 9093 database: path: ./tests/local_db.lmdb name: database_events_test @@ -31,7 +31,7 @@ events: records: 10 commands: broker: kafka - port: 9092 + port: 9093 items: - name: AddOrder controller: minos.services.OrderService From d61529c1c31aed0aaf86b54708b68682b4329f2a Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 23 Apr 2021 15:29:19 +0200 Subject: [PATCH 062/239] Update test_config.yaml --- tests/test_config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_config.yaml b/tests/test_config.yaml index 0fae30f4..03b3dc3a 100644 --- a/tests/test_config.yaml +++ b/tests/test_config.yaml @@ -11,7 +11,7 @@ rest: action: add_order events: broker: kafka - port: 9093 + port: 9092 database: path: ./tests/local_db.lmdb name: database_events_test @@ -31,7 +31,7 @@ events: records: 10 commands: broker: kafka - port: 9093 + port: 9092 items: - name: AddOrder controller: minos.services.OrderService From 722c1473ab79c2e91babe5d5db7de0112334c3e9 Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Fri, 23 Apr 2021 15:54:17 +0200 Subject: [PATCH 063/239] updated action for kafka service --- .github/workflows/python-tests.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 4208b076..00c97e14 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -28,18 +28,17 @@ jobs: # Maps tcp port 5432 on service container to the host - 5432:5432 zookeeper: - image: bitnami/zookeeper:latest + image: wurstmeister/zookeeper:latest ports: - "2181:2181" kafka: - image: bitnami/kafka:latest + image: wurstmeister/kafka:latest ports: - "9092:9092" env: - KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_LISTENERS: INTERNAL://:9092,EXTERNAL://:9093 + steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 15ea11a580c7519f6240503873238de2f129791f Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Fri, 23 Apr 2021 15:59:35 +0200 Subject: [PATCH 064/239] modified actions removed unused folder of kafka service modified port in action from string to integer --- .github/workflows/python-tests.yml | 4 +- tests/kafka_container/.travis.yml | 89 ------- tests/kafka_container/CHANGELOG.md | 120 ---------- tests/kafka_container/Dockerfile | 44 ---- tests/kafka_container/LICENSE | 202 ---------------- tests/kafka_container/README.md | 222 ------------------ tests/kafka_container/broker-list.sh | 5 - tests/kafka_container/create-topics.sh | 57 ----- .../docker-compose-single-broker.yml | 16 -- .../kafka_container/docker-compose-swarm.yml | 22 -- tests/kafka_container/docker-compose.yml | 15 -- tests/kafka_container/docker_push | 14 -- tests/kafka_container/download-kafka.sh | 18 -- tests/kafka_container/overrides/0.9.0.1.sh | 6 - tests/kafka_container/start-kafka-shell.sh | 2 - tests/kafka_container/start-kafka.sh | 149 ------------ .../test/0.0/test.broker-list.kafka.sh | 19 -- ...st.create-topics-custom-separator.kafka.sh | 43 ---- .../test/0.0/test.path.kafka.sh | 15 -- .../test/0.0/test.read-write.kafkacat.sh | 10 - .../test.start-kafka-advertised-host.kafka.sh | 20 -- .../0.0/test.start-kafka-broker-id.kafka.sh | 51 ---- ...est.start-kafka-bug-312-kafka-env.kafka.sh | 25 -- ...st.start-kafka-bug-313-kafka-opts.kafka.sh | 23 -- .../0.0/test.start-kafka-host-name.kafka.sh | 18 -- .../test.start-kafka-log4j-config.kafka.sh | 17 -- .../test.start-kafka-port-command.kafka.sh | 21 -- .../0.0/test.start-kafka-restart.kafka.sh | 18 -- .../test/0.10/test.create-topics.kafka.sh | 43 ---- .../test/0.9/test.snappy.kafkacat.sh | 10 - ....start-kafka-advertised-listeners.kafka.sh | 18 -- .../0.9/test.start-kafka-listeners.kafka.sh | 19 -- ...st.start-kafka-multiple-listeners.kafka.sh | 25 -- tests/kafka_container/test/Readme.md | 31 --- tests/kafka_container/test/docker-compose.yml | 59 ----- .../test/internal-broker-list.sh | 4 - tests/kafka_container/test/runAllTests.sh | 25 -- tests/kafka_container/test/runTestPattern.sh | 50 ---- .../kafka_container/test/scenarios/Readme.md | 27 --- .../test/scenarios/jmx/docker-compose.yml | 44 ---- .../test/scenarios/jmx/jmxexporter.yml | 9 - .../test/scenarios/jmx/test.sh | 12 - .../test/scenarios/runJmxScenario.sh | 9 - tests/kafka_container/test/test.functions | 78 ------ .../kafka_container/test/verifyImageLabels.sh | 20 -- tests/kafka_container/test/version.functions | 37 --- tests/kafka_container/versions.sh | 7 - tests/local_db.lmdb/data.mdb | Bin 8192 -> 0 bytes tests/local_db.lmdb/lock.mdb | Bin 8192 -> 0 bytes 49 files changed, 2 insertions(+), 1790 deletions(-) delete mode 100644 tests/kafka_container/.travis.yml delete mode 100644 tests/kafka_container/CHANGELOG.md delete mode 100644 tests/kafka_container/Dockerfile delete mode 100644 tests/kafka_container/LICENSE delete mode 100644 tests/kafka_container/README.md delete mode 100755 tests/kafka_container/broker-list.sh delete mode 100755 tests/kafka_container/create-topics.sh delete mode 100644 tests/kafka_container/docker-compose-single-broker.yml delete mode 100644 tests/kafka_container/docker-compose-swarm.yml delete mode 100644 tests/kafka_container/docker-compose.yml delete mode 100755 tests/kafka_container/docker_push delete mode 100755 tests/kafka_container/download-kafka.sh delete mode 100755 tests/kafka_container/overrides/0.9.0.1.sh delete mode 100755 tests/kafka_container/start-kafka-shell.sh delete mode 100755 tests/kafka_container/start-kafka.sh delete mode 100755 tests/kafka_container/test/0.0/test.broker-list.kafka.sh delete mode 100755 tests/kafka_container/test/0.0/test.create-topics-custom-separator.kafka.sh delete mode 100755 tests/kafka_container/test/0.0/test.path.kafka.sh delete mode 100755 tests/kafka_container/test/0.0/test.read-write.kafkacat.sh delete mode 100755 tests/kafka_container/test/0.0/test.start-kafka-advertised-host.kafka.sh delete mode 100755 tests/kafka_container/test/0.0/test.start-kafka-broker-id.kafka.sh delete mode 100755 tests/kafka_container/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh delete mode 100755 tests/kafka_container/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh delete mode 100755 tests/kafka_container/test/0.0/test.start-kafka-host-name.kafka.sh delete mode 100755 tests/kafka_container/test/0.0/test.start-kafka-log4j-config.kafka.sh delete mode 100755 tests/kafka_container/test/0.0/test.start-kafka-port-command.kafka.sh delete mode 100755 tests/kafka_container/test/0.0/test.start-kafka-restart.kafka.sh delete mode 100755 tests/kafka_container/test/0.10/test.create-topics.kafka.sh delete mode 100755 tests/kafka_container/test/0.9/test.snappy.kafkacat.sh delete mode 100755 tests/kafka_container/test/0.9/test.start-kafka-advertised-listeners.kafka.sh delete mode 100755 tests/kafka_container/test/0.9/test.start-kafka-listeners.kafka.sh delete mode 100755 tests/kafka_container/test/0.9/test.start-kafka-multiple-listeners.kafka.sh delete mode 100644 tests/kafka_container/test/Readme.md delete mode 100644 tests/kafka_container/test/docker-compose.yml delete mode 100755 tests/kafka_container/test/internal-broker-list.sh delete mode 100755 tests/kafka_container/test/runAllTests.sh delete mode 100755 tests/kafka_container/test/runTestPattern.sh delete mode 100644 tests/kafka_container/test/scenarios/Readme.md delete mode 100644 tests/kafka_container/test/scenarios/jmx/docker-compose.yml delete mode 100644 tests/kafka_container/test/scenarios/jmx/jmxexporter.yml delete mode 100755 tests/kafka_container/test/scenarios/jmx/test.sh delete mode 100755 tests/kafka_container/test/scenarios/runJmxScenario.sh delete mode 100644 tests/kafka_container/test/test.functions delete mode 100755 tests/kafka_container/test/verifyImageLabels.sh delete mode 100644 tests/kafka_container/test/version.functions delete mode 100755 tests/kafka_container/versions.sh delete mode 100644 tests/local_db.lmdb/data.mdb delete mode 100644 tests/local_db.lmdb/lock.mdb diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 00c97e14..7225f642 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -30,11 +30,11 @@ jobs: zookeeper: image: wurstmeister/zookeeper:latest ports: - - "2181:2181" + - 2181:2181 kafka: image: wurstmeister/kafka:latest ports: - - "9092:9092" + - 9092:9092 env: KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ADVERTISED_HOST_NAME: kafka diff --git a/tests/kafka_container/.travis.yml b/tests/kafka_container/.travis.yml deleted file mode 100644 index 30595423..00000000 --- a/tests/kafka_container/.travis.yml +++ /dev/null @@ -1,89 +0,0 @@ -sudo: required - -language: scala - -services: - - docker - -# This version will be also tagged as 'latest' -env: - global: - - LATEST="2.13-2.7.0" - -# Build recommended versions based on: http://kafka.apache.org/downloads -matrix: - include: - - scala: "2.10" - env: KAFKA_VERSION=0.8.2.2 - - scala: 2.11 - env: KAFKA_VERSION=0.9.0.1 - - scala: 2.11 - env: KAFKA_VERSION=0.10.2.2 - - scala: 2.11 - env: KAFKA_VERSION=0.11.0.3 - - scala: 2.11 - env: KAFKA_VERSION=1.0.2 - - scala: 2.11 - env: KAFKA_VERSION=1.1.1 - - scala: 2.12 - env: KAFKA_VERSION=2.0.1 - - scala: 2.12 - env: KAFKA_VERSION=2.1.1 - - scala: 2.12 - env: KAFKA_VERSION=2.2.2 - - scala: 2.12 - env: KAFKA_VERSION=2.3.1 - - scala: 2.12 - env: KAFKA_VERSION=2.4.1 - - scala: 2.12 - env: KAFKA_VERSION=2.5.0 - - scala: 2.13 - env: KAFKA_VERSION=2.6.0 - - scala: 2.13 - env: KAFKA_VERSION=2.7.0 - -install: - - docker --version - - docker-compose --version - - echo "KAFKA VERSION $KAFKA_VERSION" - - echo "SCALA VERSION $TRAVIS_SCALA_VERSION" - - echo "LATEST VERSION $LATEST" - - export CURRENT=${TRAVIS_SCALA_VERSION}-${KAFKA_VERSION} - - docker build --build-arg kafka_version=$KAFKA_VERSION --build-arg scala_version=$TRAVIS_SCALA_VERSION --build-arg vcs_ref=$TRAVIS_COMMIT --build-arg build_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -t wurstmeister/kafka . - - docker pull confluentinc/cp-kafkacat - -before_script: - - docker-compose -f test/docker-compose.yml up -d zookeeper kafka_1 kafka_2 - -script: - # Shellcheck main source files - - shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh versions.sh - - cd test - # Shellcheck the tests - - shellcheck -x -e SC1090 -s bash *.sh **/*.sh - - ./verifyImageLabels.sh # Verify docker image's label - - sleep 5 # Wait for containers to start - - docker-compose logs - - docker ps -a - - ./runAllTests.sh - # End-to-End scenario tests - - cd scenarios - - ./runJmxScenario.sh - - cd $TRAVIS_BUILD_DIR - -after_script: - - docker-compose stop - -# This will deploy from master. Might want to have a single release branch for a little more control -deploy: - - provider: script - script: bash docker_push latest - on: - repo: wurstmeister/kafka-docker - branch: master - condition: $CURRENT = $LATEST - - provider: script - script: bash docker_push "${TRAVIS_SCALA_VERSION}-${KAFKA_VERSION}" - on: - repo: wurstmeister/kafka-docker - # branch: release diff --git a/tests/kafka_container/CHANGELOG.md b/tests/kafka_container/CHANGELOG.md deleted file mode 100644 index 0e0fce52..00000000 --- a/tests/kafka_container/CHANGELOG.md +++ /dev/null @@ -1,120 +0,0 @@ -Changelog -========= - -Kafka features are not tied to a specific kafka-docker version (ideally all changes will be merged into all branches). Therefore, this changelog will track changes to the image by date. - -30-Dec-2020 ------------ - -- Add support for Kafka `2.7.0` - -06-Aug-2020 ------------ - -- Add support for Kafka `2.6.0` - -20-Apr-2020 ------------ - -- Add support for Kafka `2.5.0` - -16-Mar-2020 ------------ - -- Add support for Kafka `2.4.1` -- Update glibc to `2.31-r0` - -20-Dec-2019 ------------ - -- Add support for Kafka `2.2.2` -- Update glibc to 2.30-r0 - -17-Dec-2019 ------------ - -- Add support for Kafka `2.4.0` - -26-Oct-2019 ------------ - -- Add support for Kafka `2.3.1` - -28-Jun-2019 ------------ - -- Add support for Kafka `2.3.0` - -04-Jun-2019 ------------ - -- Updated `2.2.x` version to Kafka `2.2.1` -- Update base image to openjdk:8u212-jre-alpine - -15-Apr-2019 ------------ - -- Update base image to openjdk:8u201-jre-alpine - -27-Mar-2019 ------------ - -- Add support for Kafka `2.2.0` - -21-Feb-2019 ------------ - -- Update to latest Kafka: `2.1.1` -- Update glibc to `2.29-r0` - -21-Nov-2018 ------------ - -- Update to latest Kafka: `2.1.0` -- Set scala version for Kafka `2.1.0` and `2.0.1` to recommended `2.12` - -10-Nov-2018 ------------ - -- Update to Kafka `2.0.0` -> `2.0.1`. -- Update glibc to `2.28-r0` -- Update base image to openjdk:8u181-jre-alpine - -29-Jun-2018 ------------ - -- **MAJOR:** Use new docker image labelling (`-`) and use travis to publish images. -- Update base image to openjdk:8u171-jre-alpine - -20-Apr-2018 ------------ - -- Issue #312 - Fix conflict between KAFKA_xxx broker config values (e.g. KAFKA_JMX_OPTS) and container configuration options (e.g. KAFKA_CREATE_TOPICS) - -19-Apr-2018 ------------ - -- Issue #310 - Only return Apache download mirrors that can supply required kafka/scala version - -11-Apr-2018 ------------ - -- Issue #313 - Fix parsing of environment value substitution when spaces included. - -08-Apr-2018 ------------ - -- Issue #208 - Add `KAFKA_CREATE_TOPICS_SEPARATOR` to allow custom input, such as multi-line YAML. -- Issue #298 - Fix SNAPPY compression support by adding glibc port back into image (removed when switching to openjdk base image in #7a25ade) - -04-Apr-2018 ------------ - -- Support `_{PORT_COMMAND}` placeholder. - -03-Apr-2018 ------------ - -- **BREAKING:** removed `KAFKA_ADVERTISED_PROTOCOL_NAME` and `KAFKA_PROTOCOL_NAME`. Use the canonical [Kafka Configuration](http://kafka.apache.org/documentation.html#brokerconfigs) instead. -- Support `_{HOSTNAME_COMMAND}` placeholder. -- **BREAKING:** Make `KAFKA_ZOOKEEPER_CONNECT` mandatory diff --git a/tests/kafka_container/Dockerfile b/tests/kafka_container/Dockerfile deleted file mode 100644 index 5eca3982..00000000 --- a/tests/kafka_container/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -FROM openjdk:8u212-jre-alpine - -ARG kafka_version=2.7.0 -ARG scala_version=2.13 -ARG glibc_version=2.31-r0 -ARG vcs_ref=unspecified -ARG build_date=unspecified - -LABEL org.label-schema.name="kafka" \ - org.label-schema.description="Apache Kafka" \ - org.label-schema.build-date="${build_date}" \ - org.label-schema.vcs-url="https://github.com/wurstmeister/kafka-docker" \ - org.label-schema.vcs-ref="${vcs_ref}" \ - org.label-schema.version="${scala_version}_${kafka_version}" \ - org.label-schema.schema-version="1.0" \ - maintainer="wurstmeister" - -ENV KAFKA_VERSION=$kafka_version \ - SCALA_VERSION=$scala_version \ - KAFKA_HOME=/opt/kafka \ - GLIBC_VERSION=$glibc_version - -ENV PATH=${PATH}:${KAFKA_HOME}/bin - -COPY download-kafka.sh start-kafka.sh broker-list.sh create-topics.sh versions.sh /tmp/ - -RUN apk add --no-cache bash curl jq docker \ - && chmod a+x /tmp/*.sh \ - && mv /tmp/start-kafka.sh /tmp/broker-list.sh /tmp/create-topics.sh /tmp/versions.sh /usr/bin \ - && sync && /tmp/download-kafka.sh \ - && tar xfz /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt \ - && rm /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz \ - && ln -s /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} ${KAFKA_HOME} \ - && rm /tmp/* \ - && wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \ - && apk add --no-cache --allow-untrusted glibc-${GLIBC_VERSION}.apk \ - && rm glibc-${GLIBC_VERSION}.apk - -COPY overrides /opt/overrides - -VOLUME ["/kafka"] - -# Use "exec" form so that it runs as PID 1 (useful for graceful shutdown) -CMD ["start-kafka.sh"] diff --git a/tests/kafka_container/LICENSE b/tests/kafka_container/LICENSE deleted file mode 100644 index e06d2081..00000000 --- a/tests/kafka_container/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/tests/kafka_container/README.md b/tests/kafka_container/README.md deleted file mode 100644 index d55939ad..00000000 --- a/tests/kafka_container/README.md +++ /dev/null @@ -1,222 +0,0 @@ -[![Docker Pulls](https://img.shields.io/docker/pulls/wurstmeister/kafka.svg)](https://hub.docker.com/r/wurstmeister/kafka/) -[![Docker Stars](https://img.shields.io/docker/stars/wurstmeister/kafka.svg)](https://hub.docker.com/r/wurstmeister/kafka/) -[![](https://images.microbadger.com/badges/version/wurstmeister/kafka.svg)](https://microbadger.com/images/wurstmeister/kafka "Get your own version badge on microbadger.com") -[![](https://images.microbadger.com/badges/image/wurstmeister/kafka.svg)](https://microbadger.com/images/wurstmeister/kafka "Get your own image badge on microbadger.com") -[![Build Status](https://travis-ci.org/wurstmeister/kafka-docker.svg?branch=master)](https://travis-ci.org/wurstmeister/kafka-docker) - -kafka-docker -============ - -Dockerfile for [Apache Kafka](http://kafka.apache.org/) - -The image is available directly from [Docker Hub](https://hub.docker.com/r/wurstmeister/kafka/) - -Tags and releases ------------------ - -All versions of the image are built from the same set of scripts with only minor variations (i.e. certain features are not supported on older versions). The version format mirrors the Kafka format, `-`. Initially, all images are built with the recommended version of scala documented on [http://kafka.apache.org/downloads](http://kafka.apache.org/downloads). Available tags are: - -- `2.13-2.7.0` -- `2.13-2.6.0` -- `2.12-2.5.0` -- `2.12-2.4.1` -- `2.12-2.3.1` -- `2.12-2.2.2` -- `2.12-2.1.1` -- `2.12-2.0.1` -- `2.11-1.1.1` -- `2.11-1.0.2` -- `2.11-0.11.0.3` -- `2.11-0.10.2.2` -- `2.11-0.9.0.1` -- `2.10-0.8.2.2` - -Everytime the image is updated, all tags will be pushed with the latest updates. This should allow for greater consistency across tags, as well as any security updates that have been made to the base image. - ---- - -## Announcements - -* **04-Jun-2019** - Update base image to openjdk 212 ([Release notes](https://www.oracle.com/technetwork/java/javase/8u212-relnotes-5292913.html). Please force pull to get these latest updates - including security patches etc. - ---- - -## Pre-Requisites - -- install docker-compose [https://docs.docker.com/compose/install/](https://docs.docker.com/compose/install/) -- modify the ```KAFKA_ADVERTISED_HOST_NAME``` in [docker-compose.yml](https://raw.githubusercontent.com/wurstmeister/kafka-docker/master/docker-compose.yml) to match your docker host IP (Note: Do not use localhost or 127.0.0.1 as the host ip if you want to run multiple brokers.) -- if you want to customize any Kafka parameters, simply add them as environment variables in ```docker-compose.yml```, e.g. in order to increase the ```message.max.bytes``` parameter set the environment to ```KAFKA_MESSAGE_MAX_BYTES: 2000000```. To turn off automatic topic creation set ```KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'``` -- Kafka's log4j usage can be customized by adding environment variables prefixed with ```LOG4J_```. These will be mapped to ```log4j.properties```. For example: ```LOG4J_LOGGER_KAFKA_AUTHORIZER_LOGGER=DEBUG, authorizerAppender``` - -**NOTE:** There are several 'gotchas' with configuring networking. If you are not sure about what the requirements are, please check out the [Connectivity Guide](https://github.com/wurstmeister/kafka-docker/wiki/Connectivity) in the [Wiki](https://github.com/wurstmeister/kafka-docker/wiki) - -## Usage - -Start a cluster: - -- ```docker-compose up -d ``` - -Add more brokers: - -- ```docker-compose scale kafka=3``` - -Destroy a cluster: - -- ```docker-compose stop``` - -## Note - -The default ```docker-compose.yml``` should be seen as a starting point. By default each broker will get a new port number and broker id on restart. Depending on your use case this might not be desirable. If you need to use specific ports and broker ids, modify the docker-compose configuration accordingly, e.g. [docker-compose-single-broker.yml](https://github.com/wurstmeister/kafka-docker/blob/master/docker-compose-single-broker.yml): - -- ```docker-compose -f docker-compose-single-broker.yml up``` - -## Broker IDs - -You can configure the broker id in different ways - -1. explicitly, using ```KAFKA_BROKER_ID``` -2. via a command, using ```BROKER_ID_COMMAND```, e.g. ```BROKER_ID_COMMAND: "hostname | awk -F'-' '{print $$2}'"``` - -If you don't specify a broker id in your docker-compose file, it will automatically be generated (see [https://issues.apache.org/jira/browse/KAFKA-1070](https://issues.apache.org/jira/browse/KAFKA-1070). This allows scaling up and down. In this case it is recommended to use the ```--no-recreate``` option of docker-compose to ensure that containers are not re-created and thus keep their names and ids. - - -## Automatically create topics - -If you want to have kafka-docker automatically create topics in Kafka during -creation, a ```KAFKA_CREATE_TOPICS``` environment variable can be -added in ```docker-compose.yml```. - -Here is an example snippet from ```docker-compose.yml```: - - environment: - KAFKA_CREATE_TOPICS: "Topic1:1:3,Topic2:1:1:compact" - -```Topic 1``` will have 1 partition and 3 replicas, ```Topic 2``` will have 1 partition, 1 replica and a `cleanup.policy` set to `compact`. Also, see FAQ: [Topic compaction does not work](https://github.com/wurstmeister/kafka-docker/wiki#topic-compaction-does-not-work) - -If you wish to use multi-line YAML or some other delimiter between your topic definitions, override the default `,` separator by specifying the `KAFKA_CREATE_TOPICS_SEPARATOR` environment variable. - -For example, `KAFKA_CREATE_TOPICS_SEPARATOR: "$$'\n'"` would use a newline to split the topic definitions. Syntax has to follow docker-compose escaping rules, and [ANSI-C](https://www.gnu.org/software/bash/manual/html_node/ANSI_002dC-Quoting.html) quoting. - -## Advertised hostname - -You can configure the advertised hostname in different ways - -1. explicitly, using ```KAFKA_ADVERTISED_HOST_NAME``` -2. via a command, using ```HOSTNAME_COMMAND```, e.g. ```HOSTNAME_COMMAND: "route -n | awk '/UG[ \t]/{print $$2}'"``` - -When using commands, make sure you review the "Variable Substitution" section in [https://docs.docker.com/compose/compose-file/](https://docs.docker.com/compose/compose-file/#variable-substitution) - -If ```KAFKA_ADVERTISED_HOST_NAME``` is specified, it takes precedence over ```HOSTNAME_COMMAND``` - -For AWS deployment, you can use the Metadata service to get the container host's IP: -``` -HOSTNAME_COMMAND=wget -t3 -T2 -qO- http://169.254.169.254/latest/meta-data/local-ipv4 -``` -Reference: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html - -### Injecting HOSTNAME_COMMAND into configuration - -If you require the value of `HOSTNAME_COMMAND` in any of your other `KAFKA_XXX` variables, use the `_{HOSTNAME_COMMAND}` string in your variable value, i.e. - -``` -KAFKA_ADVERTISED_LISTENERS=SSL://_{HOSTNAME_COMMAND}:9093,PLAINTEXT://9092 -``` - -## Advertised port - -If the required advertised port is not static, it may be necessary to determine this programatically. This can be done with the `PORT_COMMAND` environment variable. - -``` -PORT_COMMAND: "docker port $$(hostname) 9092/tcp | cut -d: -f2" -``` - -This can be then interpolated in any other `KAFKA_XXX` config using the `_{PORT_COMMAND}` string, i.e. - -``` -KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://1.2.3.4:_{PORT_COMMAND} -``` - -## Listener Configuration - -It may be useful to have the [Kafka Documentation](https://kafka.apache.org/documentation/) open, to understand the various broker listener configuration options. - -Since 0.9.0, Kafka has supported [multiple listener configurations](https://issues.apache.org/jira/browse/KAFKA-1809) for brokers to help support different protocols and discriminate between internal and external traffic. Later versions of Kafka have deprecated ```advertised.host.name``` and ```advertised.port```. - -**NOTE:** ```advertised.host.name``` and ```advertised.port``` still work as expected, but should not be used if configuring the listeners. - -### Example - -The example environment below: - -``` -HOSTNAME_COMMAND: curl http://169.254.169.254/latest/meta-data/public-hostname -KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094 -KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094 -KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT -KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE -``` - -Will result in the following broker config: - -``` -advertised.listeners = OUTSIDE://ec2-xx-xx-xxx-xx.us-west-2.compute.amazonaws.com:9094,INSIDE://:9092 -listeners = OUTSIDE://:9094,INSIDE://:9092 -inter.broker.listener.name = INSIDE -``` - -### Rules - -* No listeners may share a port number. -* An advertised.listener must be present by protocol name and port number in the list of listeners. - -## Broker Rack - -You can configure the broker rack affinity in different ways - -1. explicitly, using ```KAFKA_BROKER_RACK``` -2. via a command, using ```RACK_COMMAND```, e.g. ```RACK_COMMAND: "curl http://169.254.169.254/latest/meta-data/placement/availability-zone"``` - -In the above example the AWS metadata service is used to put the instance's availability zone in the ```broker.rack``` property. - -## JMX - -For monitoring purposes you may wish to configure JMX. Additional to the standard JMX parameters, problems could arise from the underlying RMI protocol used to connect - -* java.rmi.server.hostname - interface to bind listening port -* com.sun.management.jmxremote.rmi.port - The port to service RMI requests - -For example, to connect to a kafka running locally (assumes exposing port 1099) - - KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.rmi.port=1099" - JMX_PORT: 1099 - -Jconsole can now connect at ```jconsole 192.168.99.100:1099``` - -## Docker Swarm Mode - -The listener configuration above is necessary when deploying Kafka in a Docker Swarm using an overlay network. By separating OUTSIDE and INSIDE listeners, a host can communicate with clients outside the overlay network while still benefiting from it from within the swarm. - -In addition to the multiple-listener configuration, additional best practices for operating Kafka in a Docker Swarm include: - -* Use "deploy: global" in a compose file to launch one and only one Kafka broker per swarm node. -* Use compose file version '3.2' (minimum Docker version 16.04) and the "long" port definition with the port in "host" mode instead of the default "ingress" load-balanced port binding. This ensures that outside requests are always routed to the correct broker. For example: - -``` -ports: - - target: 9094 - published: 9094 - protocol: tcp - mode: host -``` - -Older compose files using the short-version of port mapping may encounter Kafka client issues if their connection to individual brokers cannot be guaranteed. - -See the included sample compose file ```docker-compose-swarm.yml``` - -## Release process - -See the [wiki](https://github.com/wurstmeister/kafka-docker/wiki/ReleaseProcess) for information on adding or updating versions to release to Dockerhub. - -## Tutorial - -[http://wurstmeister.github.io/kafka-docker/](http://wurstmeister.github.io/kafka-docker/) diff --git a/tests/kafka_container/broker-list.sh b/tests/kafka_container/broker-list.sh deleted file mode 100755 index 73aa8220..00000000 --- a/tests/kafka_container/broker-list.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -CONTAINERS=$(docker ps | grep 9092 | awk '{print $1}') -BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done) -echo "${BROKERS/$'\n'/,}" diff --git a/tests/kafka_container/create-topics.sh b/tests/kafka_container/create-topics.sh deleted file mode 100755 index 0bacf7b5..00000000 --- a/tests/kafka_container/create-topics.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -if [[ -z "$KAFKA_CREATE_TOPICS" ]]; then - exit 0 -fi - -if [[ -z "$START_TIMEOUT" ]]; then - START_TIMEOUT=600 -fi - -start_timeout_exceeded=false -count=0 -step=10 -while netstat -lnt | awk '$4 ~ /:'"$KAFKA_PORT"'$/ {exit 1}'; do - echo "waiting for kafka to be ready" - sleep $step; - count=$((count + step)) - if [ $count -gt $START_TIMEOUT ]; then - start_timeout_exceeded=true - break - fi -done - -if $start_timeout_exceeded; then - echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)" - exit 1 -fi - -# introduced in 0.10. In earlier versions, this will fail because the topic already exists. -# shellcheck disable=SC1091 -source "/usr/bin/versions.sh" -if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -gt "9" ]] || [[ "$MAJOR_VERSION" -gt "0" ]]; then - KAFKA_0_10_OPTS="--if-not-exists" -fi - -# Expected format: -# name:partitions:replicas:cleanup.policy -IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOPICS; do - echo "creating topics: $topicToCreate" - IFS=':' read -r -a topicConfig <<< "$topicToCreate" - config= - if [ -n "${topicConfig[3]}" ]; then - config="--config=cleanup.policy=${topicConfig[3]}" - fi - - COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\ - --create \\ - --zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\ - --topic ${topicConfig[0]} \\ - --partitions ${topicConfig[1]} \\ - --replication-factor ${topicConfig[2]} \\ - ${config} \\ - ${KAFKA_0_10_OPTS} &" - eval "${COMMAND}" -done - -wait diff --git a/tests/kafka_container/docker-compose-single-broker.yml b/tests/kafka_container/docker-compose-single-broker.yml deleted file mode 100644 index 94128d5c..00000000 --- a/tests/kafka_container/docker-compose-single-broker.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: '2' -services: - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181:2181" - kafka: - build: . - ports: - - "9092:9092" - environment: - KAFKA_ADVERTISED_HOST_NAME: localhost - KAFKA_CREATE_TOPICS: "test:1:1" - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - volumes: - - /var/run/docker.sock:/var/run/docker.sock diff --git a/tests/kafka_container/docker-compose-swarm.yml b/tests/kafka_container/docker-compose-swarm.yml deleted file mode 100644 index 86e63eb1..00000000 --- a/tests/kafka_container/docker-compose-swarm.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: '3.2' -services: - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181:2181" - kafka: - image: wurstmeister/kafka:latest - ports: - - target: 9094 - published: 9094 - protocol: tcp - mode: host - environment: - HOSTNAME_COMMAND: "docker info | grep ^Name: | cut -d' ' -f 2" - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094 - KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094 - KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE - volumes: - - /var/run/docker.sock:/var/run/docker.sock diff --git a/tests/kafka_container/docker-compose.yml b/tests/kafka_container/docker-compose.yml deleted file mode 100644 index bc270dc6..00000000 --- a/tests/kafka_container/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '2' -services: - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181:2181" - kafka: - build: . - ports: - - "9092:9092" - environment: - KAFKA_ADVERTISED_HOST_NAME: localhost - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - volumes: - - /var/run/docker.sock:/var/run/docker.sock diff --git a/tests/kafka_container/docker_push b/tests/kafka_container/docker_push deleted file mode 100755 index 99975bb9..00000000 --- a/tests/kafka_container/docker_push +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -e - -BASE_IMAGE="wurstmeister/kafka" -IMAGE_VERSION="$1" - -if [ -z "$IMAGE_VERSION" ]; then - echo "No IMAGE_VERSION var specified" - exit 1 -fi - -echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin -TARGET="$BASE_IMAGE:$IMAGE_VERSION" -docker tag "$BASE_IMAGE" "$TARGET" -docker push "$TARGET" diff --git a/tests/kafka_container/download-kafka.sh b/tests/kafka_container/download-kafka.sh deleted file mode 100755 index 00bf4511..00000000 --- a/tests/kafka_container/download-kafka.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -e - -# shellcheck disable=SC1091 -source "/usr/bin/versions.sh" - -FILENAME="kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" - -url=$(curl --stderr /dev/null "https://www.apache.org/dyn/closer.cgi?path=/kafka/${KAFKA_VERSION}/${FILENAME}&as_json=1" | jq -r '"\(.preferred)\(.path_info)"') - -# Test to see if the suggested mirror has this version, currently pre 2.1.1 versions -# do not appear to be actively mirrored. This may also be useful if closer.cgi is down. -if [[ ! $(curl -s -f -I "${url}") ]]; then - echo "Mirror does not have desired version, downloading direct from Apache" - url="https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/${FILENAME}" -fi - -echo "Downloading Kafka from $url" -wget "${url}" -O "/tmp/${FILENAME}" diff --git a/tests/kafka_container/overrides/0.9.0.1.sh b/tests/kafka_container/overrides/0.9.0.1.sh deleted file mode 100755 index d5e85611..00000000 --- a/tests/kafka_container/overrides/0.9.0.1.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -e - -# Kafka 0.9.x.x has a 'listeners' config by default. We need to remove this -# as the user may be configuring via the host.name / advertised.host.name properties -echo "Removing 'listeners' from server.properties pre-bootstrap" -sed -i -e '/^listeners=/d' "$KAFKA_HOME/config/server.properties" diff --git a/tests/kafka_container/start-kafka-shell.sh b/tests/kafka_container/start-kafka-shell.sh deleted file mode 100755 index 62663e49..00000000 --- a/tests/kafka_container/start-kafka-shell.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e HOST_IP=$1 -e ZK=$2 -i -t wurstmeister/kafka /bin/bash diff --git a/tests/kafka_container/start-kafka.sh b/tests/kafka_container/start-kafka.sh deleted file mode 100755 index 85359118..00000000 --- a/tests/kafka_container/start-kafka.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash -e - -# Allow specific kafka versions to perform any unique bootstrap operations -OVERRIDE_FILE="/opt/overrides/${KAFKA_VERSION}.sh" -if [[ -x "$OVERRIDE_FILE" ]]; then - echo "Executing override file $OVERRIDE_FILE" - eval "$OVERRIDE_FILE" -fi - -# Store original IFS config, so we can restore it at various stages -ORIG_IFS=$IFS - -if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then - echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT" - exit 1 -fi - -if [[ -z "$KAFKA_PORT" ]]; then - export KAFKA_PORT=9092 -fi - -create-topics.sh & -unset KAFKA_CREATE_TOPICS - -if [[ -z "$KAFKA_ADVERTISED_PORT" && \ - -z "$KAFKA_LISTENERS" && \ - -z "$KAFKA_ADVERTISED_LISTENERS" && \ - -S /var/run/docker.sock ]]; then - KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g') - export KAFKA_ADVERTISED_PORT -fi - -if [[ -z "$KAFKA_BROKER_ID" ]]; then - if [[ -n "$BROKER_ID_COMMAND" ]]; then - KAFKA_BROKER_ID=$(eval "$BROKER_ID_COMMAND") - export KAFKA_BROKER_ID - else - # By default auto allocate broker ID - export KAFKA_BROKER_ID=-1 - fi -fi - -if [[ -z "$KAFKA_LOG_DIRS" ]]; then - export KAFKA_LOG_DIRS="/kafka/kafka-logs-$HOSTNAME" -fi - -if [[ -n "$KAFKA_HEAP_OPTS" ]]; then - sed -r -i 's/(export KAFKA_HEAP_OPTS)="(.*)"/\1="'"$KAFKA_HEAP_OPTS"'"/g' "$KAFKA_HOME/bin/kafka-server-start.sh" - unset KAFKA_HEAP_OPTS -fi - -if [[ -n "$HOSTNAME_COMMAND" ]]; then - HOSTNAME_VALUE=$(eval "$HOSTNAME_COMMAND") - - # Replace any occurences of _{HOSTNAME_COMMAND} with the value - IFS=$'\n' - for VAR in $(env); do - if [[ $VAR =~ ^KAFKA_ && "$VAR" =~ "_{HOSTNAME_COMMAND}" ]]; then - eval "export ${VAR//_\{HOSTNAME_COMMAND\}/$HOSTNAME_VALUE}" - fi - done - IFS=$ORIG_IFS -fi - -if [[ -n "$PORT_COMMAND" ]]; then - PORT_VALUE=$(eval "$PORT_COMMAND") - - # Replace any occurences of _{PORT_COMMAND} with the value - IFS=$'\n' - for VAR in $(env); do - if [[ $VAR =~ ^KAFKA_ && "$VAR" =~ "_{PORT_COMMAND}" ]]; then - eval "export ${VAR//_\{PORT_COMMAND\}/$PORT_VALUE}" - fi - done - IFS=$ORIG_IFS -fi - -if [[ -n "$RACK_COMMAND" && -z "$KAFKA_BROKER_RACK" ]]; then - KAFKA_BROKER_RACK=$(eval "$RACK_COMMAND") - export KAFKA_BROKER_RACK -fi - -# Try and configure minimal settings or exit with error if there isn't enough information -if [[ -z "$KAFKA_ADVERTISED_HOST_NAME$KAFKA_LISTENERS" ]]; then - if [[ -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then - echo "ERROR: Missing environment variable KAFKA_LISTENERS. Must be specified when using KAFKA_ADVERTISED_LISTENERS" - exit 1 - elif [[ -z "$HOSTNAME_VALUE" ]]; then - echo "ERROR: No listener or advertised hostname configuration provided in environment." - echo " Please define KAFKA_LISTENERS / (deprecated) KAFKA_ADVERTISED_HOST_NAME" - exit 1 - fi - - # Maintain existing behaviour - # If HOSTNAME_COMMAND is provided, set that to the advertised.host.name value if listeners are not defined. - export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE" -fi - -#Issue newline to config file in case there is not one already -echo "" >> "$KAFKA_HOME/config/server.properties" - -( - function updateConfig() { - key=$1 - value=$2 - file=$3 - - # Omit $value here, in case there is sensitive information - echo "[Configuring] '$key' in '$file'" - - # If config exists in file, replace it. Otherwise, append to file. - if grep -E -q "^#?$key=" "$file"; then - sed -r -i "s@^#?$key=.*@$key=$value@g" "$file" #note that no config values may contain an '@' char - else - echo "$key=$value" >> "$file" - fi - } - - # Fixes #312 - # KAFKA_VERSION + KAFKA_HOME + grep -rohe KAFKA[A-Z0-0_]* /opt/kafka/bin | sort | uniq | tr '\n' '|' - EXCLUSIONS="|KAFKA_VERSION|KAFKA_HOME|KAFKA_DEBUG|KAFKA_GC_LOG_OPTS|KAFKA_HEAP_OPTS|KAFKA_JMX_OPTS|KAFKA_JVM_PERFORMANCE_OPTS|KAFKA_LOG|KAFKA_OPTS|" - - # Read in env as a new-line separated array. This handles the case of env variables have spaces and/or carriage returns. See #313 - IFS=$'\n' - for VAR in $(env) - do - env_var=$(echo "$VAR" | cut -d= -f1) - if [[ "$EXCLUSIONS" = *"|$env_var|"* ]]; then - echo "Excluding $env_var from broker config" - continue - fi - - if [[ $env_var =~ ^KAFKA_ ]]; then - kafka_name=$(echo "$env_var" | cut -d_ -f2- | tr '[:upper:]' '[:lower:]' | tr _ .) - updateConfig "$kafka_name" "${!env_var}" "$KAFKA_HOME/config/server.properties" - fi - - if [[ $env_var =~ ^LOG4J_ ]]; then - log4j_name=$(echo "$env_var" | tr '[:upper:]' '[:lower:]' | tr _ .) - updateConfig "$log4j_name" "${!env_var}" "$KAFKA_HOME/config/log4j.properties" - fi - done -) - -if [[ -n "$CUSTOM_INIT_SCRIPT" ]] ; then - eval "$CUSTOM_INIT_SCRIPT" -fi - -exec "$KAFKA_HOME/bin/kafka-server-start.sh" "$KAFKA_HOME/config/server.properties" diff --git a/tests/kafka_container/test/0.0/test.broker-list.kafka.sh b/tests/kafka_container/test/0.0/test.broker-list.kafka.sh deleted file mode 100755 index f557e8f9..00000000 --- a/tests/kafka_container/test/0.0/test.broker-list.kafka.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e - -testBrokerList() { - # Need to get the proxied ports for kafka - PORT1=$(docker inspect -f '{{ index .NetworkSettings.Ports "9092/tcp" 0 "HostPort" }}' test_kafka_1) - PORT2=$(docker inspect -f '{{ index .NetworkSettings.Ports "9092/tcp" 0 "HostPort" }}' test_kafka_2) - - RESULT=$(HOST_IP=1.2.3.4 broker-list.sh) - - echo "$RESULT" - - if [[ "$RESULT" == "1.2.3.4:$PORT1,1.2.3.4:$PORT2" || "$RESULT" == "1.2.3.4:$PORT2,1.2.3.4:$PORT1" ]]; then - return 0 - else - return 1 - fi -} - -testBrokerList diff --git a/tests/kafka_container/test/0.0/test.create-topics-custom-separator.kafka.sh b/tests/kafka_container/test/0.0/test.create-topics-custom-separator.kafka.sh deleted file mode 100755 index abd031c1..00000000 --- a/tests/kafka_container/test/0.0/test.create-topics-custom-separator.kafka.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -e - -# NOTE: create-topics.sh requires KAFKA_PORT and KAFKA_ZOOKEEPER_CONNECT to be set (see docker-compose.yml) -testCreateTopicsCustomSeparator() { - NOW=$(date +%s) - - # TOPICS array contains the topic name to create / validate - TOPICS[0]="one-$NOW" - TOPICS[1]="two-$NOW" - TOPICS[2]="three-$NOW" - - export KAFKA_CREATE_TOPICS_SEPARATOR=$'\n' - KAFKA_CREATE_TOPICS=$(cat <<-EOF - ${TOPICS[0]}:1:1 - ${TOPICS[1]}:1:1 - ${TOPICS[2]}:1:1 - EOF - ) - export KAFKA_CREATE_TOPICS - - create-topics.sh - - # Loop through each array, validate that topic exists - for i in "${!TOPICS[@]}"; do - TOPIC=${TOPICS[i]} - - echo "Validating topic '$TOPIC'" - - EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC") - if [[ "$EXISTS" != "$TOPIC" ]]; then - echo "$TOPIC topic not created" - return 1 - fi - done - - return 0 -} - -# mock the netstat call as made by the create-topics.sh script -function netstat() { echo "1 2 3 :$KAFKA_PORT"; } -export -f netstat - -testCreateTopicsCustomSeparator diff --git a/tests/kafka_container/test/0.0/test.path.kafka.sh b/tests/kafka_container/test/0.0/test.path.kafka.sh deleted file mode 100755 index ce02401b..00000000 --- a/tests/kafka_container/test/0.0/test.path.kafka.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -e - -# NOTE: this tests to see if the /opt/kafka/bin is existing in the path within the docker container - -testPath() { - echo "Checking PATH '$PATH'" - if [[ ! "$PATH" =~ "/opt/kafka/bin" ]]; then - echo "path is not set correctly: $PATH" - return 1 - fi - - return 0 -} - -testPath diff --git a/tests/kafka_container/test/0.0/test.read-write.kafkacat.sh b/tests/kafka_container/test/0.0/test.read-write.kafkacat.sh deleted file mode 100755 index 325455bb..00000000 --- a/tests/kafka_container/test/0.0/test.read-write.kafkacat.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e - -source version.functions - -testReadWrite() { - echo 'foo,bar' | eval "kafkacat -b $BROKER_LIST $KAFKACAT_OPTS -P -D, -t readwrite" - eval "kafkacat -b $BROKER_LIST $KAFKACAT_OPTS -C -e -t readwrite" -} - -testReadWrite diff --git a/tests/kafka_container/test/0.0/test.start-kafka-advertised-host.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-advertised-host.kafka.sh deleted file mode 100755 index 1791d60b..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-advertised-host.kafka.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testAdvertisedHost() { - # Given a hostname is provided - export KAFKA_ADVERTISED_HOST_NAME=monkey - export KAFKA_ADVERTISED_PORT=8888 - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration file is correct - assertExpectedConfig "advertised.host.name=monkey" - assertExpectedConfig "advertised.port=8888" - assertAbsent 'advertised.listeners' - assertAbsent 'listeners' -} - -testAdvertisedHost diff --git a/tests/kafka_container/test/0.0/test.start-kafka-broker-id.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-broker-id.kafka.sh deleted file mode 100755 index 4b132648..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-broker-id.kafka.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testManualBrokerId() { - echo "testManualBrokerId" - - # Given a Broker Id is provided - export KAFKA_LISTENERS=PLAINTEXT://:9092 - export KAFKA_BROKER_ID=57 - - # When the script is invoked - source "$START_KAFKA" - - # Then the broker Id is set - assertExpectedConfig 'broker.id=57' -} - -testAutomaticBrokerId() { - echo "testAutomaticBrokerId" - - # Given no Broker Id is provided - export KAFKA_LISTENERS=PLAINTEXT://:9092 - unset KAFKA_BROKER_ID - - # When the script is invoked - source "$START_KAFKA" - - # Then the broker Id is configured to automatic - assertExpectedConfig 'broker.id=-1' -} - -testBrokerIdCommand() { - echo "testBrokerIdCommand" - - # Given a Broker Id command is provided - export KAFKA_LISTENERS=PLAINTEXT://:9092 - unset KAFKA_BROKER_ID - export BROKER_ID_COMMAND='f() { echo "23"; }; f' - - # When the script is invoked - source "$START_KAFKA" - - # Then the broker Id is the result of the command - assertExpectedConfig 'broker.id=23' -} - - -testManualBrokerId \ - && testAutomaticBrokerId \ - && testBrokerIdCommand diff --git a/tests/kafka_container/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh deleted file mode 100755 index 5f0167c3..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testKafkaEnv() { - # Given required settings are provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" - export KAFKA_OPTS="-Djava.security.auth.login.config=/kafka_server_jaas.conf" - - # When the script is invoked - source "$START_KAFKA" - - # Then env should remain untouched - if [[ ! "$KAFKA_OPTS" == "-Djava.security.auth.login.config=/kafka_server_jaas.conf" ]]; then - echo "KAFKA_OPTS not set to expected value. $KAFKA_OPTS" - exit 1 - fi - - # And the broker config should not be set - assertAbsent 'opts' - - echo " > Set KAFKA_OPTS=$KAFKA_OPTS" -} - -testKafkaEnv diff --git a/tests/kafka_container/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh deleted file mode 100755 index df7b4018..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testKafkaOpts() { - # Given required settings are provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" - # .. and a CUSTOM_INIT_SCRIPT with spaces - export CUSTOM_INIT_SCRIPT="export KAFKA_OPTS=-Djava.security.auth.login.config=/kafka_server_jaas.conf" - - # When the script is invoked - source "$START_KAFKA" - - # Then the custom init script should be evaluated - if [[ ! "$KAFKA_OPTS" == "-Djava.security.auth.login.config=/kafka_server_jaas.conf" ]]; then - echo "KAFKA_OPTS not set to expected value. $KAFKA_OPTS" - exit 1 - fi - - echo " > Set KAFKA_OPTS=$KAFKA_OPTS" -} - -testKafkaOpts diff --git a/tests/kafka_container/test/0.0/test.start-kafka-host-name.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-host-name.kafka.sh deleted file mode 100755 index 08dc2113..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-host-name.kafka.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testHostnameCommand() { - # Given a hostname command is provided - export HOSTNAME_COMMAND='f() { echo "my-host"; }; f' - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration uses the value from the command - assertExpectedConfig 'advertised.host.name=my-host' - assertAbsent 'advertised.listeners' - assertAbsent 'listeners' -} - -testHostnameCommand diff --git a/tests/kafka_container/test/0.0/test.start-kafka-log4j-config.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-log4j-config.kafka.sh deleted file mode 100755 index da4ff28e..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-log4j-config.kafka.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testLog4jConfig() { - # Given Log4j overrides are provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" - export LOG4J_LOGGER_KAFKA=DEBUG - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration file is correct - assertExpectedLog4jConfig "log4j.logger.kafka=DEBUG" -} - -testLog4jConfig diff --git a/tests/kafka_container/test/0.0/test.start-kafka-port-command.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-port-command.kafka.sh deleted file mode 100755 index 4c4d8833..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-port-command.kafka.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testPortCommand() { - # Given a port command is provided - export PORT_COMMAND='f() { echo "12345"; }; f' - export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://1.2.3.4:_{PORT_COMMAND}" - export KAFKA_LISTENERS="PLAINTEXT://:9092" - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration uses the value from the command - assertExpectedConfig 'advertised.listeners=PLAINTEXT://1.2.3.4:12345' - assertExpectedConfig 'listeners=PLAINTEXT://:9092' - assertAbsent 'advertised.host.name' - assertAbsent 'advertised.port' -} - -testPortCommand diff --git a/tests/kafka_container/test/0.0/test.start-kafka-restart.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-restart.kafka.sh deleted file mode 100755 index 68a25598..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-restart.kafka.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testRestart() { - # Given a hostname is provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" - - # When the container is restarted (Script invoked multiple times) - source "$START_KAFKA" - source "$START_KAFKA" - - # Then the configuration file only has one instance of the config - assertExpectedConfig 'advertised.host.name=testhost' - assertAbsent 'listeners' -} - -testRestart diff --git a/tests/kafka_container/test/0.10/test.create-topics.kafka.sh b/tests/kafka_container/test/0.10/test.create-topics.kafka.sh deleted file mode 100755 index e9dda7bd..00000000 --- a/tests/kafka_container/test/0.10/test.create-topics.kafka.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -e - -# NOTE: create-topics.sh requires KAFKA_PORT and KAFKA_ZOOKEEPER_CONNECT to be set (see docker-compose.yml) - -testCreateTopics() { - NOW=$(date +%s) - - # TOPICS array contains the topic name to create / validate - # CLEANUP array contains the expected cleanup policy configuration for the topic - TOPICS[0]="default-$NOW" - CLEANUP[0]="" - - TOPICS[1]="compact-$NOW" - CLEANUP[1]="compact,compression.type=snappy" - - KAFKA_CREATE_TOPICS="${TOPICS[0]}:1:1,${TOPICS[1]}:2:1:compact --config=compression.type=snappy" create-topics.sh - - # Loop through each array, validate that topic exists, and correct cleanup policy is set - for i in "${!TOPICS[@]}"; do - TOPIC=${TOPICS[i]} - - echo "Validating topic '$TOPIC'" - - EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC") - POLICY=$(/opt/kafka/bin/kafka-configs.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --entity-type topics --entity-name "$TOPIC" --describe | grep 'Configs for topic' | awk -F'cleanup.policy=' '{print $2}') - - RESULT="$EXISTS:$POLICY" - EXPECTED="$TOPIC:${CLEANUP[i]}" - - if [[ "$RESULT" != "$EXPECTED" ]]; then - echo "$TOPIC topic not configured correctly: '$RESULT'" - return 1 - fi - done - - return 0 -} - -# mock the netstat call as made by the create-topics.sh script -function netstat() { echo "1 2 3 :$KAFKA_PORT"; } -export -f netstat - -testCreateTopics diff --git a/tests/kafka_container/test/0.9/test.snappy.kafkacat.sh b/tests/kafka_container/test/0.9/test.snappy.kafkacat.sh deleted file mode 100755 index 1be89101..00000000 --- a/tests/kafka_container/test/0.9/test.snappy.kafkacat.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e - -source version.functions - -testSnappy() { - echo 'foo,bar' | eval "kafkacat -X compression.codec=snappy -b $BROKER_LIST $KAFKACAT_OPTS -P -D, -t snappy" - eval "kafkacat -X compression.codec=snappy -b $BROKER_LIST $KAFKACAT_OPTS -C -e -t snappy" -} - -testSnappy diff --git a/tests/kafka_container/test/0.9/test.start-kafka-advertised-listeners.kafka.sh b/tests/kafka_container/test/0.9/test.start-kafka-advertised-listeners.kafka.sh deleted file mode 100755 index 92ed26f9..00000000 --- a/tests/kafka_container/test/0.9/test.start-kafka-advertised-listeners.kafka.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testAdvertisedListeners() { - # Given a hostname is provided - export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://my.domain.com:9040" - export KAFKA_LISTENERS="PLAINTEXT://:9092" - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration file is correct - assertExpectedConfig 'advertised.listeners=PLAINTEXT://my.domain.com:9040' - assertExpectedConfig 'listeners=PLAINTEXT://:9092' -} - -testAdvertisedListeners diff --git a/tests/kafka_container/test/0.9/test.start-kafka-listeners.kafka.sh b/tests/kafka_container/test/0.9/test.start-kafka-listeners.kafka.sh deleted file mode 100755 index ee5abb10..00000000 --- a/tests/kafka_container/test/0.9/test.start-kafka-listeners.kafka.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testListeners() { - # Given a hostname is provided - export KAFKA_LISTENERS="PLAINTEXT://internal.domain.com:9040" - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration file is correct - assertAbsent 'advertised.host.name' - assertAbsent 'advertised.port' - assertAbsent 'advertised.listeners' - assertExpectedConfig 'listeners=PLAINTEXT://internal.domain.com:9040' -} - -testListeners diff --git a/tests/kafka_container/test/0.9/test.start-kafka-multiple-listeners.kafka.sh b/tests/kafka_container/test/0.9/test.start-kafka-multiple-listeners.kafka.sh deleted file mode 100755 index 0ca08f4e..00000000 --- a/tests/kafka_container/test/0.9/test.start-kafka-multiple-listeners.kafka.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -e -source test.functions - -testMultipleAdvertisedListeners() { - # Given multiple advertised listeners - export HOSTNAME_COMMAND="f() { echo 'monkey.domain'; }; f" - export KAFKA_ADVERTISED_LISTENERS="INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094" - export KAFKA_LISTENERS="INSIDE://:9092,OUTSIDE://:9094" - export KAFKA_LISTENER_SECURITY_PROTOCOL_MAP="INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" - export KAFKA_INTER_BROKER_LISTENER_NAME="INSIDE" - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration file is correct - assertAbsent "advertised.host.name" - assertAbsent "advertised.port" - - assertExpectedConfig "advertised.listeners=INSIDE://:9092,OUTSIDE://monkey.domain:9094" - assertExpectedConfig "listeners=INSIDE://:9092,OUTSIDE://:9094" - assertExpectedConfig "listener.security.protocol.map=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" - assertExpectedConfig "inter.broker.listener.name=INSIDE" -} - -testMultipleAdvertisedListeners diff --git a/tests/kafka_container/test/Readme.md b/tests/kafka_container/test/Readme.md deleted file mode 100644 index cdc72c19..00000000 --- a/tests/kafka_container/test/Readme.md +++ /dev/null @@ -1,31 +0,0 @@ -Tests -===== - -This directory contains some basic tests to validate functionality after building. - -To execute ----------- - -``` -cd test -docker-compose up -d zookeeper kafka_1 kafka_2 -./runAllTests.sh -``` - -Run selected tests ------------------- - -### Kafka - -``` -docker-compose run --rm kafkatest -``` - -### Kafkacat - -``` -BROKER_LIST=$(./internal-broker-list.sh) [KAFKA_VERSION=] docker-compose run --rm kafkacattest -``` - -- `` is the kafka version that the tests are targeting. Normally this environment variable should not need to be specified. The default should be the latest image version. Added for CI support. -- `` can be an individual filename, or a pattern such as `'0.0/test.start-kafka*.kafka.sh'` diff --git a/tests/kafka_container/test/docker-compose.yml b/tests/kafka_container/test/docker-compose.yml deleted file mode 100644 index 42a7ca59..00000000 --- a/tests/kafka_container/test/docker-compose.yml +++ /dev/null @@ -1,59 +0,0 @@ -version: '2.1' - -x-kafka-defaults: &kafka-defaults - image: wurstmeister/kafka - ports: - - "9092" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - -x-kafka-environment-defaults: &kafka-environment-defaults - HOSTNAME_COMMAND: "echo $$(hostname)" - KAFKA_ADVERTISED_PORT: 9092 - KAFKA_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - -services: - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181" - kafka_1: - <<: *kafka-defaults - container_name: test_kafka_1 - environment: - <<: *kafka-environment-defaults - KAFKA_BROKER_ID: 1 - kafka_2: - <<: *kafka-defaults - container_name: test_kafka_2 - environment: - <<: *kafka-environment-defaults - KAFKA_BROKER_ID: 2 - - kafkatest: - image: wurstmeister/kafka - environment: - KAFKA_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - volumes: - - .:/tests - - /var/run/docker.sock:/var/run/docker.sock - working_dir: /tests - entrypoint: - - ./runTestPattern.sh - command: - - "*/*.kafka.sh" - - kafkacattest: - image: confluentinc/cp-kafkacat:5.0.0 - environment: - - BROKER_LIST - - KAFKA_VERSION=${KAFKA_VERSION-2.7.0} - volumes: - - .:/tests - working_dir: /tests - entrypoint: - - ./runTestPattern.sh - command: - - "*/*.kafkacat.sh" diff --git a/tests/kafka_container/test/internal-broker-list.sh b/tests/kafka_container/test/internal-broker-list.sh deleted file mode 100755 index dac67a83..00000000 --- a/tests/kafka_container/test/internal-broker-list.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -CONTAINERS=$(docker inspect -f '{{ .NetworkSettings.Networks.test_default.IPAddress }}' test_kafka_1 test_kafka_2 | awk '{printf "%s:9092\n", $1}' | tr '\n' ',') -echo "${CONTAINERS%,}" diff --git a/tests/kafka_container/test/runAllTests.sh b/tests/kafka_container/test/runAllTests.sh deleted file mode 100755 index a575b21f..00000000 --- a/tests/kafka_container/test/runAllTests.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -e - -BROKER_LIST=$(./internal-broker-list.sh) -export BROKER_LIST - -echo "BROKER_LIST=$BROKER_LIST" - -runAll() { - # Tests that require kafka - docker-compose run --rm kafkatest - - RESULT=$? - if [[ $RESULT -eq 0 ]]; then - # Tests that require kafkacat - docker-compose run --rm kafkacattest - RESULT=$? - fi - - return $RESULT -} - -runAll -result=$? -echo "exit status $result" -exit $result diff --git a/tests/kafka_container/test/runTestPattern.sh b/tests/kafka_container/test/runTestPattern.sh deleted file mode 100755 index 16220eeb..00000000 --- a/tests/kafka_container/test/runTestPattern.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -e - -source version.functions - -PATTERN=$1 -VERSION=$KAFKA_VERSION - -# Allow version to be overridden by -v/--version flag -while [[ "$#" -gt 0 ]]; do - case $1 in - -v|--version) - VERSION="$2"; - shift - ;; - *) - PATTERN="$1" - ;; - esac - shift -done - -echo "" -echo "" -echo "Running tests for Kafka $VERSION with pattern $PATTERN" - -runPattern() { - for t in $PATTERN; do - echo - echo "====================================" - - # only run tests compatible with this version of Kafka - TARGET=$(echo "$t" | cut -d/ -f1) - RESULT=$(compareVersion "$VERSION" "$TARGET") - echo "Kafka $VERSION is '$RESULT' target $TARGET of test $t" - if [[ "$RESULT" != "<" ]]; then - echo " testing '$t'" - ( source "$t" ) - status=$? - if [[ -z "$status" || ! "$status" -eq 0 ]]; then - return $status - fi - fi - done - - return $? -} - -runPattern - -exit $? diff --git a/tests/kafka_container/test/scenarios/Readme.md b/tests/kafka_container/test/scenarios/Readme.md deleted file mode 100644 index 11c968f0..00000000 --- a/tests/kafka_container/test/scenarios/Readme.md +++ /dev/null @@ -1,27 +0,0 @@ -Scenarios (end-to-end tests) -============================ - -These tests are supposed to test the configuration of indivdiual features - -TODO: ------ - -- SSL (Client + Broker) -- Security - -Done: ------ - -- JMX - -Executing tests ---------------- - -These tests should spin up required containers for full end-to-end testing and exercise required code paths, returing zero exit code for success and non-zero exit code for failure. - -### JMX - -``` -cd test/scenarios -./runJmxScenario.sh -``` diff --git a/tests/kafka_container/test/scenarios/jmx/docker-compose.yml b/tests/kafka_container/test/scenarios/jmx/docker-compose.yml deleted file mode 100644 index aba78bf2..00000000 --- a/tests/kafka_container/test/scenarios/jmx/docker-compose.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: '2' -services: - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181" - kafka: - image: wurstmeister/kafka - ports: - - "9092" - - "1099" - environment: - KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_ADVERTISED_PORT: 9092 - KAFKA_PORT: 9092 - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka -Dcom.sun.management.jmxremote.rmi.port=1099" - JMX_PORT: 1099 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - jmxexporter: - image: sscaling/jmx-prometheus-exporter - ports: - - "5556:5556" - environment: - SERVICE_PORT: 5556 - volumes: - - $PWD/jmxexporter.yml:/opt/jmx_exporter/config.yml - - test: - image: wurstmeister/kafka - environment: - KAFKA_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - volumes: - - .:/scenario - working_dir: /scenario - entrypoint: - - /bin/bash - - -c - command: - - /scenario/test.sh - diff --git a/tests/kafka_container/test/scenarios/jmx/jmxexporter.yml b/tests/kafka_container/test/scenarios/jmx/jmxexporter.yml deleted file mode 100644 index f365449c..00000000 --- a/tests/kafka_container/test/scenarios/jmx/jmxexporter.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -startDelaySeconds: 3 -hostPort: kafka:1099 -username: -password: - -whitelistObjectNames: ["kafka.server:type=BrokerTopicMetrics,*"] -rules: - - pattern: ".*" diff --git a/tests/kafka_container/test/scenarios/jmx/test.sh b/tests/kafka_container/test/scenarios/jmx/test.sh deleted file mode 100755 index 286e9a0f..00000000 --- a/tests/kafka_container/test/scenarios/jmx/test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -e -o pipefail - -echo "Sleeping 5 seconds until Kafka is started" -sleep 5 - -echo "Checking to see if Kafka is alive" -echo "dump" | nc -w 20 zookeeper 2181 | fgrep "/brokers/ids/" - -echo "Check JMX" -curl -s jmxexporter:5556/metrics | grep 'kafka_server_BrokerTopicMetrics_MeanRate{name="MessagesInPerSec",' diff --git a/tests/kafka_container/test/scenarios/runJmxScenario.sh b/tests/kafka_container/test/scenarios/runJmxScenario.sh deleted file mode 100755 index e99b8851..00000000 --- a/tests/kafka_container/test/scenarios/runJmxScenario.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e -o pipefail - -pushd jmx -docker-compose up -d zookeeper kafka jmxexporter -docker-compose run --rm test -docker-compose stop -popd diff --git a/tests/kafka_container/test/test.functions b/tests/kafka_container/test/test.functions deleted file mode 100644 index 2894cadd..00000000 --- a/tests/kafka_container/test/test.functions +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -e - -# Sourcing this script will replace any of the external calls in the start-kafka.sh script and mock -# any outbound calls (i.e. to docker) - -export START_KAFKA=/usr/bin/start-kafka.sh -export BROKER_CONFIG=/opt/kafka/config/server.properties -export ORIG_LOG4J_CONFIG=/opt/kafka/config/log4j.properties - -enforceOriginalFile() { - SOURCE="$1" - BACKUP="$1.bak" - - if [[ ! -f "$BACKUP" ]]; then - echo "Backing up $SOURCE to $BACKUP" - cp "$SOURCE" "$BACKUP" - else - echo "Restoring $SOURCE from $BACKUP" - cp "$BACKUP" "$SOURCE" - fi -} - -setupStartKafkaScript() { - echo "Preparing $START_KAFKA script for test" - - enforceOriginalFile "$START_KAFKA" - enforceOriginalFile "$BROKER_CONFIG" - enforceOriginalFile "$ORIG_LOG4J_CONFIG" - - # We need to remove all executable commands from the start-kafka.sh script, so it can be sourced to evaluate - # the environment variables - sed -i -E -e '/^create-topics.sh/d' -e '/^exec "\$KAFKA_HOME\/bin\/kafka-server-start.sh"/d' "$START_KAFKA" - - # Mock the call to docker port to return valid result - function docker() { echo "0.0.0.0:9092"; } - export -f docker -} - -setupStartKafkaScript - - -# Will look in the server.properties file, and check if there is an exact match for the provided input -# i.e. `assertExpectedConfig 'broker.id=123'` -# This will only succeed if there is exactly one line matching `broker.id=123` -assertExpectedConfig() { - EXPECTED=$1 - - COUNT=$(grep -E '^'"$EXPECTED"'$' "$BROKER_CONFIG" | wc -l) - RESULT=$(grep -E '^'"$EXPECTED"'$' "$BROKER_CONFIG") - echo " > $COUNT matches of $RESULT" - - [[ "$RESULT" == "$EXPECTED" && "$COUNT" == "1" ]] -} - -assertExpectedLog4jConfig() { - EXPECTED=$1 - - RESULT=$(grep -E '^'"$EXPECTED"'$' "$ORIG_LOG4J_CONFIG") - echo " > $RESULT" - - [[ "$RESULT" == "$EXPECTED" ]] -} - - -assertAbsent() { - EXPECTED_ABSENT=$1 - - RESULT=$(grep -E '^'"$EXPECTED_ABSENT" "$BROKER_CONFIG" | wc -l) - echo " > $RESULT matches for ^$EXPECTED_ABSENT" - - [[ "$RESULT" == "0" ]] -} - -printBrokerConfig() { - echo "----[ $BROKER_CONFIG ]----" - cat "$BROKER_CONFIG" | sed -nE '/^[^#]/p' - echo "--------------------------" -} diff --git a/tests/kafka_container/test/verifyImageLabels.sh b/tests/kafka_container/test/verifyImageLabels.sh deleted file mode 100755 index f59c9985..00000000 --- a/tests/kafka_container/test/verifyImageLabels.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e - -VCS_REF=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.vcs-ref"}}' wurstmeister/kafka) -echo "VCS_REF=$VCS_REF" -if [ -z "$VCS_REF" ] || [ "$VCS_REF" = "unspecified" ]; then - echo "org.label-schema.vcs-ref is empty or unspecified" - exit 1 -fi -if ! git cat-file -e "$VCS_REF^{commit}"; then - echo "$VCS_REF Not a valid git commit" - exit 1 -fi - -BUILD_DATE=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.build-date"}}' wurstmeister/kafka) -echo "BUILD_DATE=$BUILD_DATE" -if ! date -d "$BUILD_DATE"; then - echo "$BUILD_DATE Not a valid date" - exit 1 -fi -exit 0 diff --git a/tests/kafka_container/test/version.functions b/tests/kafka_container/test/version.functions deleted file mode 100644 index 65c00f60..00000000 --- a/tests/kafka_container/test/version.functions +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -e - -# Modified from https://stackoverflow.com/a/4025065 -compareVersion() { - # Only care about major / minor - LEFT=$(echo "$1" | cut -d. -f1-2) - RIGHT=$(echo "$2" | cut -d. -f1-2) - if [[ "$LEFT" != "$RIGHT" ]] - then - local IFS=. - local i ver1=($LEFT) ver2=($RIGHT) - for ((i=0; i<${#ver1[@]}; i++)) - do - if (( "${ver1[i]}" > "${ver2[i]}" )) - then - echo ">" - return - fi - if (( "${ver1[i]}" < "${ver2[i]}" )) - then - echo "<" - return - fi - done - fi - echo "=" -} - -# https://github.com/edenhill/librdkafka/wiki/Broker-version-compatibility -# To support different broker versions, we need to configure kafkacat differently -VERSION_8=$(compareVersion "$KAFKA_VERSION" "0.8") -VERSION_9=$(compareVersion "$KAFKA_VERSION" "0.9") - -if [[ "$VERSION_8" == "=" || "$VERSION_9" == "=" ]]; then - export KAFKACAT_OPTS="-Xapi.version.request=false -Xbroker.version.fallback=$KAFKA_VERSION" - echo "[INFO] Using kafkacat opts on older version '$KAFKACAT_OPTS'" -fi diff --git a/tests/kafka_container/versions.sh b/tests/kafka_container/versions.sh deleted file mode 100755 index d790d1a4..00000000 --- a/tests/kafka_container/versions.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -e - -MAJOR_VERSION=$(echo "$KAFKA_VERSION" | cut -d. -f1) -export MAJOR_VERSION - -MINOR_VERSION=$(echo "$KAFKA_VERSION" | cut -d. -f2) -export MINOR_VERSION diff --git a/tests/local_db.lmdb/data.mdb b/tests/local_db.lmdb/data.mdb deleted file mode 100644 index 16266c96084b0b01544c5905f99a86cb14a6b772..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8192 zcmeI#yA6OK5CG7#vvmK;NsN=Yf+J`G5<&Zuyqu5YT>FowH|+2PF14{}*7jo+)%qCATHN4jYU5FkK+009C72oNAZfB*pk X1PBlyK!5-N0t5&UAV7csfxiMjkM$Kb From a62ba54598af3997509a215ff0ad1a9a07b3d8b6 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 23 Apr 2021 17:54:30 +0200 Subject: [PATCH 065/239] Update broker.py Broker: add a column to distinguish if the row is an Event or a Command #31 --- minos/networks/broker.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 45e4205c..fb362b3c 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -9,7 +9,10 @@ import datetime import typing as t -import aiomisc +from enum import ( + Enum, +) + import aiopg from aiokafka import AIOKafkaProducer from aiomisc.service.periodic import PeriodicService, Service @@ -51,6 +54,8 @@ class CommandModel(MinosModel): class MinosEventBroker(MinosBaseBroker): + ACTION = "event" + def __init__(self, topic: str, config: MinosConfig): self.config = config self.topic = topic @@ -66,8 +71,8 @@ async def send(self, model: Aggregate): async with MinosBrokerDatabase().get_connection(self.config) as connect: async with connect.cursor() as cur: await cur.execute( - "INSERT INTO queue (topic, model, retry, creation_date, update_date) VALUES (%s, %s, %s, %s, %s) RETURNING queue_id;", - (event_instance.topic, bin_data, 0, datetime.datetime.now(), datetime.datetime.now(),), + "INSERT INTO queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING queue_id;", + (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), ) queue_id = await cur.fetchone() @@ -77,6 +82,8 @@ async def send(self, model: Aggregate): class MinosCommandBroker(MinosBaseBroker): + ACTION = "command" + def __init__(self, topic: str, config: MinosConfig): self.config = config self.topic = topic @@ -92,8 +99,8 @@ async def send(self, model: Aggregate, callback: t.Callable): async with MinosBrokerDatabase().get_connection(self.config) as connect: async with connect.cursor() as cur: await cur.execute( - "INSERT INTO queue (topic, model, retry, creation_date, update_date) VALUES (%s, %s, %s, %s, %s) RETURNING queue_id;", - (event_instance.topic, bin_data, 0, datetime.datetime.now(), datetime.datetime.now(),), + "INSERT INTO queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING queue_id;", + (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), ) queue_id = await cur.fetchone() @@ -106,9 +113,9 @@ async def broker_table_creation(config: MinosConfig): async with MinosBrokerDatabase().get_connection(config) as connect: async with connect.cursor() as cur: await cur.execute( - 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, ' + 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" BIGSERIAL NOT NULL PRIMARY KEY, ' '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' - '"creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' + '"action" VARCHAR(255) NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' ) From 5974012c5db925778cd1561cf0377ce0c84d241e Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 23 Apr 2021 18:08:27 +0200 Subject: [PATCH 066/239] Update broker.py Broker must load retry parameter from MinosConfig Queue #9 --- minos/networks/broker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index fb362b3c..9695b042 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -150,7 +150,7 @@ async def send_to_kafka(topic: str, message: bytes, config: MinosConfig): async def broker_queue_dispatcher(config: MinosConfig): async with MinosBrokerDatabase().get_connection(config) as connect: async with connect.cursor() as cur: - await cur.execute("SELECT * FROM queue WHERE retry <= 2 ORDER BY creation_date ASC LIMIT 10;") + await cur.execute("SELECT * FROM queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;", (config.events.queue.retry, config.events.queue.records)) async for row in cur: sent_to_kafka = False try: From 7c6b027009ecaaa5294b04bacabaf3b09bbebcb3 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 23 Apr 2021 16:08:50 +0000 Subject: [PATCH 067/239] Restyled by black --- minos/networks/broker.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 9695b042..1578814c 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -9,9 +9,7 @@ import datetime import typing as t -from enum import ( - Enum, -) +from enum import Enum import aiopg from aiokafka import AIOKafkaProducer @@ -150,7 +148,10 @@ async def send_to_kafka(topic: str, message: bytes, config: MinosConfig): async def broker_queue_dispatcher(config: MinosConfig): async with MinosBrokerDatabase().get_connection(config) as connect: async with connect.cursor() as cur: - await cur.execute("SELECT * FROM queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;", (config.events.queue.retry, config.events.queue.records)) + await cur.execute( + "SELECT * FROM queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;", + (config.events.queue.retry, config.events.queue.records), + ) async for row in cur: sent_to_kafka = False try: From 01f00b360541834f8b7ced760c7d0677c9996ee3 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 23 Apr 2021 16:08:57 +0000 Subject: [PATCH 068/239] Restyled by isort --- minos/networks/broker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 1578814c..8c27f3b1 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -8,7 +8,6 @@ import asyncio import datetime import typing as t - from enum import Enum import aiopg From 8be9d4f1a81dd297f9c00c2d7e5a0ae5347ddeb8 Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Fri, 23 Apr 2021 18:48:01 +0200 Subject: [PATCH 069/239] working on event handler --- minos/networks/event.py | 141 +++++----------------------------------- tests/test_config.yaml | 3 - 2 files changed, 16 insertions(+), 128 deletions(-) diff --git a/minos/networks/event.py b/minos/networks/event.py index 9dcaaf52..2fa0e8f2 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -1,96 +1,16 @@ import asyncio import functools import typing as t -from collections import Counter -from aiokafka import AIOKafkaConsumer, ConsumerRebalanceListener +from aiokafka import AIOKafkaConsumer from aiomisc import Service -from kafka.errors import OffsetOutOfRangeError from minos.common.configuration.config import MinosConfig from minos.common.importlib import import_module from minos.common.logs import log -from minos.common.storage.abstract import MinosStorage -from minos.common.storage.lmdb import MinosStorageLmdb from minos.networks.exceptions import MinosNetworkException -class MinosLocalState: - __slots__ = "_counts", "_offsets", "_storage", "_dbname" - - def __init__(self, storage: MinosStorage, db_name="LocalState"): - self._counts = {} - self._offsets = {} - self._dbname = db_name - self._storage = storage - - def dump_local_state(self): - - for tp in self._counts: - key = f"{tp.topic}:{tp.partition}" - state = { - "last_offset": self._offsets[tp], - "counts": dict(self._counts[tp]) - } - actual_state = self._storage.get(self._dbname, key) - if actual_state is not None: - self._storage.update(self._dbname, key, state) - else: - self._storage.add(self._dbname, key, state) - - def load_local_state(self, partitions): - self._counts.clear() - self._offsets.clear() - for tp in partitions: - # prepare the default state - state = { - "last_offset": -1, # Non existing, will reset - "counts": {} - } - key = f"{tp.topic}:{tp.partition}" - returned_val = self._storage.get(self._dbname, key) - if returned_val is not None: - state = returned_val - self._counts[tp] = Counter(state['counts']) - self._offsets[tp] = state['last_offset'] - - def discard_state(self, tps): - """ - reset the entire memory database with the default values - """ - for tp in tps: - self._offsets[tp] = -1 - self._counts[tp] = Counter() - - def get_last_offset(self, tp): - return self._offsets[tp] - - def add_counts(self, tp, counts, las_offset): - self._counts[tp] += counts - self._offsets[tp] = las_offset - - -class MinosRebalanceListener(ConsumerRebalanceListener): - - def __init__(self, consumer, database_state: MinosLocalState): - self._consumer = consumer - self._state = database_state - - async def on_partitions_revoked(self, revoked): - log.debug("KAFKA Consumer: Revoked %s", revoked) - self._state.dump_local_state() - - async def on_partitions_assigned(self, assigned): - log.debug("KAFKA Consumer: Assigned %s", assigned) - self._state.load_local_state(partitions=assigned) - for tp in assigned: - last_offset = self._state.get_last_offset(tp) - if last_offset < 0: - await self._consumer.seek_to_beginning(tp) - else: - self._consumer.seek(tp, last_offset + 1) - - class MinosEventServer(Service): """ Event Manager @@ -98,17 +18,15 @@ class MinosEventServer(Service): Consumer for the Broker ( at the moment only Kafka is supported ) """ - __slots__ = "_tasks", "_local_state", "_storage", "_handlers", "_broker_host", "_broker_port", "_broker_group" + __slots__ = "_tasks", "_handlers", "_topics", "_broker_group_name" - def __init__(self, *, conf: MinosConfig, storage: MinosStorage = MinosStorageLmdb, **kwargs: t.Any): + def __init__(self, *, conf: MinosConfig, **kwargs: t.Any): self._tasks = set() # type: t.Set[asyncio.Task] - self._broker_host = conf.events.broker.host - self._broker_port = conf.events.broker.port - self._broker_group = f"{conf.service.name}_group_id" - self._handler = conf.events.items - self._storage = storage.build(conf.events.database.path) - self._local_state = MinosLocalState(storage=self._storage, db_name=conf.events.database.name) - + self._db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ + f"password={conf.events.queue.password} host={conf.events.queue.host}" + self._handler = {item['name']: {'controller': item['controller'], 'action': item['action']} + for item in conf.events.items} + self._topics = list(self._handler.keys()) super().__init__(**kwargs) def create_task(self, coro: t.Awaitable[t.Any]): @@ -116,54 +34,27 @@ def create_task(self, coro: t.Awaitable[t.Any]): self._tasks.add(task) task.add_done_callback(self._tasks.remove) - async def save_state_every_second(self): - while True: - try: - await asyncio.sleep(1) - except asyncio.CancelledError: - break - self._local_state.dump_local_state() - async def handle_message(self, consumer: t.Any): while True: - try: - msg_set = await consumer.getmany(timeout_ms=1000) - except OffsetOutOfRangeError as err: - # this is the case that the database is not updated and must be reset - tps = err.args[0].keys() - self._local_state.discard_state(tps) - await consumer.seek_to_beginning(*tps) - continue - for tp, msgs in msg_set.items(): - log.debug(f"EVENT Manager topic: {tp}") - log.debug(f"EVENT Manager msg: {msgs}") - # check if the topic is managed by the handler - if tp.topic in self._handlers: - counts = Counter() - for msg in msgs: - counts[msg.key] += 1 - func = self._get_event_handler(tp.topic) - func(msg.value) - self._local_state.add_counts(tp, counts, msg.offset) + async for msg in consumer: + # the handler receive a message and store in the queue database + topic = msg.topic + partition = msg.partition + event_binary = msg.value async def start(self) -> t.Any: self.start_event.set() log.debug("Event Consumer Manager: Started") # start the Service Event Consumer for Kafka consumer = AIOKafkaConsumer(loop=self.loop, - enable_auto_commit=False, - auto_offset_reset="none", group_id=self._broker_group, + auto_offset_reset="latest", bootstrap_servers=f"{self._broker_host}:{self._broker_port}", - key_deserializer=lambda key: key.decode("utf-8") if key else "", ) + ) await consumer.start() - # prepare the database interface - listener = MinosRebalanceListener(consumer, self._local_state) - topics: t.List = list(self._handlers.keys()) - consumer.subscribe(topics=topics, listener=listener) + consumer.subscribe(self._topics) - self.create_task(self.save_state_every_second()) self.create_task(self.handle_message(consumer)) def _get_event_handler(self, topic: str) -> t.Callable: diff --git a/tests/test_config.yaml b/tests/test_config.yaml index 03b3dc3a..bd3e0326 100644 --- a/tests/test_config.yaml +++ b/tests/test_config.yaml @@ -12,9 +12,6 @@ rest: events: broker: kafka port: 9092 - database: - path: ./tests/local_db.lmdb - name: database_events_test items: - name: TicketAdded controller: tests.services.CqrsTestService.CqrsService From d5ef10a4091374e6537171386d79e0b5041d5423 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Mon, 26 Apr 2021 12:33:17 +0200 Subject: [PATCH 070/239] Command Broker reply_on Broker Command must have callback parameter #50 --- minos/networks/broker.py | 6 +++--- tests/broker/test_broker.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 8c27f3b1..f06ed288 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -47,7 +47,7 @@ class CommandModel(MinosModel): model: str # items: list[ModelRef[Aggregate]] items: list[str] - callback: str + reply_on: str class MinosEventBroker(MinosBaseBroker): @@ -89,8 +89,8 @@ def __init__(self, topic: str, config: MinosConfig): def _database(self): pass - async def send(self, model: Aggregate, callback: t.Callable): - event_instance = CommandModel(topic=self.topic, model="Change", items=[str(model)], callback=callback) + async def send(self, model: Aggregate, reply_on: str): + event_instance = CommandModel(topic=self.topic, model="Change", items=[str(model)], reply_on=reply_on) bin_data = event_instance.avro_bytes async with MinosBrokerDatabase().get_connection(self.config) as connect: diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index dc7b81c0..1c3d0db9 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -73,15 +73,15 @@ async def test_commands_broker_insertion(self): m = MinosCommandBroker("CommandBroker", self._broker_config()) - affected_rows, queue_id = await m.send(model=a, callback="test") + affected_rows, queue_id = await m.send(model=a, reply_on="test_reply_on") assert affected_rows == 1 assert queue_id > 0 async def test_if_commands_was_deleted(self): a = AggregateTest(test_id=1, test=2) m = MinosCommandBroker("CommandBroker-Delete", self._broker_config()) - affected_rows_1, queue_id_1 = await m.send(a, callback="test1") - affected_rows_2, queue_id_2 = await m.send(a, callback="test2") + affected_rows_1, queue_id_1 = await m.send(a, reply_on="test_reply_on") + affected_rows_2, queue_id_2 = await m.send(a, reply_on="test_reply_on") await broker_queue_dispatcher(self._broker_config()) @@ -100,8 +100,8 @@ async def test_if_commands_was_deleted(self): async def test_if_commands_retry_was_incremented(self): a = AggregateTest(test_id=1, test=2) m = MinosCommandBroker("CommandBroker-Delete", self._broker_config()) - affected_rows_1, queue_id_1 = await m.send(a, callback="test1") - affected_rows_2, queue_id_2 = await m.send(a, callback="test2") + affected_rows_1, queue_id_1 = await m.send(a, reply_on="test_reply_on") + affected_rows_2, queue_id_2 = await m.send(a, reply_on="test_reply_on") config = MinosConfig(path="./tests/wrong_test_config.yaml") From 38fcb013f1a80c0e5ab7dea41a9d081b009373dd Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Mon, 26 Apr 2021 12:58:57 +0200 Subject: [PATCH 071/239] Modify Broker table to producer_queue Broker Event: modify the name of the broker table from queue to producer_queue #47 --- minos/networks/broker.py | 12 ++++++------ tests/broker/test_broker.py | 18 +++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index f06ed288..4517f3ec 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -68,7 +68,7 @@ async def send(self, model: Aggregate): async with MinosBrokerDatabase().get_connection(self.config) as connect: async with connect.cursor() as cur: await cur.execute( - "INSERT INTO queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING queue_id;", + "INSERT INTO queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), ) @@ -96,7 +96,7 @@ async def send(self, model: Aggregate, reply_on: str): async with MinosBrokerDatabase().get_connection(self.config) as connect: async with connect.cursor() as cur: await cur.execute( - "INSERT INTO queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING queue_id;", + "INSERT INTO queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), ) @@ -110,7 +110,7 @@ async def broker_table_creation(config: MinosConfig): async with MinosBrokerDatabase().get_connection(config) as connect: async with connect.cursor() as cur: await cur.execute( - 'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" BIGSERIAL NOT NULL PRIMARY KEY, ' + 'CREATE TABLE IF NOT EXISTS "producer_queue" ("id" BIGSERIAL NOT NULL PRIMARY KEY, ' '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' '"action" VARCHAR(255) NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' ) @@ -148,7 +148,7 @@ async def broker_queue_dispatcher(config: MinosConfig): async with MinosBrokerDatabase().get_connection(config) as connect: async with connect.cursor() as cur: await cur.execute( - "SELECT * FROM queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;", + "SELECT * FROM producer_queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;", (config.events.queue.retry, config.events.queue.records), ) async for row in cur: @@ -158,14 +158,14 @@ async def broker_queue_dispatcher(config: MinosConfig): if sent_to_kafka: # Delete from database If the event was sent successfully to Kafka. async with connect.cursor() as cur2: - await cur2.execute("DELETE FROM queue WHERE queue_id=%d;" % row[0]) + await cur2.execute("DELETE FROM producer_queue WHERE id=%d;" % row[0]) except: sent_to_kafka = False finally: if not sent_to_kafka: # Update queue retry column. Increase by 1. async with connect.cursor() as cur3: - await cur3.execute("UPDATE queue SET retry = retry + 1 WHERE queue_id=%d;" % row[0]) + await cur3.execute("UPDATE producer_queue SET retry = retry + 1 WHERE id=%d;" % row[0]) class EventBrokerQueueDispatcher(PeriodicService): diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index 1c3d0db9..24a41935 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -26,7 +26,7 @@ async def test_if_queue_table_exists(self): async with connect.cursor() as cur: await cur.execute( - "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'queue';" + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'producer_queue';" ) ret = [] async for row in cur: @@ -59,7 +59,7 @@ async def test_if_events_was_deleted(self): database = await self._database() async with database as connect: async with connect.cursor() as cur: - await cur.execute("SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "EventBroker-Delete") + await cur.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "EventBroker-Delete") records = await cur.fetchone() assert affected_rows_1 == 1 @@ -88,7 +88,7 @@ async def test_if_commands_was_deleted(self): database = await self._database() async with database as connect: async with connect.cursor() as cur: - await cur.execute("SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "CommandBroker-Delete") + await cur.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "CommandBroker-Delete") records = await cur.fetchone() assert affected_rows_1 == 1 @@ -110,13 +110,13 @@ async def test_if_commands_retry_was_incremented(self): database = await self._database() async with database as connect: async with connect.cursor() as cur: - await cur.execute("SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "CommandBroker-Delete") + await cur.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "CommandBroker-Delete") records = await cur.fetchone() - await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_1) + await cur.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_1) retry_1 = await cur.fetchone() - await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_2) + await cur.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_2) retry_2 = await cur.fetchone() assert affected_rows_1 == 1 @@ -140,13 +140,13 @@ async def test_if_events_retry_was_incremented(self): database = await self._database() async with database as connect: async with connect.cursor() as cur: - await cur.execute("SELECT COUNT(*) FROM queue WHERE topic = '%s'" % "EventBroker-Delete") + await cur.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "EventBroker-Delete") records = await cur.fetchone() - await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_1) + await cur.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_1) retry_1 = await cur.fetchone() - await cur.execute("SELECT retry FROM queue WHERE queue_id=%d;" % queue_id_2) + await cur.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_2) retry_2 = await cur.fetchone() assert affected_rows_1 == 1 From d8c32f09cecaedad54af6eaac8d227a72fb76e5e Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Mon, 26 Apr 2021 16:12:12 +0200 Subject: [PATCH 072/239] Update requirements_dev.txt Update minos_common_library to 0.0.3 #52 --- requirements_dev.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index c2cf2c7e..8f61b0b8 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -19,6 +19,7 @@ colorlog==5.0.1 coverage==5.5 distlib==0.3.1 docutils==0.16 +fastavro==1.4.0 filelock==3.0.12 flake8==3.9.0 idna==2.10 @@ -31,7 +32,7 @@ keyring==23.0.1 lmdb==1.1.1 MarkupSafe==1.1.1 mccabe==0.6.1 -minos-microservice-common==0.0.2 +minos-microservice-common==0.0.3 more-itertools==8.7.0 mypy-extensions==0.4.3 orjson==3.5.1 From e52335dcc524a60a5da140e2959a069de23d995a Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Mon, 26 Apr 2021 16:37:40 +0200 Subject: [PATCH 073/239] Adjust Broker Models Broker must create and store EventModel/CommandModel #6 --- minos/networks/broker.py | 33 +++++++-------------------------- tests/broker/test_broker.py | 17 ++++++++++------- 2 files changed, 17 insertions(+), 33 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 4517f3ec..4b6050a4 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -13,8 +13,8 @@ import aiopg from aiokafka import AIOKafkaProducer from aiomisc.service.periodic import PeriodicService, Service -from minos.common import MinosModel, ModelRef -from minos.common.broker import MinosBaseBroker +from minos.common import Aggregate +from minos.common.broker import MinosBaseBroker, Event, Command from minos.common.configuration.config import MinosConfig from minos.common.logs import log @@ -31,25 +31,6 @@ def get_connection(self, conf): return conn -class Aggregate(MinosModel): - test_id: int - - -class EventModel(MinosModel): - topic: str - model: str - # items: list[ModelRef[Aggregate]] - items: list[str] - - -class CommandModel(MinosModel): - topic: str - model: str - # items: list[ModelRef[Aggregate]] - items: list[str] - reply_on: str - - class MinosEventBroker(MinosBaseBroker): ACTION = "event" @@ -62,13 +43,13 @@ def _database(self): pass async def send(self, model: Aggregate): - event_instance = EventModel(topic=self.topic, model="Change", items=[str(model)]) + event_instance = Event(topic=self.topic, model=model.classname, items=[model]) bin_data = event_instance.avro_bytes async with MinosBrokerDatabase().get_connection(self.config) as connect: async with connect.cursor() as cur: await cur.execute( - "INSERT INTO queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", + "INSERT INTO producer_queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), ) @@ -90,13 +71,13 @@ def _database(self): pass async def send(self, model: Aggregate, reply_on: str): - event_instance = CommandModel(topic=self.topic, model="Change", items=[str(model)], reply_on=reply_on) + event_instance = Command(topic=self.topic, model=model.classname, items=[model], reply_on=reply_on) bin_data = event_instance.avro_bytes async with MinosBrokerDatabase().get_connection(self.config) as connect: async with connect.cursor() as cur: await cur.execute( - "INSERT INTO queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", + "INSERT INTO producer_queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), ) @@ -148,7 +129,7 @@ async def broker_queue_dispatcher(config: MinosConfig): async with MinosBrokerDatabase().get_connection(config) as connect: async with connect.cursor() as cur: await cur.execute( - "SELECT * FROM producer_queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;", + "SELECT * FROM producer_queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;" % (config.events.queue.retry, config.events.queue.records), ) async for row in cur: diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index 24a41935..e73638f6 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -4,10 +4,11 @@ import pytest from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import (Aggregate, MinosCommandBroker, +from minos.networks.broker import (MinosCommandBroker, MinosEventBroker, broker_queue_dispatcher, send_to_kafka) from tests.broker.database_testcase import PostgresAsyncTestCase +from minos.common import Aggregate class AggregateTest(Aggregate): @@ -34,12 +35,14 @@ async def test_if_queue_table_exists(self): assert ret == [(1,)] + async def test_send_to_kafka_ok(self): response = await send_to_kafka(topic="TestKafkaSend", message=bytes(), config=self._broker_config()) assert response is True + async def test_events_broker_insertion(self): - a = AggregateTest(test_id=1, test=2) + a = AggregateTest(test_id=1, test=2, id=1, version=1) m = MinosEventBroker("EventBroker", self._broker_config()) @@ -49,7 +52,7 @@ async def test_events_broker_insertion(self): assert queue_id > 0 async def test_if_events_was_deleted(self): - a = AggregateTest(test_id=1, test=2) + a = AggregateTest(test_id=1, test=2, id=1, version=1) m = MinosEventBroker("EventBroker-Delete", self._broker_config()) affected_rows_1, queue_id_1 = await m.send(a) affected_rows_2, queue_id_2 = await m.send(a) @@ -69,7 +72,7 @@ async def test_if_events_was_deleted(self): assert records[0] == 0 async def test_commands_broker_insertion(self): - a = AggregateTest(test_id=1, test=2) + a = AggregateTest(test_id=1, test=2, id=1, version=1) m = MinosCommandBroker("CommandBroker", self._broker_config()) @@ -78,7 +81,7 @@ async def test_commands_broker_insertion(self): assert queue_id > 0 async def test_if_commands_was_deleted(self): - a = AggregateTest(test_id=1, test=2) + a = AggregateTest(test_id=1, test=2, id=1, version=1) m = MinosCommandBroker("CommandBroker-Delete", self._broker_config()) affected_rows_1, queue_id_1 = await m.send(a, reply_on="test_reply_on") affected_rows_2, queue_id_2 = await m.send(a, reply_on="test_reply_on") @@ -98,7 +101,7 @@ async def test_if_commands_was_deleted(self): assert records[0] == 0 async def test_if_commands_retry_was_incremented(self): - a = AggregateTest(test_id=1, test=2) + a = AggregateTest(test_id=1, test=2, id=1, version=1) m = MinosCommandBroker("CommandBroker-Delete", self._broker_config()) affected_rows_1, queue_id_1 = await m.send(a, reply_on="test_reply_on") affected_rows_2, queue_id_2 = await m.send(a, reply_on="test_reply_on") @@ -128,7 +131,7 @@ async def test_if_commands_retry_was_incremented(self): assert retry_2[0] > 0 async def test_if_events_retry_was_incremented(self): - a = AggregateTest(test_id=1, test=2) + a = AggregateTest(test_id=1, test=2, id=1, version=1) m = MinosEventBroker("EventBroker-Delete", self._broker_config()) affected_rows_1, queue_id_1 = await m.send(a) affected_rows_2, queue_id_2 = await m.send(a) From 30a26580a6abe2019e3da786390f4feb69f55fe9 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Mon, 26 Apr 2021 16:46:33 +0200 Subject: [PATCH 074/239] Missing Minos Config queue > retry parameter Bug - Missing Minos Config queue > retry parameter on *.yaml files #54 --- tests/test_config.yaml | 2 ++ tests/wrong_test_config.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/test_config.yaml b/tests/test_config.yaml index 03b3dc3a..40bce1ee 100644 --- a/tests/test_config.yaml +++ b/tests/test_config.yaml @@ -29,6 +29,7 @@ events: host: postgres port: 5432 records: 10 + retry: 2 commands: broker: kafka port: 9092 @@ -52,3 +53,4 @@ commands: host: postgres port: 5432 records: 10 + retry: 2 diff --git a/tests/wrong_test_config.yaml b/tests/wrong_test_config.yaml index 6013672e..a368e997 100644 --- a/tests/wrong_test_config.yaml +++ b/tests/wrong_test_config.yaml @@ -29,6 +29,7 @@ events: host: postgres port: 5432 records: 10 + retry: 2 commands: broker: localhost port: 4092 @@ -52,3 +53,4 @@ commands: host: postgres port: 5432 records: 10 + retry: 2 From 9e4b3db2809cb83f2a805ad065946daeeb380608 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 26 Apr 2021 14:46:44 +0000 Subject: [PATCH 075/239] Restyled by black --- minos/networks/broker.py | 4 ++-- tests/broker/test_broker.py | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 4b6050a4..4ef37a7d 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -129,8 +129,8 @@ async def broker_queue_dispatcher(config: MinosConfig): async with MinosBrokerDatabase().get_connection(config) as connect: async with connect.cursor() as cur: await cur.execute( - "SELECT * FROM producer_queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;" % - (config.events.queue.retry, config.events.queue.records), + "SELECT * FROM producer_queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;" + % (config.events.queue.retry, config.events.queue.records), ) async for row in cur: sent_to_kafka = False diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index e73638f6..3d859e5c 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -4,9 +4,7 @@ import pytest from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import (MinosCommandBroker, - MinosEventBroker, broker_queue_dispatcher, - send_to_kafka) +from minos.networks.broker import MinosCommandBroker, MinosEventBroker, broker_queue_dispatcher, send_to_kafka from tests.broker.database_testcase import PostgresAsyncTestCase from minos.common import Aggregate @@ -35,12 +33,10 @@ async def test_if_queue_table_exists(self): assert ret == [(1,)] - async def test_send_to_kafka_ok(self): response = await send_to_kafka(topic="TestKafkaSend", message=bytes(), config=self._broker_config()) assert response is True - async def test_events_broker_insertion(self): a = AggregateTest(test_id=1, test=2, id=1, version=1) From 94ea4b08eebbcc5674de09da71f9368d546cfcda Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 26 Apr 2021 14:46:45 +0000 Subject: [PATCH 076/239] Restyled by isort --- minos/networks/broker.py | 2 +- tests/broker/test_broker.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index 4ef37a7d..aab79d21 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -14,7 +14,7 @@ from aiokafka import AIOKafkaProducer from aiomisc.service.periodic import PeriodicService, Service from minos.common import Aggregate -from minos.common.broker import MinosBaseBroker, Event, Command +from minos.common.broker import Command, Event, MinosBaseBroker from minos.common.configuration.config import MinosConfig from minos.common.logs import log diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index 3d859e5c..8905d920 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -2,11 +2,12 @@ import time import pytest +from minos.common import Aggregate from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import MinosCommandBroker, MinosEventBroker, broker_queue_dispatcher, send_to_kafka +from minos.networks.broker import (MinosCommandBroker, MinosEventBroker, + broker_queue_dispatcher, send_to_kafka) from tests.broker.database_testcase import PostgresAsyncTestCase -from minos.common import Aggregate class AggregateTest(Aggregate): From 684930391e076f37921141f22fe98fe6d5feadad Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Mon, 26 Apr 2021 17:50:38 +0200 Subject: [PATCH 077/239] Update event.py --- minos/networks/event.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/minos/networks/event.py b/minos/networks/event.py index 2fa0e8f2..eb461290 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -41,6 +41,9 @@ async def handle_message(self, consumer: t.Any): topic = msg.topic partition = msg.partition event_binary = msg.value + # check if the event binary string is well formatted + + async def start(self) -> t.Any: self.start_event.set() From de7e89fb963157ae41109cbc912b71ed59de8b6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Tue, 27 Apr 2021 11:26:30 +0200 Subject: [PATCH 078/239] * Add `black` + `isort` config. --- setup.cfg | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index f549cb1b..38353861 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,12 +14,28 @@ replace = __version__ = '{new_version}' [bdist_wheel] universal = 1 -[flake8] -exclude = docs - [aliases] # Define setup.py command aliases here test = pytest [tool:pytest] collect_ignore = ['setup.py'] + +[coverage:report] +exclude_lines = + pragma: no cover + raise NotImplementedError + if TYPE_CHECKING: + pass +precision = 2 + +[flake8] +exclude = docs +max-line-length = 120 + +[isort] +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 1 +use_parentheses = True +line_length = 120 From 485a6543c1ef957c65491a424a3aa4deeb467047 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 27 Apr 2021 11:41:54 +0200 Subject: [PATCH 079/239] Update test_config.yaml Minos Config yaml file must be complete #59 --- tests/test_config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_config.yaml b/tests/test_config.yaml index bd3e0326..40bce1ee 100644 --- a/tests/test_config.yaml +++ b/tests/test_config.yaml @@ -12,6 +12,9 @@ rest: events: broker: kafka port: 9092 + database: + path: ./tests/local_db.lmdb + name: database_events_test items: - name: TicketAdded controller: tests.services.CqrsTestService.CqrsService @@ -26,6 +29,7 @@ events: host: postgres port: 5432 records: 10 + retry: 2 commands: broker: kafka port: 9092 @@ -49,3 +53,4 @@ commands: host: postgres port: 5432 records: 10 + retry: 2 From c4d178a6793cfd9cf66349ea1a841a8d60113d2e Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 27 Apr 2021 11:44:18 +0200 Subject: [PATCH 080/239] EventHandler database EventHandler database #55 --- minos/networks/event.py | 24 +++++++- tests/database_testcase.py | 67 ++++++++++++++++++++++ tests/test_event_manager.py | 109 +++++++++++++++++++++--------------- 3 files changed, 155 insertions(+), 45 deletions(-) create mode 100644 tests/database_testcase.py diff --git a/minos/networks/event.py b/minos/networks/event.py index eb461290..7a96e781 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -4,6 +4,7 @@ from aiokafka import AIOKafkaConsumer from aiomisc import Service +import aiopg from minos.common.configuration.config import MinosConfig from minos.common.importlib import import_module from minos.common.logs import log @@ -42,7 +43,7 @@ async def handle_message(self, consumer: t.Any): partition = msg.partition event_binary = msg.value # check if the event binary string is well formatted - + async def start(self) -> t.Any: @@ -71,3 +72,24 @@ def _get_event_handler(self, topic: str) -> t.Callable: return functools.partial(instance_class.action) raise MinosNetworkException(f"topic {topic} have no controller/action configured, " f"please review th configuration file") + + +async def event_handler_table_creation(conf: MinosConfig): + db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ + f"password={conf.events.queue.password} host={conf.events.queue.host}" + async with aiopg.connect(db_dsn) as connect: + async with connect.cursor() as cur: + await cur.execute( + 'CREATE TABLE IF NOT EXISTS "event_queue" ("id" SERIAL NOT NULL PRIMARY KEY, ' + '"topic" VARCHAR(255) NOT NULL, "binary" BYTEA NOT NULL, "creation_date" TIMESTAMP NOT NULL);' + ) + + +class EventHandlerDatabaseInitializer(Service): + async def start(self): + # Send signal to entrypoint for continue running + self.start_event.set() + + await event_handler_table_creation(conf=self.config) + + await self.stop(self) diff --git a/tests/database_testcase.py b/tests/database_testcase.py new file mode 100644 index 00000000..86faa6c5 --- /dev/null +++ b/tests/database_testcase.py @@ -0,0 +1,67 @@ +""" +Copyright (C) 2021 Clariteia SL +This file is part of minos framework. +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +import os +import unittest + +import aiopg +from minos.common.configuration.config import MinosConfig +from minos.networks.event import event_handler_table_creation + + +class EventHandlerPostgresAsyncTestCase(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self._meta_kwargs = { + "host": os.getenv("POSTGRES_HOST", "localhost"), + "port": os.getenv("POSTGRES_PORT", 5432), + "database": os.getenv("POSTGRES_DATABASE", "postgres"), + "user": os.getenv("POSTGRES_USER", "postgres"), + "password": os.getenv("POSTGRES_PASSWORD", ""), + } + + self.kwargs = self._meta_kwargs | { + "database": "broker_db", + "user": "broker", + "password": "br0k3r", + } + + async def asyncSetUp(self): + await event_handler_table_creation(self._broker_config()) + """ + async with aiopg.connect(**self._meta_kwargs) as connection: + async with connection.cursor() as cursor: + + template = "DROP DATABASE IF EXISTS {database};" + await cursor.execute(template.format(**self.kwargs)) + + template = "DROP ROLE IF EXISTS {user};" + await cursor.execute(template.format(**self.kwargs)) + + template = "CREATE ROLE {user} WITH CREATEDB LOGIN ENCRYPTED PASSWORD {password!r};" + await cursor.execute(template.format(**self.kwargs)) + + template = "CREATE DATABASE {database} WITH OWNER = {user};" + await cursor.execute(template.format(**self.kwargs)) + """ + + @staticmethod + def _broker_config(): + return MinosConfig(path="./tests/test_config.yaml") + + async def _database(self): + conf = self._broker_config() + db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ + f"password={conf.events.queue.password} host={conf.events.queue.host}" + return await aiopg.connect(db_dsn) + + async def asyncTearDown(self): + pass + """ + database = await self._database() + async with database as connection: + async with connection.cursor() as cursor: + template = "DROP TABLE IF EXISTS queue" + await cursor.execute(template) + """ diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 4df37c27..43ea263d 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -1,44 +1,65 @@ -# import asyncio -# import logging -# -# import pytest -# import string -# from aiokafka import AIOKafkaProducer -# import random -# -# from aiomisc.log import basic_config -# from minos.common.configuration.config import MinosConfig -# -# from minos.networks.event import MinosEventServer -# -# -# @pytest.fixture() -# def config(): -# return MinosConfig(path='./tests/test_config.yaml') -# -# -# @pytest.fixture() -# def services(config): -# return [MinosEventServer(conf=config)] -# -# -# async def test_producer_kafka(loop): -# basic_config( -# level=logging.INFO, -# buffered=True, -# log_format='color', -# flush_interval=2 -# ) -# -# producer = AIOKafkaProducer(loop=loop, bootstrap_servers='localhost:9092') -# # Get cluster layout and topic/partition allocation -# await producer.start() -# # Produce messages -# string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=20)) -# await producer.send_and_wait("TicketAdded", string_to_send.encode()) -# await asyncio.sleep(1) -# -# other_string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=40)) -# await producer.send_and_wait("TicketDeleted", other_string_to_send.encode()) -# await asyncio.sleep(1) -# await producer.stop() +import asyncio +import logging + +import pytest +import string +from aiokafka import AIOKafkaProducer +import random + +from aiomisc.log import basic_config +from minos.common.configuration.config import MinosConfig + +from minos.networks.event import MinosEventServer, EventHandlerDatabaseInitializer +from tests.database_testcase import EventHandlerPostgresAsyncTestCase + +@pytest.fixture() +def config(): + return MinosConfig(path='./tests/test_config.yaml') + + +@pytest.fixture() +def services(config): + return [EventHandlerDatabaseInitializer(config=config)] # , MinosEventServer(conf=config) + + +class TestPostgreSqlMinosEventHandler(EventHandlerPostgresAsyncTestCase): + async def test_database_connection(self): + database = await self._database() + async with database as connect: + assert database.closed == 0 + + async def test_if_queue_table_exists(self): + database = await self._database() + async with database as connect: + async with connect.cursor() as cur: + + await cur.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'event_queue';" + ) + ret = [] + async for row in cur: + ret.append(row) + + assert ret == [(1,)] + + +async def test_producer_kafka(loop): + basic_config( + level=logging.INFO, + buffered=True, + log_format='color', + flush_interval=2 + ) + + producer = AIOKafkaProducer(loop=loop, bootstrap_servers='localhost:9092') + # Get cluster layout and topic/partition allocation + await producer.start() + # Produce messages + string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=20)) + await producer.send_and_wait("TicketAdded", string_to_send.encode()) + await asyncio.sleep(1) + + other_string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=40)) + await producer.send_and_wait("TicketDeleted", other_string_to_send.encode()) + await asyncio.sleep(1) + await producer.stop() From 7f7fbd1a762b33fa5432d075e7c20b9442a44b6b Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 27 Apr 2021 13:30:49 +0200 Subject: [PATCH 081/239] Add partition field to EventHandler database EventHandler database #55 --- minos/networks/event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minos/networks/event.py b/minos/networks/event.py index 7a96e781..84fd96ba 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -81,7 +81,7 @@ async def event_handler_table_creation(conf: MinosConfig): async with connect.cursor() as cur: await cur.execute( 'CREATE TABLE IF NOT EXISTS "event_queue" ("id" SERIAL NOT NULL PRIMARY KEY, ' - '"topic" VARCHAR(255) NOT NULL, "binary" BYTEA NOT NULL, "creation_date" TIMESTAMP NOT NULL);' + '"topic" VARCHAR(255) NOT NULL, "partition" INTEGER , "binary" BYTEA NOT NULL, "creation_date" TIMESTAMP NOT NULL);' ) From 463623e4a072af557364bfc3ed2ce5c9ba7945c3 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 27 Apr 2021 17:18:40 +0200 Subject: [PATCH 082/239] The EventHandler must get the list of Topics from the config file and register The EventHandler must get the list of Topics from the config file and register, trough the Kafka Client, to that topics. #60 --- minos/networks/event.py | 12 +++---- tests/test_event_manager.py | 62 ++++++++++++++++++++++++------------- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/minos/networks/event.py b/minos/networks/event.py index 84fd96ba..54e11065 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -19,15 +19,17 @@ class MinosEventServer(Service): Consumer for the Broker ( at the moment only Kafka is supported ) """ - __slots__ = "_tasks", "_handlers", "_topics", "_broker_group_name" + __slots__ = "_tasks", "_db_dsn", "_handlers", "_topics", "_kafka_conn_data", "_broker_group_name" def __init__(self, *, conf: MinosConfig, **kwargs: t.Any): self._tasks = set() # type: t.Set[asyncio.Task] self._db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ f"password={conf.events.queue.password} host={conf.events.queue.host}" - self._handler = {item['name']: {'controller': item['controller'], 'action': item['action']} + self._handler = {item.name: {'controller': item.controller, 'action': item.action} for item in conf.events.items} self._topics = list(self._handler.keys()) + self._kafka_conn_data = f"{conf.events.broker.host}:{conf.events.broker.port}" + self._broker_group_name = f"event_{conf.service.name}" super().__init__(**kwargs) def create_task(self, coro: t.Awaitable[t.Any]): @@ -44,16 +46,14 @@ async def handle_message(self, consumer: t.Any): event_binary = msg.value # check if the event binary string is well formatted - - async def start(self) -> t.Any: self.start_event.set() log.debug("Event Consumer Manager: Started") # start the Service Event Consumer for Kafka consumer = AIOKafkaConsumer(loop=self.loop, - group_id=self._broker_group, + group_id=self._broker_group_name, auto_offset_reset="latest", - bootstrap_servers=f"{self._broker_host}:{self._broker_port}", + bootstrap_servers=self._kafka_conn_data, ) await consumer.start() diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 43ea263d..0779240c 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -3,7 +3,7 @@ import pytest import string -from aiokafka import AIOKafkaProducer +from aiokafka import AIOKafkaProducer, AIOKafkaConsumer import random from aiomisc.log import basic_config @@ -19,7 +19,7 @@ def config(): @pytest.fixture() def services(config): - return [EventHandlerDatabaseInitializer(config=config)] # , MinosEventServer(conf=config) + return [EventHandlerDatabaseInitializer(config=config), MinosEventServer(conf=config)] class TestPostgreSqlMinosEventHandler(EventHandlerPostgresAsyncTestCase): @@ -44,22 +44,42 @@ async def test_if_queue_table_exists(self): async def test_producer_kafka(loop): - basic_config( - level=logging.INFO, - buffered=True, - log_format='color', - flush_interval=2 - ) - - producer = AIOKafkaProducer(loop=loop, bootstrap_servers='localhost:9092') - # Get cluster layout and topic/partition allocation - await producer.start() - # Produce messages - string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=20)) - await producer.send_and_wait("TicketAdded", string_to_send.encode()) - await asyncio.sleep(1) - - other_string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=40)) - await producer.send_and_wait("TicketDeleted", other_string_to_send.encode()) - await asyncio.sleep(1) - await producer.stop() + basic_config( + level=logging.DEBUG, + buffered=True, + log_format='color', + flush_interval=2 + ) + + producer = AIOKafkaProducer(loop=loop, bootstrap_servers='localhost:9092') + # Get cluster layout and topic/partition allocation + await producer.start() + # Produce messages + string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=20)) + await producer.send_and_wait("TicketAdded", string_to_send.encode()) + await asyncio.sleep(1) + + other_string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=40)) + await producer.send_and_wait("TicketDeleted", other_string_to_send.encode()) + await asyncio.sleep(1) + await producer.stop() + + +async def test_event_handle_message(config, loop): + handler = {item.name: {'controller': item.controller, 'action': item.action} + for item in config.events.items} + topics = list(handler.keys()) + kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" + broker_group_name = f"event_{config.service.name}" + + m = MinosEventServer(conf=config) + consumer = AIOKafkaConsumer(loop=loop, + group_id=broker_group_name, + auto_offset_reset="latest", + bootstrap_servers=kafka_conn_data, + ) + + await consumer.start() + consumer.subscribe(topics) + + await m.handle_message(consumer) From 4a2b6f1b8fcb5f98789dd0757ecd5e07d45b839d Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 28 Apr 2021 13:26:14 +0200 Subject: [PATCH 083/239] EventHandler must store the event on database EventHandler must store the event on database #61 --- minos/networks/event.py | 86 +++++++++++++++++++++++++++++++------ tests/test_event_manager.py | 37 +++++++++++++--- 2 files changed, 102 insertions(+), 21 deletions(-) diff --git a/minos/networks/event.py b/minos/networks/event.py index 54e11065..f904275f 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -1,6 +1,14 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + import asyncio import functools import typing as t +import datetime from aiokafka import AIOKafkaConsumer from aiomisc import Service @@ -10,7 +18,7 @@ from minos.common.logs import log from minos.networks.exceptions import MinosNetworkException - +from minos.common.broker import Event class MinosEventServer(Service): """ @@ -37,14 +45,62 @@ def create_task(self, coro: t.Awaitable[t.Any]): self._tasks.add(task) task.add_done_callback(self._tasks.remove) + async def event_queue_add(self, topic: str, partition: int, binary: bytes): + """Insert row to event_queue table. + + Retrieves number of affected rows and row ID. + + Args: + topic: Kafka topic. Example: "TicketAdded" + partition: Kafka partition number. + binary: Event Model in bytes. + + Returns: + Affected rows and queue ID. + + Example: 1, 12 + + Raises: + Exception: An error occurred inserting record. + """ + + async with aiopg.create_pool(self._db_dsn) as pool: + async with pool.acquire() as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO event_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + (topic, partition, binary, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + return affected_rows, queue_id[0] + async def handle_message(self, consumer: t.Any): - while True: - async for msg in consumer: - # the handler receive a message and store in the queue database - topic = msg.topic - partition = msg.partition - event_binary = msg.value - # check if the event binary string is well formatted + """Handle Kafka messages. + + For each message evaluate if the binary is an Event instance. + Add Event instance to the event_queue table. + + Args: + consumer: Kafka Consumer instance (at the moment only Kafka consumer is supported). + + Raises: + Exception: An error occurred inserting record. + """ + + async for msg in consumer: + # the handler receive a message and store in the queue database + topic = msg.topic + partition = msg.partition + event_binary = msg.value + # check if the event binary string is well formatted + try: + event_instance = Event.from_avro_bytes(event_binary) + await self.event_queue_add(msg.topic, msg.partition, event_binary) + except: + pass async def start(self) -> t.Any: self.start_event.set() @@ -77,12 +133,13 @@ def _get_event_handler(self, topic: str) -> t.Callable: async def event_handler_table_creation(conf: MinosConfig): db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ f"password={conf.events.queue.password} host={conf.events.queue.host}" - async with aiopg.connect(db_dsn) as connect: - async with connect.cursor() as cur: - await cur.execute( - 'CREATE TABLE IF NOT EXISTS "event_queue" ("id" SERIAL NOT NULL PRIMARY KEY, ' - '"topic" VARCHAR(255) NOT NULL, "partition" INTEGER , "binary" BYTEA NOT NULL, "creation_date" TIMESTAMP NOT NULL);' - ) + async with aiopg.create_pool(db_dsn) as pool: + async with pool.acquire() as connect: + async with connect.cursor() as cur: + await cur.execute( + 'CREATE TABLE IF NOT EXISTS "event_queue" ("id" SERIAL NOT NULL PRIMARY KEY, ' + '"topic" VARCHAR(255) NOT NULL, "partition_id" INTEGER , "binary_data" BYTEA NOT NULL, "creation_date" TIMESTAMP NOT NULL);' + ) class EventHandlerDatabaseInitializer(Service): @@ -93,3 +150,4 @@ async def start(self): await event_handler_table_creation(conf=self.config) await self.stop(self) + diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 0779240c..62cdc62f 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -8,7 +8,8 @@ from aiomisc.log import basic_config from minos.common.configuration.config import MinosConfig - +from minos.common import Aggregate +from minos.common.broker import Event from minos.networks.event import MinosEventServer, EventHandlerDatabaseInitializer from tests.database_testcase import EventHandlerPostgresAsyncTestCase @@ -16,11 +17,13 @@ def config(): return MinosConfig(path='./tests/test_config.yaml') - +""" @pytest.fixture() def services(config): return [EventHandlerDatabaseInitializer(config=config), MinosEventServer(conf=config)] - +""" +class AggregateTest(Aggregate): + test: int class TestPostgreSqlMinosEventHandler(EventHandlerPostgresAsyncTestCase): async def test_database_connection(self): @@ -42,6 +45,18 @@ async def test_if_queue_table_exists(self): assert ret == [(1,)] + async def test_event_queue_add(self): + model = AggregateTest(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) + bin_data = event_instance.avro_bytes + Event.from_avro_bytes(bin_data) + + m = MinosEventServer(conf=self._broker_config()) + affected_rows, id = await m.event_queue_add(topic=event_instance.topic, partition=0, binary=bin_data) + + assert affected_rows == 1 + assert id > 0 + async def test_producer_kafka(loop): basic_config( @@ -55,12 +70,18 @@ async def test_producer_kafka(loop): # Get cluster layout and topic/partition allocation await producer.start() # Produce messages - string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=20)) - await producer.send_and_wait("TicketAdded", string_to_send.encode()) + + model = AggregateTest(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TicketAdded", model=model.classname, items=[]) + bin_data = event_instance.avro_bytes + + await producer.send_and_wait(event_instance.topic, bin_data) await asyncio.sleep(1) - other_string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=40)) - await producer.send_and_wait("TicketDeleted", other_string_to_send.encode()) + model2 = AggregateTest(test_id=2, test=2, id=1, version=1) + event_instance_2 = Event(topic="TicketDeleted", model=model2.classname, items=[]) + bin_data2 = event_instance_2.avro_bytes + await producer.send_and_wait(event_instance_2.topic, bin_data2) await asyncio.sleep(1) await producer.stop() @@ -83,3 +104,5 @@ async def test_event_handle_message(config, loop): consumer.subscribe(topics) await m.handle_message(consumer) + + From 26072b7a668c723258652787e9b5d8c085ce8814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 28 Apr 2021 16:24:04 +0200 Subject: [PATCH 084/239] REFS #66 * Update `minos.common` module to the latest version --- requirements_dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 8f61b0b8..213a82e1 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -32,7 +32,7 @@ keyring==23.0.1 lmdb==1.1.1 MarkupSafe==1.1.1 mccabe==0.6.1 -minos-microservice-common==0.0.3 +minos-microservice-common==0.0.4 more-itertools==8.7.0 mypy-extensions==0.4.3 orjson==3.5.1 From 0fe99a0afb9a7d1e54aab76d89a5795d7364045a Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 28 Apr 2021 14:25:17 +0000 Subject: [PATCH 085/239] Restyled by black --- setup.py | 49 ++++++++++++++++++++----------------- tests/broker/conftest.py | 4 +-- tests/broker/test_broker.py | 3 +-- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/setup.py b/setup.py index 3c166562..281a8846 100644 --- a/setup.py +++ b/setup.py @@ -4,43 +4,48 @@ from setuptools import setup, find_namespace_packages -with open('README.md') as readme_file: +with open("README.md") as readme_file: readme = readme_file.read() -with open('HISTORY.md') as history_file: +with open("HISTORY.md") as history_file: history = history_file.read() -requirements = ['minos-microservice-common', 'aiokafka', 'aiomisc', 'aiopg'] +requirements = ["minos-microservice-common", "aiokafka", "aiomisc", "aiopg"] -setup_requirements = ['pytest-runner',] +setup_requirements = [ + "pytest-runner", +] -test_requirements = ['pytest>=3', 'pytest-asyncio', ] +test_requirements = [ + "pytest>=3", + "pytest-asyncio", +] setup( author="Andrea Mucci", - author_email='andrea@clariteia.com', - python_requires='>=3.5', + author_email="andrea@clariteia.com", + python_requires=">=3.5", classifiers=[ - 'Development Status :: 2 - Pre-Alpha', - 'Intended Audience :: Developers', - 'Natural Language :: English', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], description="Python Package with the common network classes and utlities used in Minos Microservice", install_requires=requirements, - long_description_content_type='text/markdown', - long_description=readme + '\n\n' + history, + long_description_content_type="text/markdown", + long_description=readme + "\n\n" + history, include_package_data=True, - keywords='minos_microservice_networks', - name='minos_microservice_networks', - packages=find_namespace_packages(include=['minos.*']), + keywords="minos_microservice_networks", + name="minos_microservice_networks", + packages=find_namespace_packages(include=["minos.*"]), setup_requires=setup_requirements, - test_suite='tests', + test_suite="tests", tests_require=test_requirements, - version='0.0.1-alpha', + version="0.0.1-alpha", zip_safe=False, ) diff --git a/tests/broker/conftest.py b/tests/broker/conftest.py index c6bc320e..15068def 100644 --- a/tests/broker/conftest.py +++ b/tests/broker/conftest.py @@ -3,9 +3,7 @@ from aiomisc.service.periodic import Service from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import (BrokerDatabaseInitializer, - EventBrokerQueueDispatcher, - MinosBrokerDatabase) +from minos.networks.broker import BrokerDatabaseInitializer, EventBrokerQueueDispatcher, MinosBrokerDatabase @pytest.fixture diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index 8905d920..7f693103 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -5,8 +5,7 @@ from minos.common import Aggregate from minos.common.configuration.config import MinosConfig from minos.common.logs import log -from minos.networks.broker import (MinosCommandBroker, MinosEventBroker, - broker_queue_dispatcher, send_to_kafka) +from minos.networks.broker import MinosCommandBroker, MinosEventBroker, broker_queue_dispatcher, send_to_kafka from tests.broker.database_testcase import PostgresAsyncTestCase From b11cd1f18fbeb4e8a47f1621003214b4ca2d9ddb Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 28 Apr 2021 14:25:31 +0000 Subject: [PATCH 086/239] Restyled by isort --- minos/networks/broker.py | 31 ++++++++++++++++++++++++------- setup.py | 5 ++++- tests/broker/conftest.py | 22 +++++++++++++++++----- tests/broker/database_testcase.py | 9 +++++++-- tests/broker/test_broker.py | 23 ++++++++++++++++++----- 5 files changed, 70 insertions(+), 20 deletions(-) diff --git a/minos/networks/broker.py b/minos/networks/broker.py index aab79d21..6c66be02 100644 --- a/minos/networks/broker.py +++ b/minos/networks/broker.py @@ -8,15 +8,32 @@ import asyncio import datetime import typing as t -from enum import Enum +from enum import ( + Enum, +) import aiopg -from aiokafka import AIOKafkaProducer -from aiomisc.service.periodic import PeriodicService, Service -from minos.common import Aggregate -from minos.common.broker import Command, Event, MinosBaseBroker -from minos.common.configuration.config import MinosConfig -from minos.common.logs import log +from aiokafka import ( + AIOKafkaProducer, +) +from aiomisc.service.periodic import ( + PeriodicService, + Service, +) +from minos.common import ( + Aggregate, +) +from minos.common.broker import ( + Command, + Event, + MinosBaseBroker, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.logs import ( + log, +) class MinosBrokerDatabase: diff --git a/setup.py b/setup.py index 281a8846..86970804 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,10 @@ """The setup script.""" -from setuptools import setup, find_namespace_packages +from setuptools import ( + find_namespace_packages, + setup, +) with open("README.md") as readme_file: readme = readme_file.read() diff --git a/tests/broker/conftest.py b/tests/broker/conftest.py index 15068def..9abbca53 100644 --- a/tests/broker/conftest.py +++ b/tests/broker/conftest.py @@ -1,9 +1,21 @@ import pytest -from aiokafka import AIOKafkaConsumer -from aiomisc.service.periodic import Service -from minos.common.configuration.config import MinosConfig -from minos.common.logs import log -from minos.networks.broker import BrokerDatabaseInitializer, EventBrokerQueueDispatcher, MinosBrokerDatabase +from aiokafka import ( + AIOKafkaConsumer, +) +from aiomisc.service.periodic import ( + Service, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.logs import ( + log, +) +from minos.networks.broker import ( + BrokerDatabaseInitializer, + EventBrokerQueueDispatcher, + MinosBrokerDatabase, +) @pytest.fixture diff --git a/tests/broker/database_testcase.py b/tests/broker/database_testcase.py index 559556b4..493379af 100644 --- a/tests/broker/database_testcase.py +++ b/tests/broker/database_testcase.py @@ -7,8 +7,13 @@ import unittest import aiopg -from minos.common.configuration.config import MinosConfig -from minos.networks.broker import MinosBrokerDatabase, broker_table_creation +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.networks.broker import ( + MinosBrokerDatabase, + broker_table_creation, +) class PostgresAsyncTestCase(unittest.IsolatedAsyncioTestCase): diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py index 7f693103..b9ec65b3 100644 --- a/tests/broker/test_broker.py +++ b/tests/broker/test_broker.py @@ -2,11 +2,24 @@ import time import pytest -from minos.common import Aggregate -from minos.common.configuration.config import MinosConfig -from minos.common.logs import log -from minos.networks.broker import MinosCommandBroker, MinosEventBroker, broker_queue_dispatcher, send_to_kafka -from tests.broker.database_testcase import PostgresAsyncTestCase +from minos.common import ( + Aggregate, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.logs import ( + log, +) +from minos.networks.broker import ( + MinosCommandBroker, + MinosEventBroker, + broker_queue_dispatcher, + send_to_kafka, +) +from tests.broker.database_testcase import ( + PostgresAsyncTestCase, +) class AggregateTest(Aggregate): From 434a53317c0ee656747daefcf1a07b84df94dabe Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 28 Apr 2021 16:29:41 +0200 Subject: [PATCH 087/239] EventHandler - adjust tests (pass model to Event items param) EventHandler - adjust tests (pass model to Event items param) #62 --- tests/test_event_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 62cdc62f..5af67d5a 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -60,7 +60,7 @@ async def test_event_queue_add(self): async def test_producer_kafka(loop): basic_config( - level=logging.DEBUG, + level=logging.INFO, buffered=True, log_format='color', flush_interval=2 @@ -72,14 +72,14 @@ async def test_producer_kafka(loop): # Produce messages model = AggregateTest(test_id=1, test=2, id=1, version=1) - event_instance = Event(topic="TicketAdded", model=model.classname, items=[]) + event_instance = Event(topic="TicketAdded", model=model.classname, items=[model]) bin_data = event_instance.avro_bytes await producer.send_and_wait(event_instance.topic, bin_data) await asyncio.sleep(1) model2 = AggregateTest(test_id=2, test=2, id=1, version=1) - event_instance_2 = Event(topic="TicketDeleted", model=model2.classname, items=[]) + event_instance_2 = Event(topic="TicketDeleted", model=model2.classname, items=[model2]) bin_data2 = event_instance_2.avro_bytes await producer.send_and_wait(event_instance_2.topic, bin_data2) await asyncio.sleep(1) From a301544f2191d8f1ef6b3a88326033e1bc39d9ef Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 28 Apr 2021 16:34:06 +0200 Subject: [PATCH 088/239] Remove events>database from .yaml Minos Config yaml file must be complete #59 --- tests/test_config.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_config.yaml b/tests/test_config.yaml index 40bce1ee..9107147b 100644 --- a/tests/test_config.yaml +++ b/tests/test_config.yaml @@ -12,9 +12,6 @@ rest: events: broker: kafka port: 9092 - database: - path: ./tests/local_db.lmdb - name: database_events_test items: - name: TicketAdded controller: tests.services.CqrsTestService.CqrsService From e7d348efa5fce9f21b5a4458e7c1f8004a65d4c4 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 28 Apr 2021 16:35:54 +0200 Subject: [PATCH 089/239] Update requirements_dev.txt --- requirements_dev.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index c2cf2c7e..213a82e1 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -19,6 +19,7 @@ colorlog==5.0.1 coverage==5.5 distlib==0.3.1 docutils==0.16 +fastavro==1.4.0 filelock==3.0.12 flake8==3.9.0 idna==2.10 @@ -31,7 +32,7 @@ keyring==23.0.1 lmdb==1.1.1 MarkupSafe==1.1.1 mccabe==0.6.1 -minos-microservice-common==0.0.2 +minos-microservice-common==0.0.4 more-itertools==8.7.0 mypy-extensions==0.4.3 orjson==3.5.1 From 11e09570a189f3850092cb90cf9ed5363f7ac07f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 28 Apr 2021 18:59:14 +0200 Subject: [PATCH 090/239] REFS #66 * Integrate with latest version of `minos.common`. --- .github/workflows/python-tests.yml | 69 ++++---- minos/networks/__init__.py | 9 +- minos/networks/broker.py | 154 ---------------- minos/networks/broker/__init__.py | 17 ++ minos/networks/broker/abc.py | 63 +++++++ minos/networks/broker/command.py | 72 ++++++++ minos/networks/broker/dispatcher.py | 96 ++++++++++ minos/networks/broker/event.py | 68 +++++++ tests/broker/__init__.py | 0 tests/broker/conftest.py | 51 ------ tests/broker/database_testcase.py | 64 ------- tests/broker/test_broker.py | 167 ------------------ tests/services/CqrsTestService.py | 3 - tests/services/__init__.py | 0 tests/test_config.yaml | 26 +-- tests/test_networks/__init__.py | 7 + tests/test_networks/test_broker/__init__.py | 7 + .../test_networks/test_broker/test_command.py | 102 +++++++++++ .../test_broker/test_dispatcher.py | 30 ++++ .../test_networks/test_broker/test_events.py | 115 ++++++++++++ tests/utils.py | 12 ++ tests/wrong_test_config.yaml | 22 ++- 22 files changed, 661 insertions(+), 493 deletions(-) delete mode 100644 minos/networks/broker.py create mode 100644 minos/networks/broker/__init__.py create mode 100644 minos/networks/broker/abc.py create mode 100644 minos/networks/broker/command.py create mode 100644 minos/networks/broker/dispatcher.py create mode 100644 minos/networks/broker/event.py delete mode 100644 tests/broker/__init__.py delete mode 100644 tests/broker/conftest.py delete mode 100644 tests/broker/database_testcase.py delete mode 100644 tests/broker/test_broker.py delete mode 100644 tests/services/CqrsTestService.py delete mode 100644 tests/services/__init__.py create mode 100644 tests/test_networks/__init__.py create mode 100644 tests/test_networks/test_broker/__init__.py create mode 100644 tests/test_networks/test_broker/test_command.py create mode 100644 tests/test_networks/test_broker/test_dispatcher.py create mode 100644 tests/test_networks/test_broker/test_events.py create mode 100644 tests/utils.py diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 7225f642..83b1d563 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -6,31 +6,22 @@ jobs: runs-on: ubuntu-latest container: python:3.9-buster - # Service containers to run with `container-job` + services: - # Label used to access the service container postgres: - # Docker Hub image image: postgres - # Provide the password for postgres env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_HOST: localhost - POSTGRES_HOST_AUTH_METHOD: trust - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 + POSTGRES_USER: minos + POSTGRES_PASSWORD: min0s ports: - # Maps tcp port 5432 on service container to the host - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + zookeeper: image: wurstmeister/zookeeper:latest ports: - 2181:2181 + kafka: image: wurstmeister/kafka:latest ports: @@ -38,35 +29,43 @@ jobs: env: KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ADVERTISED_HOST_NAME: kafka + env: + MINOS_EVENTS_QUEUE_HOST: postgres + MINOS_EVENTS_BROKER: kafka + MINOS_COMMANDS_QUEUE_HOST: postgres + MINOS_COMMANDS_BROKER: kafka + MINOS_REPOSITORY_HOST: postgres steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | + - name: Check out repository code + uses: actions/checkout@v2 + + - name: Install dependencies + run: | python -m pip install --upgrade pip if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi python setup.py install - - name: Install Postgresql client - run: | + + - name: Install Postgresql client + run: | apt-get update apt-get install --yes postgresql-client - - name: Create database - run: | - PGPASSWORD=postgres psql -U postgres -h postgres -p 5432 -tc "CREATE ROLE broker with createdb login password 'br0k3r';" - PGPASSWORD=postgres psql -U postgres -h postgres -p 5432 -tc "CREATE DATABASE broker_db OWNER broker;" - - name: Test with pytest - run: | + + - name: Create database + run: | + PGPASSWORD=min0s psql -U minos -h postgres -tc "CREATE DATABASE order_db OWNER minos;" + + - name: Test with pytest + run: | make test - - name: Generate coverage report + + - name: Generate coverage report run: | make coverage - - name: Codecov + + - name: Codecov uses: codecov/codecov-action@v1.3.1 with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + fail_ci_if_error: true diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 57ffe7ee..c9f93575 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -1 +1,8 @@ -__version__ = '0.0.1.1-alpha' +__version__ = "0.0.1.1-alpha" + +from .broker import ( + EventBrokerQueueDispatcher, + MinosCommandBroker, + MinosEventBroker, + MinosQueueDispatcher, +) diff --git a/minos/networks/broker.py b/minos/networks/broker.py deleted file mode 100644 index aab79d21..00000000 --- a/minos/networks/broker.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright (C) 2020 Clariteia SL -# -# This file is part of minos framework. -# -# Minos framework can not be copied and/or distributed without the express -# permission of Clariteia SL. -import abc -import asyncio -import datetime -import typing as t -from enum import Enum - -import aiopg -from aiokafka import AIOKafkaProducer -from aiomisc.service.periodic import PeriodicService, Service -from minos.common import Aggregate -from minos.common.broker import Command, Event, MinosBaseBroker -from minos.common.configuration.config import MinosConfig -from minos.common.logs import log - - -class MinosBrokerDatabase: - def get_connection(self, conf): - conn = aiopg.connect( - database=conf.events.queue.database, - user=conf.events.queue.user, - password=conf.events.queue.password, - host=conf.events.queue.host, - port=conf.events.queue.port, - ) - return conn - - -class MinosEventBroker(MinosBaseBroker): - ACTION = "event" - - def __init__(self, topic: str, config: MinosConfig): - self.config = config - self.topic = topic - self._database() - - def _database(self): - pass - - async def send(self, model: Aggregate): - event_instance = Event(topic=self.topic, model=model.classname, items=[model]) - bin_data = event_instance.avro_bytes - - async with MinosBrokerDatabase().get_connection(self.config) as connect: - async with connect.cursor() as cur: - await cur.execute( - "INSERT INTO producer_queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", - (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), - ) - - queue_id = await cur.fetchone() - affected_rows = cur.rowcount - - return affected_rows, queue_id[0] - - -class MinosCommandBroker(MinosBaseBroker): - ACTION = "command" - - def __init__(self, topic: str, config: MinosConfig): - self.config = config - self.topic = topic - self._database() - - def _database(self): - pass - - async def send(self, model: Aggregate, reply_on: str): - event_instance = Command(topic=self.topic, model=model.classname, items=[model], reply_on=reply_on) - bin_data = event_instance.avro_bytes - - async with MinosBrokerDatabase().get_connection(self.config) as connect: - async with connect.cursor() as cur: - await cur.execute( - "INSERT INTO producer_queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", - (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), - ) - - queue_id = await cur.fetchone() - affected_rows = cur.rowcount - - return affected_rows, queue_id[0] - - -async def broker_table_creation(config: MinosConfig): - async with MinosBrokerDatabase().get_connection(config) as connect: - async with connect.cursor() as cur: - await cur.execute( - 'CREATE TABLE IF NOT EXISTS "producer_queue" ("id" BIGSERIAL NOT NULL PRIMARY KEY, ' - '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' - '"action" VARCHAR(255) NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' - ) - - -class BrokerDatabaseInitializer(Service): - async def start(self): - # Send signal to entrypoint for continue running - self.start_event.set() - - await broker_table_creation(self.config) - - await self.stop(self) - - -async def send_to_kafka(topic: str, message: bytes, config: MinosConfig): - flag = False - producer = AIOKafkaProducer( - bootstrap_servers="{host}:{port}".format(host=config.events.broker.host, port=config.events.broker.port) - ) - # Get cluster layout and initial topic/partition leadership information - await producer.start() - try: - # Produce message - await producer.send_and_wait(topic, message) - flag = True - finally: - # Wait for all pending messages to be delivered or expire. - await producer.stop() - - return flag - - -async def broker_queue_dispatcher(config: MinosConfig): - async with MinosBrokerDatabase().get_connection(config) as connect: - async with connect.cursor() as cur: - await cur.execute( - "SELECT * FROM producer_queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;" - % (config.events.queue.retry, config.events.queue.records), - ) - async for row in cur: - sent_to_kafka = False - try: - sent_to_kafka = await send_to_kafka(topic=row[1], message=row[2], config=config) - if sent_to_kafka: - # Delete from database If the event was sent successfully to Kafka. - async with connect.cursor() as cur2: - await cur2.execute("DELETE FROM producer_queue WHERE id=%d;" % row[0]) - except: - sent_to_kafka = False - finally: - if not sent_to_kafka: - # Update queue retry column. Increase by 1. - async with connect.cursor() as cur3: - await cur3.execute("UPDATE producer_queue SET retry = retry + 1 WHERE id=%d;" % row[0]) - - -class EventBrokerQueueDispatcher(PeriodicService): - async def callback(self): - await broker_queue_dispatcher(self.config) diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py new file mode 100644 index 00000000..050afc17 --- /dev/null +++ b/minos/networks/broker/__init__.py @@ -0,0 +1,17 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from .event import ( + MinosEventBroker, +) +from .command import ( + MinosCommandBroker, +) +from .dispatcher import ( + MinosQueueDispatcher, + EventBrokerQueueDispatcher, +) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py new file mode 100644 index 00000000..c89d76d3 --- /dev/null +++ b/minos/networks/broker/abc.py @@ -0,0 +1,63 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from typing import ( + NoReturn, +) + +import aiopg +from minos.common import ( + MinosBaseBroker, + MinosSetup, +) + + +class MinosBrokerSetup(MinosSetup): + def __init__( + self, + *args, + host: str = None, + port: int = None, + database: str = None, + user: str = None, + password: str = None, + **kwargs + ): + super().__init__(*args, **kwargs) + + self.host = host + self.port = port + self.database = database + self.user = user + self.password = password + + async def _setup(self) -> NoReturn: + await self.broker_table_creation() + + async def broker_table_creation(self): + async with self._connection() as connect: + async with connect.cursor() as cur: + await cur.execute( + 'CREATE TABLE IF NOT EXISTS "producer_queue" ("id" BIGSERIAL NOT NULL PRIMARY KEY, ' + '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' + '"action" VARCHAR(255) NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' + ) + + def _connection(self): + return aiopg.connect( + host=self.host, + port=self.port, + dbname=self.database, + user=self.user, + password=self.password, + ) + + +class MinosBroker(MinosBaseBroker, MinosBrokerSetup): + def __init__(self, topic: str, *args, **kwargs): + MinosBaseBroker.__init__(self, topic) + MinosBrokerSetup.__init__(self, *args, **kwargs) diff --git a/minos/networks/broker/command.py b/minos/networks/broker/command.py new file mode 100644 index 00000000..2ee56ce4 --- /dev/null +++ b/minos/networks/broker/command.py @@ -0,0 +1,72 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from __future__ import ( + annotations, +) +import datetime +from typing import ( + NoReturn, Optional, +) + +from minos.common import ( + Aggregate, + Command, + MinosConfig, +) + +from .abc import ( + MinosBroker, +) + + +class MinosCommandBroker(MinosBroker): + """TODO""" + + ACTION = "command" + + def __init__(self, *args, reply_on: str, **kwargs): + super().__init__(*args, **kwargs) + self.reply_on = reply_on + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosCommandBroker]: + """Build a new repository from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + return None + # noinspection PyProtectedMember + return cls(*args, **config.commands.queue._asdict(), **kwargs) + + async def send(self, items: list[Aggregate]) -> NoReturn: + event_instance = Command(self.topic, items, self.reply_on) + bin_data = event_instance.avro_bytes + + async with self._connection() as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO producer_queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", + ( + event_instance.topic, + bin_data, + 0, + self.ACTION, + datetime.datetime.now(), + datetime.datetime.now(), + ), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + return affected_rows, queue_id[0] diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py new file mode 100644 index 00000000..041d6d47 --- /dev/null +++ b/minos/networks/broker/dispatcher.py @@ -0,0 +1,96 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from __future__ import ( + annotations, +) +from typing import Optional, Any + +from aiokafka import ( + AIOKafkaProducer, +) +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .abc import MinosBrokerSetup + + +class EventBrokerQueueDispatcher(PeriodicService): + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosQueueDispatcher.from_config(config=config) + + async def callback(self): + await self.dispatcher.dispatch() + + +class MinosQueueDispatcher(MinosBrokerSetup): + def __init__(self, *args, queue=None, broker, **kwargs): + super().__init__(*args, **queue._asdict(), **kwargs) + self.retry = queue.retry + self.records = queue.records + self.broker = broker + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosQueueDispatcher]: + """Build a new repository from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + return None + # noinspection PyProtectedMember + return cls(*args, **config.events._asdict(), **kwargs) + + async def dispatch(self): + async with self._connection() as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT * FROM producer_queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;" + % (self.retry, self.records), + ) + async for row in cur: + sent_to_kafka = False + try: + sent_to_kafka = await self.send_to_kafka(topic=row[1], message=row[2]) + if sent_to_kafka: + # Delete from database If the event was sent successfully to Kafka. + async with connect.cursor() as cur2: + await cur2.execute("DELETE FROM producer_queue WHERE id=%d;" % row[0]) + except: + sent_to_kafka = False + finally: + if not sent_to_kafka: + # Update queue retry column. Increase by 1. + async with connect.cursor() as cur3: + await cur3.execute("UPDATE producer_queue SET retry = retry + 1 WHERE id=%d;" % row[0]) + + async def send_to_kafka(self, topic: str, message: bytes): + flag = False + producer = AIOKafkaProducer( + bootstrap_servers="{host}:{port}".format(host=self.broker.host, port=self.broker.port) + ) + # Get cluster layout and initial topic/partition leadership information + await producer.start() + try: + # Produce message + await producer.send_and_wait(topic, message) + flag = True + finally: + # Wait for all pending messages to be delivered or expire. + await producer.stop() + + return flag diff --git a/minos/networks/broker/event.py b/minos/networks/broker/event.py new file mode 100644 index 00000000..2164545c --- /dev/null +++ b/minos/networks/broker/event.py @@ -0,0 +1,68 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. +from __future__ import ( + annotations, +) +import datetime +from typing import ( + NoReturn, + Optional, +) + +from minos.common import ( + Aggregate, + Event, + MinosConfig, +) + +from .abc import ( + MinosBroker, +) + + +class MinosEventBroker(MinosBroker): + """TODO""" + + ACTION = "event" + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosEventBroker]: + """Build a new repository from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + return None + # noinspection PyProtectedMember + return cls(*args, **config.events.queue._asdict(), **kwargs) + + async def send(self, items: list[Aggregate]) -> NoReturn: + event_instance = Event(self.topic, items) + bin_data = event_instance.avro_bytes + + async with self._connection() as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO producer_queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", + ( + event_instance.topic, + bin_data, + 0, + self.ACTION, + datetime.datetime.now(), + datetime.datetime.now(), + ), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + return affected_rows, queue_id[0] diff --git a/tests/broker/__init__.py b/tests/broker/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/broker/conftest.py b/tests/broker/conftest.py deleted file mode 100644 index c6bc320e..00000000 --- a/tests/broker/conftest.py +++ /dev/null @@ -1,51 +0,0 @@ -import pytest -from aiokafka import AIOKafkaConsumer -from aiomisc.service.periodic import Service -from minos.common.configuration.config import MinosConfig -from minos.common.logs import log -from minos.networks.broker import (BrokerDatabaseInitializer, - EventBrokerQueueDispatcher, - MinosBrokerDatabase) - - -@pytest.fixture -def broker_config(): - return MinosConfig(path="./tests/test_config.yaml") - - -@pytest.fixture -async def database(broker_config): - return await MinosBrokerDatabase().get_connection(broker_config) - - -@pytest.fixture -def services(broker_config): - return [ - BrokerDatabaseInitializer(config=broker_config), - EventBrokerQueueDispatcher(interval=0.5, delay=0, config=broker_config), - ] - - -class KafkaConsumer(Service): - async def start(self): - self.start_event.set() - - consumer = AIOKafkaConsumer("EventBroker", "CommandBroker", loop=self.loop, bootstrap_servers="localhost:9092") - # Get cluster layout and join group `my-group` - - await consumer.start() - try: - # Consume messages - async for msg in consumer: - log.debug("+++++++++++++++++++++++++++++++++++++++++++++++") - log.debug(msg.topic) - log.debug(msg.key) - log.debug(msg.value) - log.debug("++++++++++++++++++++++++++++++++++++++++++++++++") - # log.debug("consumed: ", msg.topic, msg.partition, msg.offset, - # msg.key, msg.value, msg.timestamp) - finally: - # Will leave consumer group; perform autocommit if enabled. - await consumer.stop() - - await self.stop(self) diff --git a/tests/broker/database_testcase.py b/tests/broker/database_testcase.py deleted file mode 100644 index 559556b4..00000000 --- a/tests/broker/database_testcase.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Copyright (C) 2021 Clariteia SL -This file is part of minos framework. -Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. -""" -import os -import unittest - -import aiopg -from minos.common.configuration.config import MinosConfig -from minos.networks.broker import MinosBrokerDatabase, broker_table_creation - - -class PostgresAsyncTestCase(unittest.IsolatedAsyncioTestCase): - def setUp(self) -> None: - self._meta_kwargs = { - "host": os.getenv("POSTGRES_HOST", "localhost"), - "port": os.getenv("POSTGRES_PORT", 5432), - "database": os.getenv("POSTGRES_DATABASE", "postgres"), - "user": os.getenv("POSTGRES_USER", "postgres"), - "password": os.getenv("POSTGRES_PASSWORD", ""), - } - - self.kwargs = self._meta_kwargs | { - "database": "broker_db", - "user": "broker", - "password": "br0k3r", - } - - async def asyncSetUp(self): - await broker_table_creation(self._broker_config()) - """ - async with aiopg.connect(**self._meta_kwargs) as connection: - async with connection.cursor() as cursor: - - template = "DROP DATABASE IF EXISTS {database};" - await cursor.execute(template.format(**self.kwargs)) - - template = "DROP ROLE IF EXISTS {user};" - await cursor.execute(template.format(**self.kwargs)) - - template = "CREATE ROLE {user} WITH CREATEDB LOGIN ENCRYPTED PASSWORD {password!r};" - await cursor.execute(template.format(**self.kwargs)) - - template = "CREATE DATABASE {database} WITH OWNER = {user};" - await cursor.execute(template.format(**self.kwargs)) - """ - - @staticmethod - def _broker_config(): - return MinosConfig(path="./tests/test_config.yaml") - - async def _database(self): - return await MinosBrokerDatabase().get_connection(self._broker_config()) - - async def asyncTearDown(self): - pass - """ - database = await self._database() - async with database as connection: - async with connection.cursor() as cursor: - template = "DROP TABLE IF EXISTS queue" - await cursor.execute(template) - """ diff --git a/tests/broker/test_broker.py b/tests/broker/test_broker.py deleted file mode 100644 index 8905d920..00000000 --- a/tests/broker/test_broker.py +++ /dev/null @@ -1,167 +0,0 @@ -import asyncio -import time - -import pytest -from minos.common import Aggregate -from minos.common.configuration.config import MinosConfig -from minos.common.logs import log -from minos.networks.broker import (MinosCommandBroker, MinosEventBroker, - broker_queue_dispatcher, send_to_kafka) -from tests.broker.database_testcase import PostgresAsyncTestCase - - -class AggregateTest(Aggregate): - test: int - - -class TestPostgreSqlMinosBroker(PostgresAsyncTestCase): - async def test_database_connection(self): - database = await self._database() - async with database as connect: - assert database.closed == 0 - - async def test_if_queue_table_exists(self): - database = await self._database() - async with database as connect: - async with connect.cursor() as cur: - - await cur.execute( - "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'producer_queue';" - ) - ret = [] - async for row in cur: - ret.append(row) - - assert ret == [(1,)] - - async def test_send_to_kafka_ok(self): - response = await send_to_kafka(topic="TestKafkaSend", message=bytes(), config=self._broker_config()) - assert response is True - - async def test_events_broker_insertion(self): - a = AggregateTest(test_id=1, test=2, id=1, version=1) - - m = MinosEventBroker("EventBroker", self._broker_config()) - - affected_rows, queue_id = await m.send(a) - - assert affected_rows == 1 - assert queue_id > 0 - - async def test_if_events_was_deleted(self): - a = AggregateTest(test_id=1, test=2, id=1, version=1) - m = MinosEventBroker("EventBroker-Delete", self._broker_config()) - affected_rows_1, queue_id_1 = await m.send(a) - affected_rows_2, queue_id_2 = await m.send(a) - - await broker_queue_dispatcher(self._broker_config()) - - database = await self._database() - async with database as connect: - async with connect.cursor() as cur: - await cur.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "EventBroker-Delete") - records = await cur.fetchone() - - assert affected_rows_1 == 1 - assert queue_id_1 > 0 - assert affected_rows_2 == 1 - assert queue_id_2 > 0 - assert records[0] == 0 - - async def test_commands_broker_insertion(self): - a = AggregateTest(test_id=1, test=2, id=1, version=1) - - m = MinosCommandBroker("CommandBroker", self._broker_config()) - - affected_rows, queue_id = await m.send(model=a, reply_on="test_reply_on") - assert affected_rows == 1 - assert queue_id > 0 - - async def test_if_commands_was_deleted(self): - a = AggregateTest(test_id=1, test=2, id=1, version=1) - m = MinosCommandBroker("CommandBroker-Delete", self._broker_config()) - affected_rows_1, queue_id_1 = await m.send(a, reply_on="test_reply_on") - affected_rows_2, queue_id_2 = await m.send(a, reply_on="test_reply_on") - - await broker_queue_dispatcher(self._broker_config()) - - database = await self._database() - async with database as connect: - async with connect.cursor() as cur: - await cur.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "CommandBroker-Delete") - records = await cur.fetchone() - - assert affected_rows_1 == 1 - assert queue_id_1 > 0 - assert affected_rows_2 == 1 - assert queue_id_2 > 0 - assert records[0] == 0 - - async def test_if_commands_retry_was_incremented(self): - a = AggregateTest(test_id=1, test=2, id=1, version=1) - m = MinosCommandBroker("CommandBroker-Delete", self._broker_config()) - affected_rows_1, queue_id_1 = await m.send(a, reply_on="test_reply_on") - affected_rows_2, queue_id_2 = await m.send(a, reply_on="test_reply_on") - - config = MinosConfig(path="./tests/wrong_test_config.yaml") - - await broker_queue_dispatcher(config) - - database = await self._database() - async with database as connect: - async with connect.cursor() as cur: - await cur.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "CommandBroker-Delete") - records = await cur.fetchone() - - await cur.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_1) - retry_1 = await cur.fetchone() - - await cur.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_2) - retry_2 = await cur.fetchone() - - assert affected_rows_1 == 1 - assert queue_id_1 > 0 - assert affected_rows_2 == 1 - assert queue_id_2 > 0 - assert records[0] == 2 - assert retry_1[0] > 0 - assert retry_2[0] > 0 - - async def test_if_events_retry_was_incremented(self): - a = AggregateTest(test_id=1, test=2, id=1, version=1) - m = MinosEventBroker("EventBroker-Delete", self._broker_config()) - affected_rows_1, queue_id_1 = await m.send(a) - affected_rows_2, queue_id_2 = await m.send(a) - - config = MinosConfig(path="./tests/wrong_test_config.yaml") - - await broker_queue_dispatcher(config) - - database = await self._database() - async with database as connect: - async with connect.cursor() as cur: - await cur.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "EventBroker-Delete") - records = await cur.fetchone() - - await cur.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_1) - retry_1 = await cur.fetchone() - - await cur.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_2) - retry_2 = await cur.fetchone() - - assert affected_rows_1 == 1 - assert queue_id_1 > 0 - assert affected_rows_2 == 1 - assert queue_id_2 > 0 - assert records[0] == 2 - assert retry_1[0] > 0 - assert retry_2[0] > 0 - - -""" -create role broker with createdb login password 'br0k3r'; -CREATE DATABASE broker_db OWNER broker; -'CREATE TABLE IF NOT EXISTS "queue" ("queue_id" SERIAL NOT NULL PRIMARY KEY, "topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL)', [] -'SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = %s ORDER BY tablename', ('public',) -'DROP TABLE IF EXISTS "queue"' -""" diff --git a/tests/services/CqrsTestService.py b/tests/services/CqrsTestService.py deleted file mode 100644 index 512422eb..00000000 --- a/tests/services/CqrsTestService.py +++ /dev/null @@ -1,3 +0,0 @@ -class CqrsService(object): - def ticket_added(self, request): - return "request_added" diff --git a/tests/services/__init__.py b/tests/services/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/test_config.yaml b/tests/test_config.yaml index 40bce1ee..4a76721d 100644 --- a/tests/test_config.yaml +++ b/tests/test_config.yaml @@ -9,8 +9,14 @@ rest: method: POST controller: minos.services.OrderService action: add_order +repository: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 events: - broker: kafka + broker: localhost port: 9092 database: path: ./tests/local_db.lmdb @@ -23,15 +29,15 @@ events: controller: tests.services.CqrsTestService.CqrsService action: ticket_deleted queue: - database: broker_db - user: broker - password: br0k3r - host: postgres + database: order_db + user: minos + password: min0s + host: localhost port: 5432 records: 10 retry: 2 commands: - broker: kafka + broker: localhost port: 9092 items: - name: AddOrder @@ -47,10 +53,10 @@ commands: controller: minos.service.OrderService action: get_order queue: - database: broker_db - user: broker - password: br0k3r - host: postgres + database: order_db + user: minos + password: min0s + host: localhost port: 5432 records: 10 retry: 2 diff --git a/tests/test_networks/__init__.py b/tests/test_networks/__init__.py new file mode 100644 index 00000000..fed3716d --- /dev/null +++ b/tests/test_networks/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" diff --git a/tests/test_networks/test_broker/__init__.py b/tests/test_networks/test_broker/__init__.py new file mode 100644 index 00000000..fed3716d --- /dev/null +++ b/tests/test_networks/test_broker/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py new file mode 100644 index 00000000..fe10fd94 --- /dev/null +++ b/tests/test_networks/test_broker/test_command.py @@ -0,0 +1,102 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +import unittest + +import aiopg +from minos.common import ( + Aggregate, + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) + +from minos.networks import ( + MinosCommandBroker, + MinosQueueDispatcher, +) +from tests.utils import ( + BASE_PATH, +) + + +class AggregateTest(Aggregate): + test: int + + +class TestMinosCommandBroker(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yaml" + + async def test_commands_broker_insertion(self): + broker = MinosCommandBroker.from_config("CommandBroker", config=self.config, reply_on="test_reply_on") + await broker.setup() + + a = AggregateTest(test_id=1, test=2, id=1, version=1) + + affected_rows, queue_id = await broker.send_one(a) + assert affected_rows == 1 + assert queue_id > 0 + + async def test_if_commands_was_deleted(self): + broker = MinosCommandBroker.from_config("CommandBroker-Delete", config=self.config, reply_on="test_reply_on") + await broker.setup() + + a = AggregateTest(test_id=1, test=2, id=1, version=1) + + affected_rows_1, queue_id_1 = await broker.send_one(a) + affected_rows_2, queue_id_2 = await broker.send_one(a) + + await MinosQueueDispatcher.from_config(config=self.config).dispatch() + + async with aiopg.connect(**self.events_queue_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "CommandBroker-Delete") + records = await cursor.fetchone() + + assert affected_rows_1 == 1 + assert queue_id_1 > 0 + assert affected_rows_2 == 1 + assert queue_id_2 > 0 + assert records[0] == 0 + + async def test_if_commands_retry_was_incremented(self): + broker = MinosCommandBroker.from_config("CommandBroker-Delete", config=self.config, reply_on="test_reply_on") + await broker.setup() + + a = AggregateTest(test_id=1, test=2, id=1, version=1) + + affected_rows_1, queue_id_1 = await broker.send_one(a) + affected_rows_2, queue_id_2 = await broker.send_one(a) + + config = MinosConfig( + path=BASE_PATH / "wrong_test_config.yaml", events_queue_database="test_db", events_queue_user="test_user" + ) + await MinosQueueDispatcher.from_config(config=config).dispatch() + + async with aiopg.connect(**self.events_queue_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "CommandBroker-Delete") + records = await cursor.fetchone() + + await cursor.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_1) + retry_1 = await cursor.fetchone() + + await cursor.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_2) + retry_2 = await cursor.fetchone() + + assert affected_rows_1 == 1 + assert queue_id_1 > 0 + assert affected_rows_2 == 1 + assert queue_id_2 > 0 + assert records[0] == 2 + assert retry_1[0] > 0 + assert retry_2[0] > 0 + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py new file mode 100644 index 00000000..e32fd4ee --- /dev/null +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -0,0 +1,30 @@ +import unittest + +from minos.common import ( + Aggregate, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) + +from minos.networks.broker import MinosQueueDispatcher +from tests.utils import ( + BASE_PATH, +) + + +class AggregateTest(Aggregate): + test: int + + +class TestQueueDispatcher(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yaml" + + def test_from_config(self): + dispatcher = MinosQueueDispatcher.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosQueueDispatcher) + + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py new file mode 100644 index 00000000..5f1efbaf --- /dev/null +++ b/tests/test_networks/test_broker/test_events.py @@ -0,0 +1,115 @@ +import unittest + +import aiopg +from minos.common import ( + Aggregate, + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) + +from minos.networks import ( + MinosEventBroker, + MinosQueueDispatcher, +) +from tests.utils import ( + BASE_PATH, +) + + +class AggregateTest(Aggregate): + test: int + + +class TestMinosEventBroker(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yaml" + + async def test_if_queue_table_exists(self): + broker = MinosEventBroker.from_config("EventBroker", config=self.config) + await broker.setup() + + async with aiopg.connect(**self.events_queue_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'producer_queue';" + ) + ret = [] + async for row in cursor: + ret.append(row) + + assert ret == [(1,)] + + async def test_send_to_kafka_ok(self): + dispatcher = MinosQueueDispatcher.from_config(config=self.config) + response = await dispatcher.send_to_kafka(topic="TestKafkaSend", message=bytes()) + assert response is True + + async def test_events_broker_insertion(self): + broker = MinosEventBroker.from_config("EventBroker", config=self.config) + await broker.setup() + + a = AggregateTest(test_id=1, test=2, id=1, version=1) + affected_rows, queue_id = await broker.send_one(a) + + assert affected_rows == 1 + assert queue_id > 0 + + async def test_if_events_was_deleted(self): + broker = MinosEventBroker.from_config("EventBroker-Delete", config=self.config) + await broker.setup() + + a = AggregateTest(test_id=1, test=2, id=1, version=1) + affected_rows_1, queue_id_1 = await broker.send_one(a) + affected_rows_2, queue_id_2 = await broker.send_one(a) + + await MinosQueueDispatcher.from_config(config=self.config).dispatch() + + async with aiopg.connect(**self.events_queue_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "EventBroker-Delete") + records = await cursor.fetchone() + + assert affected_rows_1 == 1 + assert queue_id_1 > 0 + assert affected_rows_2 == 1 + assert queue_id_2 > 0 + assert records[0] == 0 + + async def test_if_events_retry_was_incremented(self): + broker = MinosEventBroker.from_config("EventBroker-Delete", config=self.config) + await broker.setup() + + a = AggregateTest(test_id=1, test=2, id=1, version=1) + + affected_rows_1, queue_id_1 = await broker.send_one(a) + affected_rows_2, queue_id_2 = await broker.send_one(a) + + config = MinosConfig( + path=BASE_PATH / "wrong_test_config.yaml", events_queue_database="test_db", events_queue_user="test_user" + ) + + await MinosQueueDispatcher.from_config(config=config).dispatch() + + async with aiopg.connect(**self.events_queue_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "EventBroker-Delete") + records = await cursor.fetchone() + + await cursor.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_1) + retry_1 = await cursor.fetchone() + + await cursor.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_2) + retry_2 = await cursor.fetchone() + + assert affected_rows_1 == 1 + assert queue_id_1 > 0 + assert affected_rows_2 == 1 + assert queue_id_2 > 0 + assert records[0] == 2 + assert retry_1[0] > 0 + assert retry_2[0] > 0 + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 00000000..e41317a2 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,12 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from pathlib import ( + Path, +) + +BASE_PATH = Path(__file__).parent diff --git a/tests/wrong_test_config.yaml b/tests/wrong_test_config.yaml index a368e997..57eeb572 100644 --- a/tests/wrong_test_config.yaml +++ b/tests/wrong_test_config.yaml @@ -9,6 +9,12 @@ rest: method: POST controller: minos.services.OrderService action: add_order +repository: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 events: broker: localhost port: 4092 @@ -23,10 +29,10 @@ events: controller: tests.services.CqrsTestService.CqrsService action: ticket_deleted queue: - database: broker_db - user: broker - password: br0k3r - host: postgres + database: order_db + user: minos + password: min0s + host: localhost port: 5432 records: 10 retry: 2 @@ -47,10 +53,10 @@ commands: controller: minos.service.OrderService action: get_order queue: - database: broker_db - user: broker - password: br0k3r - host: postgres + database: order_db + user: minos + password: min0s + host: localhost port: 5432 records: 10 retry: 2 From b9af1ced3fbb29946eb15675cf71483014d07ef6 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 28 Apr 2021 17:01:58 +0000 Subject: [PATCH 091/239] Restyled by black --- minos/networks/broker/__init__.py | 8 ++------ minos/networks/broker/abc.py | 10 ++-------- minos/networks/broker/command.py | 20 +++++-------------- minos/networks/broker/dispatcher.py | 17 ++++------------ minos/networks/broker/event.py | 17 +++------------- .../test_networks/test_broker/test_command.py | 8 ++------ .../test_broker/test_dispatcher.py | 13 +++--------- .../test_networks/test_broker/test_events.py | 8 ++------ tests/utils.py | 4 +--- 9 files changed, 24 insertions(+), 81 deletions(-) diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py index 050afc17..fd849821 100644 --- a/minos/networks/broker/__init__.py +++ b/minos/networks/broker/__init__.py @@ -5,12 +5,8 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .event import ( - MinosEventBroker, -) -from .command import ( - MinosCommandBroker, -) +from .event import MinosEventBroker +from .command import MinosCommandBroker from .dispatcher import ( MinosQueueDispatcher, EventBrokerQueueDispatcher, diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index c89d76d3..0887d963 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - NoReturn, -) +from typing import NoReturn import aiopg from minos.common import ( @@ -49,11 +47,7 @@ async def broker_table_creation(self): def _connection(self): return aiopg.connect( - host=self.host, - port=self.port, - dbname=self.database, - user=self.user, - password=self.password, + host=self.host, port=self.port, dbname=self.database, user=self.user, password=self.password, ) diff --git a/minos/networks/broker/command.py b/minos/networks/broker/command.py index 2ee56ce4..a7f84fc9 100644 --- a/minos/networks/broker/command.py +++ b/minos/networks/broker/command.py @@ -5,12 +5,11 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations import datetime from typing import ( - NoReturn, Optional, + NoReturn, + Optional, ) from minos.common import ( @@ -19,9 +18,7 @@ MinosConfig, ) -from .abc import ( - MinosBroker, -) +from .abc import MinosBroker class MinosCommandBroker(MinosBroker): @@ -56,14 +53,7 @@ async def send(self, items: list[Aggregate]) -> NoReturn: async with connect.cursor() as cur: await cur.execute( "INSERT INTO producer_queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", - ( - event_instance.topic, - bin_data, - 0, - self.ACTION, - datetime.datetime.now(), - datetime.datetime.now(), - ), + (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), ) queue_id = await cur.fetchone() diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py index 041d6d47..23b70b29 100644 --- a/minos/networks/broker/dispatcher.py +++ b/minos/networks/broker/dispatcher.py @@ -5,26 +5,17 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import Optional, Any -from aiokafka import ( - AIOKafkaProducer, -) -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) +from aiokafka import AIOKafkaProducer +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig from .abc import MinosBrokerSetup class EventBrokerQueueDispatcher(PeriodicService): - def __init__(self, config: MinosConfig = None, **kwargs): super().__init__(**kwargs) self.dispatcher = MinosQueueDispatcher.from_config(config=config) diff --git a/minos/networks/broker/event.py b/minos/networks/broker/event.py index 2164545c..217bd754 100644 --- a/minos/networks/broker/event.py +++ b/minos/networks/broker/event.py @@ -4,9 +4,7 @@ # # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import ( - annotations, -) +from __future__ import annotations import datetime from typing import ( NoReturn, @@ -19,9 +17,7 @@ MinosConfig, ) -from .abc import ( - MinosBroker, -) +from .abc import MinosBroker class MinosEventBroker(MinosBroker): @@ -52,14 +48,7 @@ async def send(self, items: list[Aggregate]) -> NoReturn: async with connect.cursor() as cur: await cur.execute( "INSERT INTO producer_queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", - ( - event_instance.topic, - bin_data, - 0, - self.ACTION, - datetime.datetime.now(), - datetime.datetime.now(), - ), + (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), ) queue_id = await cur.fetchone() diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index fe10fd94..d100a226 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -12,17 +12,13 @@ Aggregate, MinosConfig, ) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class AggregateTest(Aggregate): diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py index e32fd4ee..e16ecb3c 100644 --- a/tests/test_networks/test_broker/test_dispatcher.py +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -1,16 +1,10 @@ import unittest -from minos.common import ( - Aggregate, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import Aggregate +from minos.common.testing import PostgresAsyncTestCase from minos.networks.broker import MinosQueueDispatcher -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class AggregateTest(Aggregate): @@ -25,6 +19,5 @@ def test_from_config(self): self.assertIsInstance(dispatcher, MinosQueueDispatcher) - if __name__ == "__main__": unittest.main() diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index 5f1efbaf..edb74871 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -5,17 +5,13 @@ Aggregate, MinosConfig, ) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosEventBroker, MinosQueueDispatcher, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class AggregateTest(Aggregate): diff --git a/tests/utils.py b/tests/utils.py index e41317a2..e6faf60d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,8 +5,6 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from pathlib import ( - Path, -) +from pathlib import Path BASE_PATH = Path(__file__).parent From ab18659888acc065e2b7cfdd3476c8c0cc658996 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 28 Apr 2021 17:02:06 +0000 Subject: [PATCH 092/239] Restyled by isort --- minos/networks/broker/__init__.py | 10 ++++--- minos/networks/broker/abc.py | 4 ++- minos/networks/broker/command.py | 9 +++++-- minos/networks/broker/dispatcher.py | 26 ++++++++++++++----- minos/networks/broker/event.py | 9 +++++-- .../test_networks/test_broker/test_command.py | 9 ++++--- .../test_broker/test_dispatcher.py | 17 ++++++++---- .../test_networks/test_broker/test_events.py | 9 ++++--- tests/utils.py | 4 ++- 9 files changed, 71 insertions(+), 26 deletions(-) diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py index fd849821..31413e47 100644 --- a/minos/networks/broker/__init__.py +++ b/minos/networks/broker/__init__.py @@ -5,9 +5,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .event import MinosEventBroker -from .command import MinosCommandBroker +from .command import ( + MinosCommandBroker, +) from .dispatcher import ( - MinosQueueDispatcher, EventBrokerQueueDispatcher, + MinosQueueDispatcher, +) +from .event import ( + MinosEventBroker, ) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index 0887d963..60ee36e0 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import NoReturn +from typing import ( + NoReturn, +) import aiopg from minos.common import ( diff --git a/minos/networks/broker/command.py b/minos/networks/broker/command.py index a7f84fc9..9799effc 100644 --- a/minos/networks/broker/command.py +++ b/minos/networks/broker/command.py @@ -5,7 +5,10 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) + import datetime from typing import ( NoReturn, @@ -18,7 +21,9 @@ MinosConfig, ) -from .abc import MinosBroker +from .abc import ( + MinosBroker, +) class MinosCommandBroker(MinosBroker): diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py index 23b70b29..5937c3a1 100644 --- a/minos/networks/broker/dispatcher.py +++ b/minos/networks/broker/dispatcher.py @@ -5,14 +5,28 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations -from typing import Optional, Any +from __future__ import ( + annotations, +) -from aiokafka import AIOKafkaProducer -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig +from typing import ( + Any, + Optional, +) -from .abc import MinosBrokerSetup +from aiokafka import ( + AIOKafkaProducer, +) +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .abc import ( + MinosBrokerSetup, +) class EventBrokerQueueDispatcher(PeriodicService): diff --git a/minos/networks/broker/event.py b/minos/networks/broker/event.py index 217bd754..cc180204 100644 --- a/minos/networks/broker/event.py +++ b/minos/networks/broker/event.py @@ -4,7 +4,10 @@ # # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import annotations +from __future__ import ( + annotations, +) + import datetime from typing import ( NoReturn, @@ -17,7 +20,9 @@ MinosConfig, ) -from .abc import MinosBroker +from .abc import ( + MinosBroker, +) class MinosEventBroker(MinosBroker): diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index d100a226..7b53b2ea 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -12,13 +12,16 @@ Aggregate, MinosConfig, ) -from minos.common.testing import PostgresAsyncTestCase - +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class AggregateTest(Aggregate): diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py index e16ecb3c..bfb0cd41 100644 --- a/tests/test_networks/test_broker/test_dispatcher.py +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -1,10 +1,17 @@ import unittest -from minos.common import Aggregate -from minos.common.testing import PostgresAsyncTestCase - -from minos.networks.broker import MinosQueueDispatcher -from tests.utils import BASE_PATH +from minos.common import ( + Aggregate, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks.broker import ( + MinosQueueDispatcher, +) +from tests.utils import ( + BASE_PATH, +) class AggregateTest(Aggregate): diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index edb74871..3b389c05 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -5,13 +5,16 @@ Aggregate, MinosConfig, ) -from minos.common.testing import PostgresAsyncTestCase - +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosEventBroker, MinosQueueDispatcher, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class AggregateTest(Aggregate): diff --git a/tests/utils.py b/tests/utils.py index e6faf60d..e41317a2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,6 +5,8 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from pathlib import Path +from pathlib import ( + Path, +) BASE_PATH = Path(__file__).parent From 83ff13f1c9437742c080b257410e290e725e5b3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 28 Apr 2021 19:06:41 +0200 Subject: [PATCH 093/239] REFS #66 * Minor import-related improvements. --- minos/networks/__init__.py | 3 +++ minos/networks/broker/abc.py | 4 ++++ minos/networks/broker/dispatcher.py | 3 ++- minos/networks/exceptions.py | 2 +- tests/test_networks/test_broker/test_dispatcher.py | 2 +- 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index c9f93575..7d64112b 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -6,3 +6,6 @@ MinosEventBroker, MinosQueueDispatcher, ) +from .exceptions import ( + MinosNetworkException, +) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index 60ee36e0..9f6ef002 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -17,6 +17,8 @@ class MinosBrokerSetup(MinosSetup): + """TODO""" + def __init__( self, *args, @@ -54,6 +56,8 @@ def _connection(self): class MinosBroker(MinosBaseBroker, MinosBrokerSetup): + """TODO""" + def __init__(self, topic: str, *args, **kwargs): MinosBaseBroker.__init__(self, topic) MinosBrokerSetup.__init__(self, *args, **kwargs) diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py index 5937c3a1..c206cd29 100644 --- a/minos/networks/broker/dispatcher.py +++ b/minos/networks/broker/dispatcher.py @@ -10,7 +10,6 @@ ) from typing import ( - Any, Optional, ) @@ -30,6 +29,8 @@ class EventBrokerQueueDispatcher(PeriodicService): + """TODO""" + def __init__(self, config: MinosConfig = None, **kwargs): super().__init__(**kwargs) self.dispatcher = MinosQueueDispatcher.from_config(config=config) diff --git a/minos/networks/exceptions.py b/minos/networks/exceptions.py index 8e573e1f..66f4faa3 100644 --- a/minos/networks/exceptions.py +++ b/minos/networks/exceptions.py @@ -1,4 +1,4 @@ -from minos.common.exceptions import MinosException +from minos.common import MinosException class MinosNetworkException(MinosException): pass diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py index bfb0cd41..de9abbbf 100644 --- a/tests/test_networks/test_broker/test_dispatcher.py +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -6,7 +6,7 @@ from minos.common.testing import ( PostgresAsyncTestCase, ) -from minos.networks.broker import ( +from minos.networks import ( MinosQueueDispatcher, ) from tests.utils import ( From 129ed5aec200a55e31938de26f3123914157b637 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 28 Apr 2021 17:06:59 +0000 Subject: [PATCH 094/239] Restyled by black --- minos/networks/__init__.py | 4 +--- minos/networks/broker/__init__.py | 8 ++----- minos/networks/broker/abc.py | 4 +--- minos/networks/broker/command.py | 8 ++----- minos/networks/broker/dispatcher.py | 24 +++++-------------- minos/networks/broker/event.py | 8 ++----- minos/networks/exceptions.py | 3 ++- .../test_networks/test_broker/test_command.py | 8 ++----- .../test_broker/test_dispatcher.py | 16 ++++--------- .../test_networks/test_broker/test_events.py | 8 ++----- tests/utils.py | 4 +--- 11 files changed, 25 insertions(+), 70 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 7d64112b..61a0d7c6 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -6,6 +6,4 @@ MinosEventBroker, MinosQueueDispatcher, ) -from .exceptions import ( - MinosNetworkException, -) +from .exceptions import MinosNetworkException diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py index 31413e47..afa71f3e 100644 --- a/minos/networks/broker/__init__.py +++ b/minos/networks/broker/__init__.py @@ -5,13 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .command import ( - MinosCommandBroker, -) +from .command import MinosCommandBroker from .dispatcher import ( EventBrokerQueueDispatcher, MinosQueueDispatcher, ) -from .event import ( - MinosEventBroker, -) +from .event import MinosEventBroker diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index 9f6ef002..5826f459 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - NoReturn, -) +from typing import NoReturn import aiopg from minos.common import ( diff --git a/minos/networks/broker/command.py b/minos/networks/broker/command.py index 9799effc..5a87b4a7 100644 --- a/minos/networks/broker/command.py +++ b/minos/networks/broker/command.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations import datetime from typing import ( @@ -21,9 +19,7 @@ MinosConfig, ) -from .abc import ( - MinosBroker, -) +from .abc import MinosBroker class MinosCommandBroker(MinosBroker): diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py index c206cd29..41ef947c 100644 --- a/minos/networks/broker/dispatcher.py +++ b/minos/networks/broker/dispatcher.py @@ -5,27 +5,15 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations -from typing import ( - Optional, -) +from typing import Optional -from aiokafka import ( - AIOKafkaProducer, -) -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) +from aiokafka import AIOKafkaProducer +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig -from .abc import ( - MinosBrokerSetup, -) +from .abc import MinosBrokerSetup class EventBrokerQueueDispatcher(PeriodicService): diff --git a/minos/networks/broker/event.py b/minos/networks/broker/event.py index cc180204..95652176 100644 --- a/minos/networks/broker/event.py +++ b/minos/networks/broker/event.py @@ -4,9 +4,7 @@ # # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import ( - annotations, -) +from __future__ import annotations import datetime from typing import ( @@ -20,9 +18,7 @@ MinosConfig, ) -from .abc import ( - MinosBroker, -) +from .abc import MinosBroker class MinosEventBroker(MinosBroker): diff --git a/minos/networks/exceptions.py b/minos/networks/exceptions.py index 66f4faa3..36777e6e 100644 --- a/minos/networks/exceptions.py +++ b/minos/networks/exceptions.py @@ -1,4 +1,5 @@ from minos.common import MinosException -class MinosNetworkException(MinosException): pass +class MinosNetworkException(MinosException): + pass diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 7b53b2ea..9c2dc309 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -12,16 +12,12 @@ Aggregate, MinosConfig, ) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class AggregateTest(Aggregate): diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py index de9abbbf..e6d0fdcf 100644 --- a/tests/test_networks/test_broker/test_dispatcher.py +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -1,17 +1,9 @@ import unittest -from minos.common import ( - Aggregate, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) -from minos.networks import ( - MinosQueueDispatcher, -) -from tests.utils import ( - BASE_PATH, -) +from minos.common import Aggregate +from minos.common.testing import PostgresAsyncTestCase +from minos.networks import MinosQueueDispatcher +from tests.utils import BASE_PATH class AggregateTest(Aggregate): diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index 3b389c05..c1cb614b 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -5,16 +5,12 @@ Aggregate, MinosConfig, ) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosEventBroker, MinosQueueDispatcher, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class AggregateTest(Aggregate): diff --git a/tests/utils.py b/tests/utils.py index e41317a2..e6faf60d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,8 +5,6 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from pathlib import ( - Path, -) +from pathlib import Path BASE_PATH = Path(__file__).parent From 99e5cc95b248b946d56870d76feff076cedbb144 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 28 Apr 2021 17:07:06 +0000 Subject: [PATCH 095/239] Restyled by isort --- minos/networks/__init__.py | 4 +++- minos/networks/broker/__init__.py | 8 +++++-- minos/networks/broker/abc.py | 4 +++- minos/networks/broker/command.py | 8 +++++-- minos/networks/broker/dispatcher.py | 24 ++++++++++++++----- minos/networks/broker/event.py | 8 +++++-- minos/networks/exceptions.py | 4 +++- .../test_networks/test_broker/test_command.py | 8 +++++-- .../test_broker/test_dispatcher.py | 16 +++++++++---- .../test_networks/test_broker/test_events.py | 8 +++++-- tests/utils.py | 4 +++- 11 files changed, 72 insertions(+), 24 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 61a0d7c6..7d64112b 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -6,4 +6,6 @@ MinosEventBroker, MinosQueueDispatcher, ) -from .exceptions import MinosNetworkException +from .exceptions import ( + MinosNetworkException, +) diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py index afa71f3e..31413e47 100644 --- a/minos/networks/broker/__init__.py +++ b/minos/networks/broker/__init__.py @@ -5,9 +5,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .command import MinosCommandBroker +from .command import ( + MinosCommandBroker, +) from .dispatcher import ( EventBrokerQueueDispatcher, MinosQueueDispatcher, ) -from .event import MinosEventBroker +from .event import ( + MinosEventBroker, +) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index 5826f459..9f6ef002 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import NoReturn +from typing import ( + NoReturn, +) import aiopg from minos.common import ( diff --git a/minos/networks/broker/command.py b/minos/networks/broker/command.py index 5a87b4a7..9799effc 100644 --- a/minos/networks/broker/command.py +++ b/minos/networks/broker/command.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) import datetime from typing import ( @@ -19,7 +21,9 @@ MinosConfig, ) -from .abc import MinosBroker +from .abc import ( + MinosBroker, +) class MinosCommandBroker(MinosBroker): diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py index 41ef947c..c206cd29 100644 --- a/minos/networks/broker/dispatcher.py +++ b/minos/networks/broker/dispatcher.py @@ -5,15 +5,27 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) -from typing import Optional +from typing import ( + Optional, +) -from aiokafka import AIOKafkaProducer -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig +from aiokafka import ( + AIOKafkaProducer, +) +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) -from .abc import MinosBrokerSetup +from .abc import ( + MinosBrokerSetup, +) class EventBrokerQueueDispatcher(PeriodicService): diff --git a/minos/networks/broker/event.py b/minos/networks/broker/event.py index 95652176..cc180204 100644 --- a/minos/networks/broker/event.py +++ b/minos/networks/broker/event.py @@ -4,7 +4,9 @@ # # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import annotations +from __future__ import ( + annotations, +) import datetime from typing import ( @@ -18,7 +20,9 @@ MinosConfig, ) -from .abc import MinosBroker +from .abc import ( + MinosBroker, +) class MinosEventBroker(MinosBroker): diff --git a/minos/networks/exceptions.py b/minos/networks/exceptions.py index 36777e6e..14a1b797 100644 --- a/minos/networks/exceptions.py +++ b/minos/networks/exceptions.py @@ -1,4 +1,6 @@ -from minos.common import MinosException +from minos.common import ( + MinosException, +) class MinosNetworkException(MinosException): diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 9c2dc309..7b53b2ea 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -12,12 +12,16 @@ Aggregate, MinosConfig, ) -from minos.common.testing import PostgresAsyncTestCase +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class AggregateTest(Aggregate): diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py index e6d0fdcf..de9abbbf 100644 --- a/tests/test_networks/test_broker/test_dispatcher.py +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -1,9 +1,17 @@ import unittest -from minos.common import Aggregate -from minos.common.testing import PostgresAsyncTestCase -from minos.networks import MinosQueueDispatcher -from tests.utils import BASE_PATH +from minos.common import ( + Aggregate, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosQueueDispatcher, +) +from tests.utils import ( + BASE_PATH, +) class AggregateTest(Aggregate): diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index c1cb614b..3b389c05 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -5,12 +5,16 @@ Aggregate, MinosConfig, ) -from minos.common.testing import PostgresAsyncTestCase +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosEventBroker, MinosQueueDispatcher, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class AggregateTest(Aggregate): diff --git a/tests/utils.py b/tests/utils.py index e6faf60d..e41317a2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,6 +5,8 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from pathlib import Path +from pathlib import ( + Path, +) BASE_PATH = Path(__file__).parent From b0daf58897217cbd7934d8f6d24ba86c4f66b7e5 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 28 Apr 2021 20:48:40 +0200 Subject: [PATCH 096/239] Initial version of EventHandler PeriodicService EventHandler PeriodicService #71 --- minos/networks/event.py | 96 +++++++++++++++++++++++++++---- tests/services/CqrsTestService.py | 8 ++- tests/test_event_manager.py | 39 +++++++++++-- 3 files changed, 126 insertions(+), 17 deletions(-) diff --git a/minos/networks/event.py b/minos/networks/event.py index f904275f..0b2c8a60 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -9,9 +9,11 @@ import functools import typing as t import datetime +import inspect from aiokafka import AIOKafkaConsumer from aiomisc import Service +from aiomisc.service.periodic import PeriodicService import aiopg from minos.common.configuration.config import MinosConfig from minos.common.importlib import import_module @@ -20,6 +22,7 @@ from minos.networks.exceptions import MinosNetworkException from minos.common.broker import Event + class MinosEventServer(Service): """ Event Manager @@ -117,17 +120,6 @@ async def start(self) -> t.Any: self.create_task(self.handle_message(consumer)) - def _get_event_handler(self, topic: str) -> t.Callable: - for event in self._handlers: - if event.name == topic: - # the topic exist, get the controller and the action - controller = event.controller - action = event.action - object_class = import_module(controller) - instance_class = object_class() - return functools.partial(instance_class.action) - raise MinosNetworkException(f"topic {topic} have no controller/action configured, " - f"please review th configuration file") async def event_handler_table_creation(conf: MinosConfig): @@ -151,3 +143,85 @@ async def start(self): await self.stop(self) + +class MinosEventHandlerPeriodicService(PeriodicService): + """ + Periodic Service Event Handler + + """ + __slots__ = "_db_dsn", "_handlers", "_event_items", "_topics" , "_conf" + + def __init__(self, *, conf: MinosConfig, **kwargs: t.Any): + super().__init__(**kwargs) + self._db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ + f"password={conf.events.queue.password} host={conf.events.queue.host}" + self._handlers = {item.name: {'controller': item.controller, 'action': item.action} + for item in conf.events.items} + self._event_items = conf.events.items + self._topics = list(self._handlers.keys()) + self._conf = conf + + def get_event_handler(self, topic: str) -> t.Callable: + """Get Event instance to call. + + Gets the instance of the class and method to call. + + Args: + topic: Kafka topic. Example: "TicketAdded" + + Raises: + MinosNetworkException: topic TicketAdded have no controller/action configured, please review th configuration file. + """ + for event in self._event_items: + if event.name == topic: + # the topic exist, get the controller and the action + controller = event.controller + action = event.action + + object_class = import_module(controller) + log.debug(object_class()) + instance_class = object_class() + class_method = getattr(instance_class, action) + + return class_method + raise MinosNetworkException(f"topic {topic} have no controller/action configured, " + f"please review th configuration file") + + async def event_queue_checker(self): + """Event Queue Checker and dispatcher. + + It is in charge of querying the database and calling the action according to the topic. + + 1. Get periodically 10 records (or as many as defined in config > queue > records). + 2. Instantiate the action (asynchronous) by passing it the model. + 3. If the invoked function terminates successfully, remove the event from the database. + + Raises: + Exception: An error occurred inserting record. + """ + db_dsn = f"dbname={self._conf.events.queue.database} user={self._conf.events.queue.user} " \ + f"password={self._conf.events.queue.password} host={self._conf.events.queue.host}" + async with aiopg.create_pool(db_dsn) as pool: + async with pool.acquire() as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT * FROM event_queue ORDER BY creation_date ASC LIMIT %d;" + % (self._conf.events.queue.records), + ) + async for row in cur: + call_ok = False + try: + reply_on = self.get_event_handler(topic=row[1]) + event_instance = Event.from_avro_bytes(row[3]) + await reply_on(topic=row[1], event=event_instance) + call_ok = True + except: + call_ok = False + finally: + if call_ok: + # Delete from database If the event was sent successfully to Kafka. + async with connect.cursor() as cur2: + await cur2.execute("DELETE FROM event_queue WHERE id=%d;" % row[0]) + + async def callback(self): + await self.event_queue_checker() diff --git a/tests/services/CqrsTestService.py b/tests/services/CqrsTestService.py index 512422eb..dfbe0293 100644 --- a/tests/services/CqrsTestService.py +++ b/tests/services/CqrsTestService.py @@ -1,3 +1,9 @@ +from minos.common.broker import Event + + class CqrsService(object): - def ticket_added(self, request): + async def ticket_added(self, topic: str, event: Event): return "request_added" + + async def ticket_deleted(self, topic: str, event: Event): + return "ticket_deleted" diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 5af67d5a..66d11f4a 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -6,22 +6,23 @@ from aiokafka import AIOKafkaProducer, AIOKafkaConsumer import random +from minos.common.logs import log from aiomisc.log import basic_config from minos.common.configuration.config import MinosConfig from minos.common import Aggregate from minos.common.broker import Event -from minos.networks.event import MinosEventServer, EventHandlerDatabaseInitializer +from minos.networks.event import MinosEventServer, EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService from tests.database_testcase import EventHandlerPostgresAsyncTestCase @pytest.fixture() def config(): return MinosConfig(path='./tests/test_config.yaml') -""" + @pytest.fixture() def services(config): - return [EventHandlerDatabaseInitializer(config=config), MinosEventServer(conf=config)] -""" + return [EventHandlerDatabaseInitializer(config=config), MinosEventServer(conf=config), MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config)] + class AggregateTest(Aggregate): test: int @@ -57,6 +58,35 @@ async def test_event_queue_add(self): assert affected_rows == 1 assert id > 0 + async def test_get_event_handler(self): + model = AggregateTest(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) + m = MinosEventHandlerPeriodicService(conf=self._broker_config(), interval=0.5) + + cls = m.get_event_handler(topic="TicketAdded") + result = await cls(topic="TicketAdded", event=event_instance) + + assert result == 'request_added' + + async def test_event_queue_checker(self): + database = await self._database() + async with database as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM event_queue") + records = await cur.fetchone() + + assert records[0] > 0 + + m = MinosEventHandlerPeriodicService(conf=self._broker_config(), interval=0.5) + await m.event_queue_checker() + + database = await self._database() + async with database as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM event_queue") + records = await cur.fetchone() + + assert records[0] == 0 async def test_producer_kafka(loop): basic_config( @@ -105,4 +135,3 @@ async def test_event_handle_message(config, loop): await m.handle_message(consumer) - From e3a4c280a82e399df48fcf23af649ad2f465073d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 08:11:32 +0200 Subject: [PATCH 097/239] REFS #66 * Add docstring. --- minos/networks/__init__.py | 7 +++++++ minos/networks/exceptions.py | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 7d64112b..b6d076c1 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -1,3 +1,10 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" __version__ = "0.0.1.1-alpha" from .broker import ( diff --git a/minos/networks/exceptions.py b/minos/networks/exceptions.py index 14a1b797..63f2ea7f 100644 --- a/minos/networks/exceptions.py +++ b/minos/networks/exceptions.py @@ -1,7 +1,14 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" from minos.common import ( MinosException, ) class MinosNetworkException(MinosException): - pass + """Base network exception.""" From 1e8a1441efb2a01786044632471f03d30fe5dc9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 09:08:17 +0200 Subject: [PATCH 098/239] REFS #66 * Minos coding style improvements. --- minos/networks/__init__.py | 2 +- minos/networks/broker/__init__.py | 2 +- minos/networks/broker/abc.py | 18 +++++-- minos/networks/broker/command.py | 9 +++- minos/networks/broker/dispatcher.py | 47 ++++++++++++++----- minos/networks/broker/event.py | 22 ++++++--- tests/{test_config.yaml => test_config.yml} | 0 tests/test_event_manager.py | 2 +- .../test_networks/test_broker/test_command.py | 27 +++++------ .../test_broker/test_dispatcher.py | 14 +++--- .../test_networks/test_broker/test_events.py | 32 +++++-------- tests/utils.py | 9 ++++ ...test_config.yaml => wrong_test_config.yml} | 0 13 files changed, 113 insertions(+), 71 deletions(-) rename tests/{test_config.yaml => test_config.yml} (100%) rename tests/{wrong_test_config.yaml => wrong_test_config.yml} (100%) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index b6d076c1..499412e7 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -8,7 +8,7 @@ __version__ = "0.0.1.1-alpha" from .broker import ( - EventBrokerQueueDispatcher, + MinosQueueDispatcherService, MinosCommandBroker, MinosEventBroker, MinosQueueDispatcher, diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py index 31413e47..ee3555fa 100644 --- a/minos/networks/broker/__init__.py +++ b/minos/networks/broker/__init__.py @@ -9,7 +9,7 @@ MinosCommandBroker, ) from .dispatcher import ( - EventBrokerQueueDispatcher, + MinosQueueDispatcherService, MinosQueueDispatcher, ) from .event import ( diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index 9f6ef002..95dc860b 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,6 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ +from abc import ABC from typing import ( NoReturn, ) @@ -41,12 +42,21 @@ async def _setup(self) -> NoReturn: await self.broker_table_creation() async def broker_table_creation(self): + """TODO + + :return:TODO + """ async with self._connection() as connect: async with connect.cursor() as cur: await cur.execute( - 'CREATE TABLE IF NOT EXISTS "producer_queue" ("id" BIGSERIAL NOT NULL PRIMARY KEY, ' - '"topic" VARCHAR(255) NOT NULL, "model" BYTEA NOT NULL, "retry" INTEGER NOT NULL, ' - '"action" VARCHAR(255) NOT NULL, "creation_date" TIMESTAMP NOT NULL, "update_date" TIMESTAMP NOT NULL);' + 'CREATE TABLE IF NOT EXISTS "producer_queue" (' + '"id" BIGSERIAL NOT NULL PRIMARY KEY, ' + '"topic" VARCHAR(255) NOT NULL, ' + '"model" BYTEA NOT NULL, ' + '"retry" INTEGER NOT NULL, ' + '"action" VARCHAR(255) NOT NULL, ' + '"creation_date" TIMESTAMP NOT NULL, ' + '"update_date" TIMESTAMP NOT NULL);' ) def _connection(self): @@ -55,7 +65,7 @@ def _connection(self): ) -class MinosBroker(MinosBaseBroker, MinosBrokerSetup): +class MinosBroker(MinosBaseBroker, MinosBrokerSetup, ABC): """TODO""" def __init__(self, topic: str, *args, **kwargs): diff --git a/minos/networks/broker/command.py b/minos/networks/broker/command.py index 9799effc..541ebc07 100644 --- a/minos/networks/broker/command.py +++ b/minos/networks/broker/command.py @@ -51,13 +51,20 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi return cls(*args, **config.commands.queue._asdict(), **kwargs) async def send(self, items: list[Aggregate]) -> NoReturn: + """Send a list of ``Aggregate`` instances. + + :param items: A list of aggregates. + :return: This method does not return anything. + """ event_instance = Command(self.topic, items, self.reply_on) bin_data = event_instance.avro_bytes async with self._connection() as connect: async with connect.cursor() as cur: await cur.execute( - "INSERT INTO producer_queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", + "INSERT INTO producer_queue (" + "topic, model, retry, action, creation_date, update_date" + ") VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), ) diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py index c206cd29..04612fc8 100644 --- a/minos/networks/broker/dispatcher.py +++ b/minos/networks/broker/dispatcher.py @@ -10,7 +10,8 @@ ) from typing import ( - Optional, + NamedTuple, + Optional, NoReturn, ) from aiokafka import ( @@ -28,7 +29,7 @@ ) -class EventBrokerQueueDispatcher(PeriodicService): +class MinosQueueDispatcherService(PeriodicService): """TODO""" def __init__(self, config: MinosConfig = None, **kwargs): @@ -36,11 +37,19 @@ def __init__(self, config: MinosConfig = None, **kwargs): self.dispatcher = MinosQueueDispatcher.from_config(config=config) async def callback(self): + """TODO + + :return:TODO + """ await self.dispatcher.dispatch() class MinosQueueDispatcher(MinosBrokerSetup): - def __init__(self, *args, queue=None, broker, **kwargs): + """TODO""" + + # noinspection PyUnresolvedReferences + def __init__(self, *args, queue: NamedTuple, broker, **kwargs): + # noinspection PyProtectedMember super().__init__(*args, **queue._asdict(), **kwargs) self.retry = queue.retry self.records = queue.records @@ -61,40 +70,54 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi # noinspection PyProtectedMember return cls(*args, **config.events._asdict(), **kwargs) - async def dispatch(self): + async def dispatch(self) -> NoReturn: + """TODO + + :return: TODO + """ async with self._connection() as connect: async with connect.cursor() as cur: + # noinspection SqlRedundantOrderingDirection await cur.execute( "SELECT * FROM producer_queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;" % (self.retry, self.records), ) async for row in cur: - sent_to_kafka = False + published = False + # noinspection PyBroadException try: - sent_to_kafka = await self.send_to_kafka(topic=row[1], message=row[2]) - if sent_to_kafka: + published = await self.publish(topic=row[1], message=row[2]) + if published: # Delete from database If the event was sent successfully to Kafka. async with connect.cursor() as cur2: await cur2.execute("DELETE FROM producer_queue WHERE id=%d;" % row[0]) - except: - sent_to_kafka = False + except Exception: + published = False finally: - if not sent_to_kafka: + if not published: # Update queue retry column. Increase by 1. async with connect.cursor() as cur3: await cur3.execute("UPDATE producer_queue SET retry = retry + 1 WHERE id=%d;" % row[0]) - async def send_to_kafka(self, topic: str, message: bytes): - flag = False + async def publish(self, topic: str, message: bytes) -> bool: + """ TODO + + :param topic:TODO + :param message: TODO + :return: TODO + """ producer = AIOKafkaProducer( bootstrap_servers="{host}:{port}".format(host=self.broker.host, port=self.broker.port) ) # Get cluster layout and initial topic/partition leadership information await producer.start() + # noinspection PyBroadException try: # Produce message await producer.send_and_wait(topic, message) flag = True + except Exception: + flag = False finally: # Wait for all pending messages to be delivered or expire. await producer.stop() diff --git a/minos/networks/broker/event.py b/minos/networks/broker/event.py index cc180204..aa41da81 100644 --- a/minos/networks/broker/event.py +++ b/minos/networks/broker/event.py @@ -1,9 +1,10 @@ -# Copyright (C) 2020 Clariteia SL -# -# This file is part of minos framework. -# -# Minos framework can not be copied and/or distributed without the express -# permission of Clariteia SL. +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" from __future__ import ( annotations, ) @@ -46,13 +47,20 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi return cls(*args, **config.events.queue._asdict(), **kwargs) async def send(self, items: list[Aggregate]) -> NoReturn: + """Send a list of ``Aggregate`` instances. + + :param items: A list of aggregates. + :return: This method does not return anything. + """ event_instance = Event(self.topic, items) bin_data = event_instance.avro_bytes async with self._connection() as connect: async with connect.cursor() as cur: await cur.execute( - "INSERT INTO producer_queue (topic, model, retry, action, creation_date, update_date) VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", + "INSERT INTO producer_queue (" + "topic, model, retry, action, creation_date, update_date" + ") VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), ) diff --git a/tests/test_config.yaml b/tests/test_config.yml similarity index 100% rename from tests/test_config.yaml rename to tests/test_config.yml diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 4df37c27..f0155890 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -14,7 +14,7 @@ # # @pytest.fixture() # def config(): -# return MinosConfig(path='./tests/test_config.yaml') +# return MinosConfig(path='./tests/test_config.yml') # # # @pytest.fixture() diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 7b53b2ea..7f266d00 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -9,7 +9,6 @@ import aiopg from minos.common import ( - Aggregate, MinosConfig, ) from minos.common.testing import ( @@ -20,24 +19,20 @@ MinosQueueDispatcher, ) from tests.utils import ( - BASE_PATH, + BASE_PATH, NaiveAggregate, ) -class AggregateTest(Aggregate): - test: int - - class TestMinosCommandBroker(PostgresAsyncTestCase): - CONFIG_FILE_PATH = BASE_PATH / "test_config.yaml" + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" async def test_commands_broker_insertion(self): broker = MinosCommandBroker.from_config("CommandBroker", config=self.config, reply_on="test_reply_on") await broker.setup() - a = AggregateTest(test_id=1, test=2, id=1, version=1) + item = NaiveAggregate(test_id=1, test=2, id=1, version=1) - affected_rows, queue_id = await broker.send_one(a) + affected_rows, queue_id = await broker.send_one(item) assert affected_rows == 1 assert queue_id > 0 @@ -45,10 +40,10 @@ async def test_if_commands_was_deleted(self): broker = MinosCommandBroker.from_config("CommandBroker-Delete", config=self.config, reply_on="test_reply_on") await broker.setup() - a = AggregateTest(test_id=1, test=2, id=1, version=1) + item = NaiveAggregate(test_id=1, test=2, id=1, version=1) - affected_rows_1, queue_id_1 = await broker.send_one(a) - affected_rows_2, queue_id_2 = await broker.send_one(a) + affected_rows_1, queue_id_1 = await broker.send_one(item) + affected_rows_2, queue_id_2 = await broker.send_one(item) await MinosQueueDispatcher.from_config(config=self.config).dispatch() @@ -67,13 +62,13 @@ async def test_if_commands_retry_was_incremented(self): broker = MinosCommandBroker.from_config("CommandBroker-Delete", config=self.config, reply_on="test_reply_on") await broker.setup() - a = AggregateTest(test_id=1, test=2, id=1, version=1) + item = NaiveAggregate(test_id=1, test=2, id=1, version=1) - affected_rows_1, queue_id_1 = await broker.send_one(a) - affected_rows_2, queue_id_2 = await broker.send_one(a) + affected_rows_1, queue_id_1 = await broker.send_one(item) + affected_rows_2, queue_id_2 = await broker.send_one(item) config = MinosConfig( - path=BASE_PATH / "wrong_test_config.yaml", events_queue_database="test_db", events_queue_user="test_user" + path=BASE_PATH / "wrong_test_config.yml", events_queue_database="test_db", events_queue_user="test_user" ) await MinosQueueDispatcher.from_config(config=config).dispatch() diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py index de9abbbf..8a15dbf2 100644 --- a/tests/test_networks/test_broker/test_dispatcher.py +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -1,8 +1,5 @@ import unittest -from minos.common import ( - Aggregate, -) from minos.common.testing import ( PostgresAsyncTestCase, ) @@ -14,17 +11,18 @@ ) -class AggregateTest(Aggregate): - test: int - - class TestQueueDispatcher(PostgresAsyncTestCase): - CONFIG_FILE_PATH = BASE_PATH / "test_config.yaml" + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" def test_from_config(self): dispatcher = MinosQueueDispatcher.from_config(config=self.config) self.assertIsInstance(dispatcher, MinosQueueDispatcher) + async def test_send_to_kafka_ok(self): + dispatcher = MinosQueueDispatcher.from_config(config=self.config) + response = await dispatcher.publish(topic="TestKafkaSend", message=bytes()) + assert response is True + if __name__ == "__main__": unittest.main() diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index 3b389c05..f3d2486d 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -2,27 +2,24 @@ import aiopg from minos.common import ( - Aggregate, MinosConfig, ) from minos.common.testing import ( PostgresAsyncTestCase, ) + from minos.networks import ( MinosEventBroker, MinosQueueDispatcher, ) from tests.utils import ( BASE_PATH, + NaiveAggregate, ) -class AggregateTest(Aggregate): - test: int - - class TestMinosEventBroker(PostgresAsyncTestCase): - CONFIG_FILE_PATH = BASE_PATH / "test_config.yaml" + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" async def test_if_queue_table_exists(self): broker = MinosEventBroker.from_config("EventBroker", config=self.config) @@ -39,17 +36,12 @@ async def test_if_queue_table_exists(self): assert ret == [(1,)] - async def test_send_to_kafka_ok(self): - dispatcher = MinosQueueDispatcher.from_config(config=self.config) - response = await dispatcher.send_to_kafka(topic="TestKafkaSend", message=bytes()) - assert response is True - async def test_events_broker_insertion(self): broker = MinosEventBroker.from_config("EventBroker", config=self.config) await broker.setup() - a = AggregateTest(test_id=1, test=2, id=1, version=1) - affected_rows, queue_id = await broker.send_one(a) + item = NaiveAggregate(test_id=1, test=2, id=1, version=1) + affected_rows, queue_id = await broker.send_one(item) assert affected_rows == 1 assert queue_id > 0 @@ -58,9 +50,9 @@ async def test_if_events_was_deleted(self): broker = MinosEventBroker.from_config("EventBroker-Delete", config=self.config) await broker.setup() - a = AggregateTest(test_id=1, test=2, id=1, version=1) - affected_rows_1, queue_id_1 = await broker.send_one(a) - affected_rows_2, queue_id_2 = await broker.send_one(a) + item = NaiveAggregate(test_id=1, test=2, id=1, version=1) + affected_rows_1, queue_id_1 = await broker.send_one(item) + affected_rows_2, queue_id_2 = await broker.send_one(item) await MinosQueueDispatcher.from_config(config=self.config).dispatch() @@ -79,13 +71,13 @@ async def test_if_events_retry_was_incremented(self): broker = MinosEventBroker.from_config("EventBroker-Delete", config=self.config) await broker.setup() - a = AggregateTest(test_id=1, test=2, id=1, version=1) + item = NaiveAggregate(test_id=1, test=2, id=1, version=1) - affected_rows_1, queue_id_1 = await broker.send_one(a) - affected_rows_2, queue_id_2 = await broker.send_one(a) + affected_rows_1, queue_id_1 = await broker.send_one(item) + affected_rows_2, queue_id_2 = await broker.send_one(item) config = MinosConfig( - path=BASE_PATH / "wrong_test_config.yaml", events_queue_database="test_db", events_queue_user="test_user" + path=BASE_PATH / "wrong_test_config.yml", events_queue_database="test_db", events_queue_user="test_user" ) await MinosQueueDispatcher.from_config(config=config).dispatch() diff --git a/tests/utils.py b/tests/utils.py index e41317a2..5411b64b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -9,4 +9,13 @@ Path, ) +from minos.common import ( + Aggregate, +) + BASE_PATH = Path(__file__).parent + + +class NaiveAggregate(Aggregate): + """Naive aggregate class to be used for testing purposes.""" + test: int diff --git a/tests/wrong_test_config.yaml b/tests/wrong_test_config.yml similarity index 100% rename from tests/wrong_test_config.yaml rename to tests/wrong_test_config.yml From 31763a434cd436dd78caafd6a375d720409613a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 09:14:22 +0200 Subject: [PATCH 099/239] REFS #66 * Move logic to store events and commands to the base class. --- minos/networks/broker/abc.py | 17 +++++++++++++++++ minos/networks/broker/command.py | 19 ++----------------- minos/networks/broker/event.py | 22 +++------------------- 3 files changed, 22 insertions(+), 36 deletions(-) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index 95dc860b..f8ba6bf3 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -6,6 +6,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ from abc import ABC +from datetime import datetime from typing import ( NoReturn, ) @@ -67,7 +68,23 @@ def _connection(self): class MinosBroker(MinosBaseBroker, MinosBrokerSetup, ABC): """TODO""" + ACTION: str def __init__(self, topic: str, *args, **kwargs): MinosBaseBroker.__init__(self, topic) MinosBrokerSetup.__init__(self, *args, **kwargs) + + async def _send_bytes(self, topic: str, raw: bytes) -> (int, int): + async with self._connection() as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO producer_queue (" + "topic, model, retry, action, creation_date, update_date" + ") VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", + (topic, raw, 0, self.ACTION, datetime.now(), datetime.now()), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + return affected_rows, queue_id[0] diff --git a/minos/networks/broker/command.py b/minos/networks/broker/command.py index 541ebc07..c7ce97cb 100644 --- a/minos/networks/broker/command.py +++ b/minos/networks/broker/command.py @@ -9,7 +9,6 @@ annotations, ) -import datetime from typing import ( NoReturn, Optional, @@ -56,19 +55,5 @@ async def send(self, items: list[Aggregate]) -> NoReturn: :param items: A list of aggregates. :return: This method does not return anything. """ - event_instance = Command(self.topic, items, self.reply_on) - bin_data = event_instance.avro_bytes - - async with self._connection() as connect: - async with connect.cursor() as cur: - await cur.execute( - "INSERT INTO producer_queue (" - "topic, model, retry, action, creation_date, update_date" - ") VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", - (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), - ) - - queue_id = await cur.fetchone() - affected_rows = cur.rowcount - - return affected_rows, queue_id[0] + command = Command(self.topic, items, self.reply_on) + return await self._send_bytes(command.topic, command.avro_bytes) diff --git a/minos/networks/broker/event.py b/minos/networks/broker/event.py index aa41da81..f3b28a28 100644 --- a/minos/networks/broker/event.py +++ b/minos/networks/broker/event.py @@ -9,9 +9,7 @@ annotations, ) -import datetime from typing import ( - NoReturn, Optional, ) @@ -46,25 +44,11 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi # noinspection PyProtectedMember return cls(*args, **config.events.queue._asdict(), **kwargs) - async def send(self, items: list[Aggregate]) -> NoReturn: + async def send(self, items: list[Aggregate]) -> (int, int): """Send a list of ``Aggregate`` instances. :param items: A list of aggregates. :return: This method does not return anything. """ - event_instance = Event(self.topic, items) - bin_data = event_instance.avro_bytes - - async with self._connection() as connect: - async with connect.cursor() as cur: - await cur.execute( - "INSERT INTO producer_queue (" - "topic, model, retry, action, creation_date, update_date" - ") VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", - (event_instance.topic, bin_data, 0, self.ACTION, datetime.datetime.now(), datetime.datetime.now(),), - ) - - queue_id = await cur.fetchone() - affected_rows = cur.rowcount - - return affected_rows, queue_id[0] + event = Event(self.topic, items) + return await self._send_bytes(event.topic, event.avro_bytes) From c947a8dfcd11938d6aa4638d0133f5540fcf1273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 09:22:35 +0200 Subject: [PATCH 100/239] REFS #66 * Add missing docstring and minor improvements. --- minos/networks/broker/abc.py | 13 ++++----- minos/networks/broker/command.py | 2 +- minos/networks/broker/dispatcher.py | 42 +++++++++++++---------------- minos/networks/broker/event.py | 2 +- 4 files changed, 26 insertions(+), 33 deletions(-) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index f8ba6bf3..ec07b0ec 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -19,7 +19,7 @@ class MinosBrokerSetup(MinosSetup): - """TODO""" + """Minos Broker Setup Class""" def __init__( self, @@ -40,13 +40,9 @@ def __init__( self.password = password async def _setup(self) -> NoReturn: - await self.broker_table_creation() + await self._create_broker_table() - async def broker_table_creation(self): - """TODO - - :return:TODO - """ + async def _create_broker_table(self) -> NoReturn: async with self._connection() as connect: async with connect.cursor() as cur: await cur.execute( @@ -67,7 +63,8 @@ def _connection(self): class MinosBroker(MinosBaseBroker, MinosBrokerSetup, ABC): - """TODO""" + """Minos Broker Class.""" + ACTION: str def __init__(self, topic: str, *args, **kwargs): diff --git a/minos/networks/broker/command.py b/minos/networks/broker/command.py index c7ce97cb..4cd1bd98 100644 --- a/minos/networks/broker/command.py +++ b/minos/networks/broker/command.py @@ -26,7 +26,7 @@ class MinosCommandBroker(MinosBroker): - """TODO""" + """Minos Command Broker Class.""" ACTION = "command" diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py index 04612fc8..a1fbc452 100644 --- a/minos/networks/broker/dispatcher.py +++ b/minos/networks/broker/dispatcher.py @@ -30,22 +30,22 @@ class MinosQueueDispatcherService(PeriodicService): - """TODO""" + """Minos QueueDispatcherService class.""" def __init__(self, config: MinosConfig = None, **kwargs): super().__init__(**kwargs) self.dispatcher = MinosQueueDispatcher.from_config(config=config) - async def callback(self): - """TODO + async def callback(self) -> None: + """Method to be called periodically by the internal ``aiomisc`` logic. - :return:TODO + :return:This method does not return anything. """ await self.dispatcher.dispatch() class MinosQueueDispatcher(MinosBrokerSetup): - """TODO""" + """Minos Queue Dispatcher Class.""" # noinspection PyUnresolvedReferences def __init__(self, *args, queue: NamedTuple, broker, **kwargs): @@ -71,9 +71,9 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi return cls(*args, **config.events._asdict(), **kwargs) async def dispatch(self) -> NoReturn: - """TODO + """Dispatch the items in the publishing queue. - :return: TODO + :return: This method does not return anything. """ async with self._connection() as connect: async with connect.cursor() as cur: @@ -83,32 +83,28 @@ async def dispatch(self) -> NoReturn: % (self.retry, self.records), ) async for row in cur: - published = False # noinspection PyBroadException try: published = await self.publish(topic=row[1], message=row[2]) - if published: - # Delete from database If the event was sent successfully to Kafka. - async with connect.cursor() as cur2: - await cur2.execute("DELETE FROM producer_queue WHERE id=%d;" % row[0]) except Exception: published = False finally: - if not published: - # Update queue retry column. Increase by 1. - async with connect.cursor() as cur3: - await cur3.execute("UPDATE producer_queue SET retry = retry + 1 WHERE id=%d;" % row[0]) + async with connect.cursor() as cur2: + if published: + # Delete from database If the event was sent successfully to Kafka. + await cur2.execute("DELETE FROM producer_queue WHERE id=%d;" % row[0]) + else: + # Update queue retry column. Increase by 1. + await cur2.execute("UPDATE producer_queue SET retry = retry + 1 WHERE id=%d;" % row[0]) async def publish(self, topic: str, message: bytes) -> bool: - """ TODO + """ Publish a new item in in the broker (kafka). - :param topic:TODO - :param message: TODO - :return: TODO + :param topic: The topic in which the message will be published. + :param message: The message to be published. + :return: A boolean flag, ``True`` when the message is properly published or ``False`` otherwise. """ - producer = AIOKafkaProducer( - bootstrap_servers="{host}:{port}".format(host=self.broker.host, port=self.broker.port) - ) + producer = AIOKafkaProducer(bootstrap_servers=f"{self.host}:{self.port}") # Get cluster layout and initial topic/partition leadership information await producer.start() # noinspection PyBroadException diff --git a/minos/networks/broker/event.py b/minos/networks/broker/event.py index f3b28a28..64de7705 100644 --- a/minos/networks/broker/event.py +++ b/minos/networks/broker/event.py @@ -25,7 +25,7 @@ class MinosEventBroker(MinosBroker): - """TODO""" + """Minos Event broker class.""" ACTION = "event" From 412879c62ab6510ceed03cb85a05bc9dd5c7cfb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 09:32:10 +0200 Subject: [PATCH 101/239] REFS #66 * Run `black` + `isort`. --- minos/networks/__init__.py | 2 +- minos/networks/broker/__init__.py | 2 +- minos/networks/broker/abc.py | 8 ++++++-- minos/networks/broker/dispatcher.py | 3 ++- tests/utils.py | 1 + 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 499412e7..414d301c 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -8,10 +8,10 @@ __version__ = "0.0.1.1-alpha" from .broker import ( - MinosQueueDispatcherService, MinosCommandBroker, MinosEventBroker, MinosQueueDispatcher, + MinosQueueDispatcherService, ) from .exceptions import ( MinosNetworkException, diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py index ee3555fa..0782c44a 100644 --- a/minos/networks/broker/__init__.py +++ b/minos/networks/broker/__init__.py @@ -9,8 +9,8 @@ MinosCommandBroker, ) from .dispatcher import ( - MinosQueueDispatcherService, MinosQueueDispatcher, + MinosQueueDispatcherService, ) from .event import ( MinosEventBroker, diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index ec07b0ec..d83c2082 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,8 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ABC -from datetime import datetime +from abc import ( + ABC, +) +from datetime import ( + datetime, +) from typing import ( NoReturn, ) diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py index a1fbc452..01ed58dd 100644 --- a/minos/networks/broker/dispatcher.py +++ b/minos/networks/broker/dispatcher.py @@ -11,7 +11,8 @@ from typing import ( NamedTuple, - Optional, NoReturn, + NoReturn, + Optional, ) from aiokafka import ( diff --git a/tests/utils.py b/tests/utils.py index 5411b64b..60075770 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -18,4 +18,5 @@ class NaiveAggregate(Aggregate): """Naive aggregate class to be used for testing purposes.""" + test: int From 54592cd1809d04c629c87f80b765376cfb94f6da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 09:35:13 +0200 Subject: [PATCH 102/239] REFS #66 * Fix stupid bug created during refactor. --- minos/networks/broker/dispatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py index 01ed58dd..0ec82f2c 100644 --- a/minos/networks/broker/dispatcher.py +++ b/minos/networks/broker/dispatcher.py @@ -105,7 +105,7 @@ async def publish(self, topic: str, message: bytes) -> bool: :param message: The message to be published. :return: A boolean flag, ``True`` when the message is properly published or ``False`` otherwise. """ - producer = AIOKafkaProducer(bootstrap_servers=f"{self.host}:{self.port}") + producer = AIOKafkaProducer(bootstrap_servers=f"{self.broker.host}:{self.broker.port}") # Get cluster layout and initial topic/partition leadership information await producer.start() # noinspection PyBroadException From 9342b8702e7b0f191fce4ebb3a383c9e2bcff4a7 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 07:35:23 +0000 Subject: [PATCH 103/239] Restyled by black --- minos/networks/__init__.py | 4 +--- minos/networks/broker/__init__.py | 8 ++------ minos/networks/broker/abc.py | 12 +++-------- minos/networks/broker/command.py | 8 ++------ minos/networks/broker/dispatcher.py | 20 +++++-------------- minos/networks/broker/event.py | 12 +++-------- minos/networks/exceptions.py | 4 +--- .../test_networks/test_broker/test_command.py | 11 ++++------ .../test_broker/test_dispatcher.py | 12 +++-------- .../test_networks/test_broker/test_events.py | 8 ++------ tests/utils.py | 8 ++------ 11 files changed, 28 insertions(+), 79 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 414d301c..37295507 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,6 +13,4 @@ MinosQueueDispatcher, MinosQueueDispatcherService, ) -from .exceptions import ( - MinosNetworkException, -) +from .exceptions import MinosNetworkException diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py index 0782c44a..2fcdaa3a 100644 --- a/minos/networks/broker/__init__.py +++ b/minos/networks/broker/__init__.py @@ -5,13 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .command import ( - MinosCommandBroker, -) +from .command import MinosCommandBroker from .dispatcher import ( MinosQueueDispatcher, MinosQueueDispatcherService, ) -from .event import ( - MinosEventBroker, -) +from .event import MinosEventBroker diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index d83c2082..4bacb2a1 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,15 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ( - ABC, -) -from datetime import ( - datetime, -) -from typing import ( - NoReturn, -) +from abc import ABC +from datetime import datetime +from typing import NoReturn import aiopg from minos.common import ( diff --git a/minos/networks/broker/command.py b/minos/networks/broker/command.py index 4cd1bd98..8b457ff0 100644 --- a/minos/networks/broker/command.py +++ b/minos/networks/broker/command.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NoReturn, @@ -20,9 +18,7 @@ MinosConfig, ) -from .abc import ( - MinosBroker, -) +from .abc import MinosBroker class MinosCommandBroker(MinosBroker): diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py index 0ec82f2c..93747b60 100644 --- a/minos/networks/broker/dispatcher.py +++ b/minos/networks/broker/dispatcher.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NamedTuple, @@ -15,19 +13,11 @@ Optional, ) -from aiokafka import ( - AIOKafkaProducer, -) -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) +from aiokafka import AIOKafkaProducer +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig -from .abc import ( - MinosBrokerSetup, -) +from .abc import MinosBrokerSetup class MinosQueueDispatcherService(PeriodicService): diff --git a/minos/networks/broker/event.py b/minos/networks/broker/event.py index 64de7705..45fc55a1 100644 --- a/minos/networks/broker/event.py +++ b/minos/networks/broker/event.py @@ -5,13 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations -from typing import ( - Optional, -) +from typing import Optional from minos.common import ( Aggregate, @@ -19,9 +15,7 @@ MinosConfig, ) -from .abc import ( - MinosBroker, -) +from .abc import MinosBroker class MinosEventBroker(MinosBroker): diff --git a/minos/networks/exceptions.py b/minos/networks/exceptions.py index 63f2ea7f..2f5f6f41 100644 --- a/minos/networks/exceptions.py +++ b/minos/networks/exceptions.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from minos.common import ( - MinosException, -) +from minos.common import MinosException class MinosNetworkException(MinosException): diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 7f266d00..a5f5b3af 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -8,18 +8,15 @@ import unittest import aiopg -from minos.common import ( - MinosConfig, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import MinosConfig +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, ) from tests.utils import ( - BASE_PATH, NaiveAggregate, + BASE_PATH, + NaiveAggregate, ) diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py index 8a15dbf2..6f33f35b 100644 --- a/tests/test_networks/test_broker/test_dispatcher.py +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -1,14 +1,8 @@ import unittest -from minos.common.testing import ( - PostgresAsyncTestCase, -) -from minos.networks import ( - MinosQueueDispatcher, -) -from tests.utils import ( - BASE_PATH, -) +from minos.common.testing import PostgresAsyncTestCase +from minos.networks import MinosQueueDispatcher +from tests.utils import BASE_PATH class TestQueueDispatcher(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index f3d2486d..5babd88b 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -1,12 +1,8 @@ import unittest import aiopg -from minos.common import ( - MinosConfig, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import MinosConfig +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosEventBroker, diff --git a/tests/utils.py b/tests/utils.py index 60075770..68dfd7c7 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,13 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from pathlib import ( - Path, -) +from pathlib import Path -from minos.common import ( - Aggregate, -) +from minos.common import Aggregate BASE_PATH = Path(__file__).parent From e4130158cd8e3cd02f272fb8435dfe6b62849cf7 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 07:35:24 +0000 Subject: [PATCH 104/239] Restyled by isort --- minos/networks/__init__.py | 4 +++- minos/networks/broker/__init__.py | 8 ++++++-- minos/networks/broker/abc.py | 12 ++++++++--- minos/networks/broker/command.py | 8 ++++++-- minos/networks/broker/dispatcher.py | 20 ++++++++++++++----- minos/networks/broker/event.py | 12 ++++++++--- minos/networks/exceptions.py | 4 +++- .../test_networks/test_broker/test_command.py | 8 ++++++-- .../test_broker/test_dispatcher.py | 12 ++++++++--- .../test_networks/test_broker/test_events.py | 9 ++++++--- tests/utils.py | 8 ++++++-- 11 files changed, 78 insertions(+), 27 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 37295507..414d301c 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,4 +13,6 @@ MinosQueueDispatcher, MinosQueueDispatcherService, ) -from .exceptions import MinosNetworkException +from .exceptions import ( + MinosNetworkException, +) diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py index 2fcdaa3a..0782c44a 100644 --- a/minos/networks/broker/__init__.py +++ b/minos/networks/broker/__init__.py @@ -5,9 +5,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .command import MinosCommandBroker +from .command import ( + MinosCommandBroker, +) from .dispatcher import ( MinosQueueDispatcher, MinosQueueDispatcherService, ) -from .event import MinosEventBroker +from .event import ( + MinosEventBroker, +) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index 4bacb2a1..d83c2082 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,9 +5,15 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ABC -from datetime import datetime -from typing import NoReturn +from abc import ( + ABC, +) +from datetime import ( + datetime, +) +from typing import ( + NoReturn, +) import aiopg from minos.common import ( diff --git a/minos/networks/broker/command.py b/minos/networks/broker/command.py index 8b457ff0..4cd1bd98 100644 --- a/minos/networks/broker/command.py +++ b/minos/networks/broker/command.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( NoReturn, @@ -18,7 +20,9 @@ MinosConfig, ) -from .abc import MinosBroker +from .abc import ( + MinosBroker, +) class MinosCommandBroker(MinosBroker): diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py index 93747b60..0ec82f2c 100644 --- a/minos/networks/broker/dispatcher.py +++ b/minos/networks/broker/dispatcher.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( NamedTuple, @@ -13,11 +15,19 @@ Optional, ) -from aiokafka import AIOKafkaProducer -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig +from aiokafka import ( + AIOKafkaProducer, +) +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) -from .abc import MinosBrokerSetup +from .abc import ( + MinosBrokerSetup, +) class MinosQueueDispatcherService(PeriodicService): diff --git a/minos/networks/broker/event.py b/minos/networks/broker/event.py index 45fc55a1..64de7705 100644 --- a/minos/networks/broker/event.py +++ b/minos/networks/broker/event.py @@ -5,9 +5,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) -from typing import Optional +from typing import ( + Optional, +) from minos.common import ( Aggregate, @@ -15,7 +19,9 @@ MinosConfig, ) -from .abc import MinosBroker +from .abc import ( + MinosBroker, +) class MinosEventBroker(MinosBroker): diff --git a/minos/networks/exceptions.py b/minos/networks/exceptions.py index 2f5f6f41..63f2ea7f 100644 --- a/minos/networks/exceptions.py +++ b/minos/networks/exceptions.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from minos.common import MinosException +from minos.common import ( + MinosException, +) class MinosNetworkException(MinosException): diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index a5f5b3af..3fa800af 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -8,8 +8,12 @@ import unittest import aiopg -from minos.common import MinosConfig -from minos.common.testing import PostgresAsyncTestCase +from minos.common import ( + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py index 6f33f35b..8a15dbf2 100644 --- a/tests/test_networks/test_broker/test_dispatcher.py +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -1,8 +1,14 @@ import unittest -from minos.common.testing import PostgresAsyncTestCase -from minos.networks import MinosQueueDispatcher -from tests.utils import BASE_PATH +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosQueueDispatcher, +) +from tests.utils import ( + BASE_PATH, +) class TestQueueDispatcher(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index 5babd88b..0519c064 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -1,9 +1,12 @@ import unittest import aiopg -from minos.common import MinosConfig -from minos.common.testing import PostgresAsyncTestCase - +from minos.common import ( + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosEventBroker, MinosQueueDispatcher, diff --git a/tests/utils.py b/tests/utils.py index 68dfd7c7..60075770 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,9 +5,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from pathlib import Path +from pathlib import ( + Path, +) -from minos.common import Aggregate +from minos.common import ( + Aggregate, +) BASE_PATH = Path(__file__).parent From 677d7b4326de6e080bde82a8ba5ac3cf4bac59f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 09:48:28 +0200 Subject: [PATCH 105/239] REFS #51 * Create skeleton to start working on the snapshot logic. --- minos/networks/__init__.py | 4 ++ minos/networks/snapshots/__init__.py | 13 ++++++ minos/networks/snapshots/dispatchers.py | 44 +++++++++++++++++++ minos/networks/snapshots/services.py | 36 +++++++++++++++ .../test_networks/test_snapshots/__init__.py | 7 +++ .../test_snapshots/test_dispatchers.py | 22 ++++++++++ .../test_snapshots/test_services.py | 26 +++++++++++ 7 files changed, 152 insertions(+) create mode 100644 minos/networks/snapshots/__init__.py create mode 100644 minos/networks/snapshots/dispatchers.py create mode 100644 minos/networks/snapshots/services.py create mode 100644 tests/test_networks/test_snapshots/__init__.py create mode 100644 tests/test_networks/test_snapshots/test_dispatchers.py create mode 100644 tests/test_networks/test_snapshots/test_services.py diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 414d301c..fbc5982d 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -16,3 +16,7 @@ from .exceptions import ( MinosNetworkException, ) +from .snapshots import ( + MinosSnapshotDispatcher, + MinosSnapshotService, +) diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py new file mode 100644 index 00000000..7e980b04 --- /dev/null +++ b/minos/networks/snapshots/__init__.py @@ -0,0 +1,13 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from .dispatchers import ( + MinosSnapshotDispatcher, +) +from .services import ( + MinosSnapshotService, +) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py new file mode 100644 index 00000000..5c8a6a1d --- /dev/null +++ b/minos/networks/snapshots/dispatchers.py @@ -0,0 +1,44 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from __future__ import ( + annotations, +) + +from typing import ( + NoReturn, + Optional, +) + +from minos.common import ( + MinosConfig, +) + + +class MinosSnapshotDispatcher(object): + """TODO""" + + def __init__(self, *args, **kwargs): + ... + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosSnapshotDispatcher]: + """Build a new Snapshot Dispatcher from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + return None + # noinspection PyProtectedMember + return cls(*args, **kwargs) + + def dispatch(self) -> NoReturn: + """TODO""" diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py new file mode 100644 index 00000000..6402b217 --- /dev/null +++ b/minos/networks/snapshots/services.py @@ -0,0 +1,36 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from typing import ( + Any, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from minos.networks.snapshots import ( + MinosSnapshotDispatcher, +) + + +class MinosSnapshotService(PeriodicService): + """TODO""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosSnapshotDispatcher.from_config(config=config) + + async def callback(self) -> Any: + """TODO + + :return: TODO + """ + pass diff --git a/tests/test_networks/test_snapshots/__init__.py b/tests/test_networks/test_snapshots/__init__.py new file mode 100644 index 00000000..fed3716d --- /dev/null +++ b/tests/test_networks/test_snapshots/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py new file mode 100644 index 00000000..ba3515ee --- /dev/null +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -0,0 +1,22 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +import unittest + +from minos.networks import ( + MinosSnapshotDispatcher, +) + + +class TestMinosSnapshotDispatcher(unittest.TestCase): + def test_type(self): + self.assertTrue(issubclass(MinosSnapshotDispatcher, object)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py new file mode 100644 index 00000000..8055400a --- /dev/null +++ b/tests/test_networks/test_snapshots/test_services.py @@ -0,0 +1,26 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +import unittest + +from aiomisc.service.periodic import ( + PeriodicService, +) + +from minos.networks import ( + MinosSnapshotService, +) + + +class TestMinosSnapshotService(unittest.TestCase): + def test_type(self): + self.assertTrue(issubclass(MinosSnapshotService, PeriodicService)) + + +if __name__ == "__main__": + unittest.main() From 3547d2493ee149b4a8f7f2ed6b6f71eb0483abe7 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 07:49:18 +0000 Subject: [PATCH 106/239] Restyled by black --- minos/networks/__init__.py | 4 +--- minos/networks/snapshots/__init__.py | 8 ++------ minos/networks/snapshots/dispatchers.py | 8 ++------ minos/networks/snapshots/services.py | 20 ++++++------------- .../test_snapshots/test_dispatchers.py | 4 +--- .../test_snapshots/test_services.py | 8 ++------ 6 files changed, 14 insertions(+), 38 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index fbc5982d..f024b5e1 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,9 +13,7 @@ MinosQueueDispatcher, MinosQueueDispatcherService, ) -from .exceptions import ( - MinosNetworkException, -) +from .exceptions import MinosNetworkException from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotService, diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index 7e980b04..b745c474 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -5,9 +5,5 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatchers import ( - MinosSnapshotDispatcher, -) -from .services import ( - MinosSnapshotService, -) +from .dispatchers import MinosSnapshotDispatcher +from .services import MinosSnapshotService diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 5c8a6a1d..feebe3aa 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,18 +5,14 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NoReturn, Optional, ) -from minos.common import ( - MinosConfig, -) +from minos.common import MinosConfig class MinosSnapshotDispatcher(object): diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 6402b217..3ef8fbd0 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,20 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - Any, -) - -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) - -from minos.networks.snapshots import ( - MinosSnapshotDispatcher, -) +from typing import Any + +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig + +from minos.networks.snapshots import MinosSnapshotDispatcher class MinosSnapshotService(PeriodicService): diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index ba3515ee..48e113d9 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,9 +8,7 @@ import unittest -from minos.networks import ( - MinosSnapshotDispatcher, -) +from minos.networks import MinosSnapshotDispatcher class TestMinosSnapshotDispatcher(unittest.TestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 8055400a..f6ff5d04 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -8,13 +8,9 @@ import unittest -from aiomisc.service.periodic import ( - PeriodicService, -) +from aiomisc.service.periodic import PeriodicService -from minos.networks import ( - MinosSnapshotService, -) +from minos.networks import MinosSnapshotService class TestMinosSnapshotService(unittest.TestCase): From 7fbb35b9fae73b336aaeac547a2c355f17192cfd Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 07:49:19 +0000 Subject: [PATCH 107/239] Restyled by isort --- minos/networks/__init__.py | 4 +++- minos/networks/snapshots/__init__.py | 8 ++++++-- minos/networks/snapshots/dispatchers.py | 8 ++++++-- minos/networks/snapshots/services.py | 19 +++++++++++++------ .../test_snapshots/test_dispatchers.py | 4 +++- .../test_snapshots/test_services.py | 9 ++++++--- 6 files changed, 37 insertions(+), 15 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index f024b5e1..fbc5982d 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,7 +13,9 @@ MinosQueueDispatcher, MinosQueueDispatcherService, ) -from .exceptions import MinosNetworkException +from .exceptions import ( + MinosNetworkException, +) from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotService, diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index b745c474..7e980b04 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -5,5 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatchers import MinosSnapshotDispatcher -from .services import MinosSnapshotService +from .dispatchers import ( + MinosSnapshotDispatcher, +) +from .services import ( + MinosSnapshotService, +) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index feebe3aa..5c8a6a1d 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,14 +5,18 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( NoReturn, Optional, ) -from minos.common import MinosConfig +from minos.common import ( + MinosConfig, +) class MinosSnapshotDispatcher(object): diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 3ef8fbd0..a64109ad 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,12 +5,19 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import Any - -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig - -from minos.networks.snapshots import MinosSnapshotDispatcher +from typing import ( + Any, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) +from minos.networks.snapshots import ( + MinosSnapshotDispatcher, +) class MinosSnapshotService(PeriodicService): diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 48e113d9..ba3515ee 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,7 +8,9 @@ import unittest -from minos.networks import MinosSnapshotDispatcher +from minos.networks import ( + MinosSnapshotDispatcher, +) class TestMinosSnapshotDispatcher(unittest.TestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index f6ff5d04..0deeaa3b 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -8,9 +8,12 @@ import unittest -from aiomisc.service.periodic import PeriodicService - -from minos.networks import MinosSnapshotService +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.networks import ( + MinosSnapshotService, +) class TestMinosSnapshotService(unittest.TestCase): From a9b9eed7a34eefdd61d1a35cf6265427adb46783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 09:51:44 +0200 Subject: [PATCH 108/239] REFS #66 * Move the service to an independent file. --- minos/networks/broker/__init__.py | 4 +++- minos/networks/broker/dispatcher.py | 20 +--------------- minos/networks/broker/services.py | 36 +++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 20 deletions(-) create mode 100644 minos/networks/broker/services.py diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py index 0782c44a..214c5045 100644 --- a/minos/networks/broker/__init__.py +++ b/minos/networks/broker/__init__.py @@ -10,8 +10,10 @@ ) from .dispatcher import ( MinosQueueDispatcher, - MinosQueueDispatcherService, ) from .event import ( MinosEventBroker, ) +from .services import ( + MinosQueueDispatcherService, +) diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatcher.py index 0ec82f2c..1cc729f0 100644 --- a/minos/networks/broker/dispatcher.py +++ b/minos/networks/broker/dispatcher.py @@ -18,9 +18,6 @@ from aiokafka import ( AIOKafkaProducer, ) -from aiomisc.service.periodic import ( - PeriodicService, -) from minos.common import ( MinosConfig, ) @@ -30,21 +27,6 @@ ) -class MinosQueueDispatcherService(PeriodicService): - """Minos QueueDispatcherService class.""" - - def __init__(self, config: MinosConfig = None, **kwargs): - super().__init__(**kwargs) - self.dispatcher = MinosQueueDispatcher.from_config(config=config) - - async def callback(self) -> None: - """Method to be called periodically by the internal ``aiomisc`` logic. - - :return:This method does not return anything. - """ - await self.dispatcher.dispatch() - - class MinosQueueDispatcher(MinosBrokerSetup): """Minos Queue Dispatcher Class.""" @@ -99,7 +81,7 @@ async def dispatch(self) -> NoReturn: await cur2.execute("UPDATE producer_queue SET retry = retry + 1 WHERE id=%d;" % row[0]) async def publish(self, topic: str, message: bytes) -> bool: - """ Publish a new item in in the broker (kafka). + """Publish a new item in in the broker (kafka). :param topic: The topic in which the message will be published. :param message: The message to be published. diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py new file mode 100644 index 00000000..5fd064ad --- /dev/null +++ b/minos/networks/broker/services.py @@ -0,0 +1,36 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from __future__ import ( + annotations, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatcher import ( + MinosQueueDispatcher, +) + + +class MinosQueueDispatcherService(PeriodicService): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosQueueDispatcher.from_config(config=config) + + async def callback(self) -> None: + """Method to be called periodically by the internal ``aiomisc`` logic. + + :return:This method does not return anything. + """ + await self.dispatcher.dispatch() From ce25c6034fd23140c648b1d1baded94989db124f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 09:52:15 +0200 Subject: [PATCH 109/239] REFS #66 * Rename service to have a cleaner name. --- minos/networks/__init__.py | 2 +- minos/networks/broker/__init__.py | 2 +- minos/networks/broker/services.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 414d301c..01742ffe 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -11,7 +11,7 @@ MinosCommandBroker, MinosEventBroker, MinosQueueDispatcher, - MinosQueueDispatcherService, + MinosQueueService, ) from .exceptions import ( MinosNetworkException, diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py index 214c5045..511e46e0 100644 --- a/minos/networks/broker/__init__.py +++ b/minos/networks/broker/__init__.py @@ -15,5 +15,5 @@ MinosEventBroker, ) from .services import ( - MinosQueueDispatcherService, + MinosQueueService, ) diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index 5fd064ad..cf05daf6 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -21,7 +21,7 @@ ) -class MinosQueueDispatcherService(PeriodicService): +class MinosQueueService(PeriodicService): """Minos QueueDispatcherService class.""" def __init__(self, config: MinosConfig = None, **kwargs): From 70376811abfc3db283cd2b9515a5e32215f11ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 09:52:55 +0200 Subject: [PATCH 110/239] REFS #66 * Rename modules. --- minos/networks/broker/__init__.py | 6 +++--- minos/networks/broker/{command.py => commands.py} | 0 minos/networks/broker/{dispatcher.py => dispatchers.py} | 0 minos/networks/broker/{event.py => events.py} | 0 minos/networks/broker/services.py | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename minos/networks/broker/{command.py => commands.py} (100%) rename minos/networks/broker/{dispatcher.py => dispatchers.py} (100%) rename minos/networks/broker/{event.py => events.py} (100%) diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py index 511e46e0..e229ea70 100644 --- a/minos/networks/broker/__init__.py +++ b/minos/networks/broker/__init__.py @@ -5,13 +5,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .command import ( +from .commands import ( MinosCommandBroker, ) -from .dispatcher import ( +from .dispatchers import ( MinosQueueDispatcher, ) -from .event import ( +from .events import ( MinosEventBroker, ) from .services import ( diff --git a/minos/networks/broker/command.py b/minos/networks/broker/commands.py similarity index 100% rename from minos/networks/broker/command.py rename to minos/networks/broker/commands.py diff --git a/minos/networks/broker/dispatcher.py b/minos/networks/broker/dispatchers.py similarity index 100% rename from minos/networks/broker/dispatcher.py rename to minos/networks/broker/dispatchers.py diff --git a/minos/networks/broker/event.py b/minos/networks/broker/events.py similarity index 100% rename from minos/networks/broker/event.py rename to minos/networks/broker/events.py diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index cf05daf6..c5dd51b5 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -16,7 +16,7 @@ MinosConfig, ) -from .dispatcher import ( +from .dispatchers import ( MinosQueueDispatcher, ) From cfa5c893e353c7dfa948efb74cdc5f293004bf4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 09:55:15 +0200 Subject: [PATCH 111/239] REFS #51 * Replace absolute by relative import. --- minos/networks/snapshots/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index a64109ad..42074734 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -15,7 +15,7 @@ from minos.common import ( MinosConfig, ) -from minos.networks.snapshots import ( +from .dispatchers import ( MinosSnapshotDispatcher, ) From f99bf1a2252c8d8591485340f17ebd203fb40355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 09:56:07 +0200 Subject: [PATCH 112/239] REFS #51 * Minor improvements. --- minos/networks/snapshots/dispatchers.py | 2 +- minos/networks/snapshots/services.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 5c8a6a1d..af96a477 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -40,5 +40,5 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi # noinspection PyProtectedMember return cls(*args, **kwargs) - def dispatch(self) -> NoReturn: + async def dispatch(self) -> NoReturn: """TODO""" diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 42074734..0163e99e 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -32,4 +32,4 @@ async def callback(self) -> Any: :return: TODO """ - pass + await self.dispatcher.dispatch() From bb4bc68f2dac31ee4e27b1b4ed5a35186f531bdc Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 07:56:26 +0000 Subject: [PATCH 113/239] Restyled by black --- minos/networks/__init__.py | 4 +--- minos/networks/snapshots/__init__.py | 8 ++------ minos/networks/snapshots/dispatchers.py | 8 ++------ minos/networks/snapshots/services.py | 18 +++++------------- .../test_snapshots/test_dispatchers.py | 4 +--- .../test_snapshots/test_services.py | 8 ++------ 6 files changed, 13 insertions(+), 37 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index fbc5982d..f024b5e1 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,9 +13,7 @@ MinosQueueDispatcher, MinosQueueDispatcherService, ) -from .exceptions import ( - MinosNetworkException, -) +from .exceptions import MinosNetworkException from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotService, diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index 7e980b04..b745c474 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -5,9 +5,5 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatchers import ( - MinosSnapshotDispatcher, -) -from .services import ( - MinosSnapshotService, -) +from .dispatchers import MinosSnapshotDispatcher +from .services import MinosSnapshotService diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index af96a477..905461b4 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,18 +5,14 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NoReturn, Optional, ) -from minos.common import ( - MinosConfig, -) +from minos.common import MinosConfig class MinosSnapshotDispatcher(object): diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 0163e99e..13ca5bda 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,19 +5,11 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - Any, -) - -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) -from .dispatchers import ( - MinosSnapshotDispatcher, -) +from typing import Any + +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig +from .dispatchers import MinosSnapshotDispatcher class MinosSnapshotService(PeriodicService): diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index ba3515ee..48e113d9 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,9 +8,7 @@ import unittest -from minos.networks import ( - MinosSnapshotDispatcher, -) +from minos.networks import MinosSnapshotDispatcher class TestMinosSnapshotDispatcher(unittest.TestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 0deeaa3b..5601d019 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -8,12 +8,8 @@ import unittest -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.networks import ( - MinosSnapshotService, -) +from aiomisc.service.periodic import PeriodicService +from minos.networks import MinosSnapshotService class TestMinosSnapshotService(unittest.TestCase): From becacd3035d34125a16efe783a3c9115d48ada79 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 07:56:32 +0000 Subject: [PATCH 114/239] Restyled by isort --- minos/networks/__init__.py | 4 +++- minos/networks/snapshots/__init__.py | 8 ++++++-- minos/networks/snapshots/dispatchers.py | 8 ++++++-- minos/networks/snapshots/services.py | 19 ++++++++++++++----- .../test_snapshots/test_dispatchers.py | 4 +++- .../test_snapshots/test_services.py | 8 ++++++-- 6 files changed, 38 insertions(+), 13 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index f024b5e1..fbc5982d 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,7 +13,9 @@ MinosQueueDispatcher, MinosQueueDispatcherService, ) -from .exceptions import MinosNetworkException +from .exceptions import ( + MinosNetworkException, +) from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotService, diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index b745c474..7e980b04 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -5,5 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatchers import MinosSnapshotDispatcher -from .services import MinosSnapshotService +from .dispatchers import ( + MinosSnapshotDispatcher, +) +from .services import ( + MinosSnapshotService, +) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 905461b4..af96a477 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,14 +5,18 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( NoReturn, Optional, ) -from minos.common import MinosConfig +from minos.common import ( + MinosConfig, +) class MinosSnapshotDispatcher(object): diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 13ca5bda..1ebf630b 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,11 +5,20 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import Any - -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig -from .dispatchers import MinosSnapshotDispatcher +from typing import ( + Any, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosSnapshotDispatcher, +) class MinosSnapshotService(PeriodicService): diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 48e113d9..ba3515ee 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,7 +8,9 @@ import unittest -from minos.networks import MinosSnapshotDispatcher +from minos.networks import ( + MinosSnapshotDispatcher, +) class TestMinosSnapshotDispatcher(unittest.TestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 5601d019..0deeaa3b 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -8,8 +8,12 @@ import unittest -from aiomisc.service.periodic import PeriodicService -from minos.networks import MinosSnapshotService +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.networks import ( + MinosSnapshotService, +) class TestMinosSnapshotService(unittest.TestCase): From 2a8847b9b2f0d4f3bd8a4f6375ceab9fe14d4ae7 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 29 Apr 2021 13:25:46 +0200 Subject: [PATCH 115/239] EventHandler - Test for MinosEventServer.handle_message EventHandler - Test for MinosEventServer.handle_message #64 --- minos/networks/event.py | 36 ++++++++++++++-------- tests/database_testcase.py | 2 +- tests/test_event_manager.py | 59 ++++++++++++++++++++++++++----------- 3 files changed, 65 insertions(+), 32 deletions(-) diff --git a/minos/networks/event.py b/minos/networks/event.py index 0b2c8a60..d771c121 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -80,30 +80,39 @@ async def event_queue_add(self, topic: str, partition: int, binary: bytes): return affected_rows, queue_id[0] - async def handle_message(self, consumer: t.Any): + async def handle_single_message(self, msg): """Handle Kafka messages. - For each message evaluate if the binary is an Event instance. + Evaluate if the binary of message is an Event instance. Add Event instance to the event_queue table. Args: - consumer: Kafka Consumer instance (at the moment only Kafka consumer is supported). + msg: Kafka message. Raises: Exception: An error occurred inserting record. """ + # the handler receive a message and store in the queue database + # check if the event binary string is well formatted + try: + Event.from_avro_bytes(msg.value) + await self.event_queue_add(msg.topic, msg.partition, msg.value) + finally: + pass + + + async def handle_message(self, consumer: t.Any): + """Message consumer. + + It consumes the messages and sends them for processing. + + Args: + consumer: Kafka Consumer instance (at the moment only Kafka consumer is supported). + """ async for msg in consumer: - # the handler receive a message and store in the queue database - topic = msg.topic - partition = msg.partition - event_binary = msg.value - # check if the event binary string is well formatted - try: - event_instance = Event.from_avro_bytes(event_binary) - await self.event_queue_add(msg.topic, msg.partition, event_binary) - except: - pass + await self.handle_single_message(msg) + async def start(self) -> t.Any: self.start_event.set() @@ -162,6 +171,7 @@ def __init__(self, *, conf: MinosConfig, **kwargs: t.Any): self._conf = conf def get_event_handler(self, topic: str) -> t.Callable: + """Get Event instance to call. Gets the instance of the class and method to call. diff --git a/tests/database_testcase.py b/tests/database_testcase.py index 86faa6c5..5b573848 100644 --- a/tests/database_testcase.py +++ b/tests/database_testcase.py @@ -62,6 +62,6 @@ async def asyncTearDown(self): database = await self._database() async with database as connection: async with connection.cursor() as cursor: - template = "DROP TABLE IF EXISTS queue" + template = "DROP TABLE IF EXISTS event_queue" await cursor.execute(template) """ diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 66d11f4a..39e0240b 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -14,6 +14,7 @@ from minos.networks.event import MinosEventServer, EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService from tests.database_testcase import EventHandlerPostgresAsyncTestCase + @pytest.fixture() def config(): return MinosConfig(path='./tests/test_config.yaml') @@ -23,9 +24,11 @@ def config(): def services(config): return [EventHandlerDatabaseInitializer(config=config), MinosEventServer(conf=config), MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config)] + class AggregateTest(Aggregate): test: int + class TestPostgreSqlMinosEventHandler(EventHandlerPostgresAsyncTestCase): async def test_database_connection(self): database = await self._database() @@ -69,13 +72,24 @@ async def test_get_event_handler(self): assert result == 'request_added' async def test_event_queue_checker(self): + model = AggregateTest(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TicketAdded", model=model.classname, items=[]) + bin_data = event_instance.avro_bytes + Event.from_avro_bytes(bin_data) + + m = MinosEventServer(conf=self._broker_config()) + affected_rows, id = await m.event_queue_add(topic=event_instance.topic, partition=0, binary=bin_data) + + assert affected_rows == 1 + assert id > 0 + database = await self._database() async with database as connect: async with connect.cursor() as cur: - await cur.execute("SELECT COUNT(*) FROM event_queue") + await cur.execute("SELECT COUNT(*) FROM event_queue WHERE id=%d" % (id)) records = await cur.fetchone() - assert records[0] > 0 + assert records[0] == 1 m = MinosEventHandlerPeriodicService(conf=self._broker_config(), interval=0.5) await m.event_queue_checker() @@ -83,18 +97,20 @@ async def test_event_queue_checker(self): database = await self._database() async with database as connect: async with connect.cursor() as cur: - await cur.execute("SELECT COUNT(*) FROM event_queue") + await cur.execute("SELECT COUNT(*) FROM event_queue WHERE id=%d" % (id)) records = await cur.fetchone() assert records[0] == 0 + async def test_producer_kafka(loop): - basic_config( - level=logging.INFO, - buffered=True, - log_format='color', - flush_interval=2 - ) + + #basic_config( + # level=logging.INFO, + # buffered=True, + # log_format='color', + # flush_interval=2 + #) producer = AIOKafkaProducer(loop=loop, bootstrap_servers='localhost:9092') # Get cluster layout and topic/partition allocation @@ -105,18 +121,18 @@ async def test_producer_kafka(loop): event_instance = Event(topic="TicketAdded", model=model.classname, items=[model]) bin_data = event_instance.avro_bytes - await producer.send_and_wait(event_instance.topic, bin_data) - await asyncio.sleep(1) - model2 = AggregateTest(test_id=2, test=2, id=1, version=1) event_instance_2 = Event(topic="TicketDeleted", model=model2.classname, items=[model2]) bin_data2 = event_instance_2.avro_bytes - await producer.send_and_wait(event_instance_2.topic, bin_data2) - await asyncio.sleep(1) + + for i in range(0, 100): + await producer.send_and_wait(event_instance.topic, bin_data) + await producer.send_and_wait(event_instance_2.topic, bin_data2) + await producer.stop() -async def test_event_handle_message(config, loop): +async def test_consumer_kafka(config,loop): handler = {item.name: {'controller': item.controller, 'action': item.action} for item in config.events.items} topics = list(handler.keys()) @@ -124,14 +140,21 @@ async def test_event_handle_message(config, loop): broker_group_name = f"event_{config.service.name}" m = MinosEventServer(conf=config) - consumer = AIOKafkaConsumer(loop=loop, + consumer = AIOKafkaConsumer( + loop=loop, group_id=broker_group_name, auto_offset_reset="latest", - bootstrap_servers=kafka_conn_data, + bootstrap_servers=kafka_conn_data, consumer_timeout_ms=3600 ) await consumer.start() consumer.subscribe(topics) - await m.handle_message(consumer) + for i in range(0, 2): + msg = await consumer.getone() + await m.handle_single_message(msg) + + await consumer.stop() + + From 9d7120e79ec94949a882a7cb9fe24452c4aaae93 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 29 Apr 2021 13:47:19 +0200 Subject: [PATCH 116/239] Update test_event_manager.py --- tests/test_event_manager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 39e0240b..bd1a54e8 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -103,7 +103,7 @@ async def test_event_queue_checker(self): assert records[0] == 0 -async def test_producer_kafka(loop): +async def test_producer_kafka(config, loop): #basic_config( # level=logging.INFO, @@ -111,8 +111,8 @@ async def test_producer_kafka(loop): # log_format='color', # flush_interval=2 #) - - producer = AIOKafkaProducer(loop=loop, bootstrap_servers='localhost:9092') + kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" + producer = AIOKafkaProducer(loop=loop, bootstrap_servers=kafka_conn_data) # Get cluster layout and topic/partition allocation await producer.start() # Produce messages @@ -125,7 +125,7 @@ async def test_producer_kafka(loop): event_instance_2 = Event(topic="TicketDeleted", model=model2.classname, items=[model2]) bin_data2 = event_instance_2.avro_bytes - for i in range(0, 100): + for i in range(0, 10): await producer.send_and_wait(event_instance.topic, bin_data) await producer.send_and_wait(event_instance_2.topic, bin_data2) @@ -144,7 +144,7 @@ async def test_consumer_kafka(config,loop): loop=loop, group_id=broker_group_name, auto_offset_reset="latest", - bootstrap_servers=kafka_conn_data, consumer_timeout_ms=3600 + bootstrap_servers=kafka_conn_data, consumer_timeout_ms=500 ) await consumer.start() From 2d3fb27bc11e9f36f18fe2b3ff5cf62dbaef8eb3 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 29 Apr 2021 14:25:02 +0200 Subject: [PATCH 117/239] Update test_event_manager.py --- tests/test_event_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index bd1a54e8..13036c9e 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -131,7 +131,7 @@ async def test_producer_kafka(config, loop): await producer.stop() - +""" async def test_consumer_kafka(config,loop): handler = {item.name: {'controller': item.controller, 'action': item.action} for item in config.events.items} @@ -155,6 +155,6 @@ async def test_consumer_kafka(config,loop): await m.handle_single_message(msg) await consumer.stop() - +""" From bed9711a3d7afce882a8158f8138207c5bcdeb37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 14:25:45 +0200 Subject: [PATCH 118/239] REFS #51 * Add initial implementation of the `minos.networks.MinosSnapshotEntry`. --- minos/networks/__init__.py | 1 + minos/networks/snapshots/__init__.py | 3 + minos/networks/snapshots/entries.py | 98 ++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 minos/networks/snapshots/entries.py diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index fbc5982d..44c88802 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -19,4 +19,5 @@ from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotService, + MinosSnapshotEntry, ) diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index 7e980b04..b919d8da 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -11,3 +11,6 @@ from .services import ( MinosSnapshotService, ) +from .entries import ( + MinosSnapshotEntry, +) diff --git a/minos/networks/snapshots/entries.py b/minos/networks/snapshots/entries.py new file mode 100644 index 00000000..86d5fa9d --- /dev/null +++ b/minos/networks/snapshots/entries.py @@ -0,0 +1,98 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from __future__ import ( + annotations, +) + +from datetime import ( + datetime, +) +from typing import ( + Iterable, + Optional, + Union, Type, +) + +from minos.common import ( + Aggregate, import_module, +) + + +class MinosSnapshotEntry(object): + """TODO""" + + __slots__ = "aggregate_id", "aggregate_name", "version", "data", "created_at", "updated_at" + + # noinspection PyShadowingBuiltins + def __init__( + self, + aggregate_id: int, + aggregate_name: str, + version: int, + data: Union[bytes, memoryview] = bytes(), + created_at: Optional[datetime] = None, + updated_at: Optional[datetime] = None, + ): + if isinstance(data, memoryview): + data = data.tobytes() + + self.aggregate_id = aggregate_id + self.aggregate_name = aggregate_name + self.version = version + self.data = data + + self.created_at = created_at + self.updated_at = updated_at + + @classmethod + def from_aggregate(cls, aggregate: Aggregate) -> MinosSnapshotEntry: + """Build a new instance from an ``Aggregate``. + + :param aggregate: The aggregate instance. + :return: A new ``MinosSnapshotEntry`` instance. + """ + # noinspection PyTypeChecker + return cls(aggregate.id, aggregate.classname, aggregate.version, aggregate.avro_bytes) + + @property + def aggregate(self) -> Aggregate: + """TODO + + :return: TODO + """ + cls = self.aggregate_cls + instance = cls.from_avro_bytes(self.data, id=self.aggregate_id, version=self.version) + return instance + + @property + def aggregate_cls(self) -> Type[Aggregate]: + """TODO + + :return: TODO + """ + # noinspection PyTypeChecker + return import_module(self.aggregate_name) + + def __eq__(self, other: MinosSnapshotEntry) -> bool: + return type(self) == type(other) and tuple(self) == tuple(other) + + def __hash__(self) -> int: + return hash(tuple(self)) + + def __iter__(self) -> Iterable: + # noinspection PyRedundantParentheses + yield from (self.aggregate_name, self.version, self.data, self.created_at, self.updated_at) + + def __repr__(self): + name = type(self).__name__ + return ( + f"{name}(aggregate_id={repr(self.aggregate_id)}, aggregate_name={repr(self.aggregate_name)}, " + f"version={repr(self.version)}, data={repr(self.data)}, " + f"created_at={repr(self.created_at)}, updated_at={repr(self.updated_at)})" + ) From 5977810be5815c386de2ef0c1d2c8eec9d9f1e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 14:27:02 +0200 Subject: [PATCH 119/239] REFS #51 * Add initial implementation of the `minos.networks.MinosSnapshotDispatcher`. --- minos/networks/snapshots/dispatchers.py | 168 +++++++++++++++++++++++- 1 file changed, 164 insertions(+), 4 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index af96a477..373bf69e 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -12,18 +12,47 @@ from typing import ( NoReturn, Optional, + Type, ) +import aiopg from minos.common import ( + Aggregate, MinosConfig, + MinosRepositoryAction, + MinosRepositoryEntry, + MinosSetup, + import_module, +) + +from .entries import ( + MinosSnapshotEntry, ) -class MinosSnapshotDispatcher(object): +class MinosSnapshotDispatcher(MinosSetup): """TODO""" - def __init__(self, *args, **kwargs): - ... + def __init__( + self, + *args, + host: str = None, + port: int = None, + database: str = None, + user: str = None, + password: str = None, + offset: int = 0, + **kwargs + ): + super().__init__(*args, **kwargs) + + self.host = host + self.port = port + self.database = database + self.user = user + self.password = password + + self.offset = offset @classmethod def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosSnapshotDispatcher]: @@ -38,7 +67,138 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi if config is None: return None # noinspection PyProtectedMember - return cls(*args, **kwargs) + return cls(*args, **config.repository._asdict(), **kwargs) + + async def _setup(self) -> NoReturn: + await self._create_broker_table() + + async def _create_broker_table(self) -> NoReturn: + async with self._connection() as connect: + async with connect.cursor() as cur: + await cur.execute(_CREATE_TABLE_QUERY) async def dispatch(self) -> NoReturn: """TODO""" + async for entry in self._new_entries: + await self._dispatch_one(entry) + + @property + async def _new_entries(self): + async with self._connection() as connect: + async with connect.cursor() as cursor: + await cursor.execute(_SELECT_EVENT_ENTRIES_QUERY, (self.offset,)) + async for raw in cursor: + yield MinosRepositoryEntry(*raw) + + async def _dispatch_one(self, event_entry: MinosRepositoryEntry) -> Optional[MinosSnapshotEntry]: + if event_entry.action is MinosRepositoryAction.DELETE: + await self._submit_delete(event_entry) + return + instance = self._build_instance(event_entry) + return await self._submit_instance(instance) + + async def _submit_delete(self, entry: MinosRepositoryEntry) -> NoReturn: + params = {"aggregate_id": entry.aggregate_id, "aggregate_name": entry.aggregate_name} + async with self._connection() as connect: + async with connect.cursor() as cursor: + await cursor.execute(_DELETE_ONE_SNAPSHOT_ENTRY_QUERY, params) + + def _build_instance(self, event_entry: MinosRepositoryEntry) -> Aggregate: + # noinspection PyTypeChecker + cls: Type[Aggregate] = import_module(event_entry.aggregate_name) + instance = cls.from_avro_bytes(event_entry.data, id=event_entry.aggregate_id, version=event_entry.version) + instance = self._update_if_exists(instance) + return instance + + def _update_if_exists(self, new: Aggregate) -> Aggregate: + try: + # noinspection PyTypeChecker + previous = await self._select_one_aggregate(new.id, new.classname) + new._fields = previous.fields | new.fields + finally: + return new + + async def _select_one_aggregate(self, aggregate_id: int, aggregate_name: str) -> Aggregate: + snapshot_entry = await self._select_one(aggregate_id, aggregate_name) + return snapshot_entry.aggregate + + async def _select_one(self, aggregate_id: int, aggregate_name: str) -> MinosSnapshotEntry: + async with self._connection() as connect: + async with connect.cursor() as cursor: + await cursor.execute(_SELECT_ONE_SNAPSHOT_ENTRY_QUERY, (aggregate_id, aggregate_name)) + raw = await cursor.fetchone() + if raw is None: + raise Exception() + return MinosSnapshotEntry(aggregate_id, aggregate_name, *raw) + + async def _submit_instance(self, aggregate: Aggregate) -> MinosSnapshotEntry: + snapshot_entry = MinosSnapshotEntry.from_aggregate(aggregate) + snapshot_entry = await self._submit_update_or_create(snapshot_entry) + return snapshot_entry + + async def _submit_update_or_create(self, entry: MinosSnapshotEntry) -> MinosSnapshotEntry: + params = { + "aggregate_id": entry.aggregate_id, + "aggregate_name": entry.aggregate_name, + "version": entry.version, + "data": entry.data, + } + async with self._connection() as connect: + async with connect.cursor() as cursor: + await cursor.execute(_INSERT_ONE_SNAPSHOT_ENTRY_QUERY, params) + response = await cursor.fetchone() + + entry.created_at, entry.updated_at = response + + return entry + + def _connection(self): + return aiopg.connect( + host=self.host, port=self.port, dbname=self.database, user=self.user, password=self.password + ) + + +_CREATE_TABLE_QUERY = """ +CREATE TABLE IF NOT EXISTS snapshot ( + aggregate_id BIGINT NOT NULL, + aggregate_name TEXT NOT NULL, + version INT NOT NULL, + data BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (aggregate_id, aggregate_name) +); +""".strip() + +_SELECT_EVENT_ENTRIES_QUERY = """ +SELECT aggregate_id, aggregate_name, version, data, id, action +FROM events +WHERE id >= %s; +""".strip() + +_DELETE_ONE_SNAPSHOT_ENTRY_QUERY = """ +DELETE FROM snapshot +WHERE aggregate_id = %(aggregate_id)s and aggregate_name = %(aggregate_name)s; +""".strip() + +_SELECT_ONE_SNAPSHOT_ENTRY_QUERY = """ +SELECT version, data, created_at, updated_at +FROM snapshot +WHERE aggregate_id = %s and aggregate_name = %s; +""".strip() + +_INSERT_ONE_SNAPSHOT_ENTRY_QUERY = """ +INSERT INTO snapshot (aggregate_id, aggregate_name, version, data, created_at, updated_at) +VALUES ( + %(aggregate_id)s, + %(aggregate_name)s, + %(version)s, + %(data)s, + default, + default +) +ON CONFLICT (aggregate_id, aggregate_name) +DO + UPDATE SET version = %(version)s, data = %(data)s, updated_at = NOW() +RETURNING created_at, updated_at; +""".strip() From 56c80ed0c12bfb7c5f24b4e061eb18d4c6d53fb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 14:27:23 +0200 Subject: [PATCH 120/239] REFS #51 * Configure setup on the start method. --- minos/networks/snapshots/services.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 0163e99e..ae95f3ab 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -27,6 +27,14 @@ def __init__(self, config: MinosConfig = None, **kwargs): super().__init__(**kwargs) self.dispatcher = MinosSnapshotDispatcher.from_config(config=config) + async def start(self) -> None: + """TODO + + :return: TODO + """ + await super().start() + await self.dispatcher.setup() + async def callback(self) -> Any: """TODO From 870c21aeba399121887914d90f3e0a14d848563a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 14:31:10 +0200 Subject: [PATCH 121/239] REFS #78 * Add setup step during the service startup. --- minos/networks/broker/services.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index c5dd51b5..c4626664 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -28,6 +28,14 @@ def __init__(self, config: MinosConfig = None, **kwargs): super().__init__(**kwargs) self.dispatcher = MinosQueueDispatcher.from_config(config=config) + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await super().start() + await self.dispatcher.setup() + async def callback(self) -> None: """Method to be called periodically by the internal ``aiomisc`` logic. From f1d259ac5dec821c9e5753a5bf87df2a1aecc4a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 14:36:03 +0200 Subject: [PATCH 122/239] ISSUE #51 * Fix bug. --- minos/networks/snapshots/dispatchers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 373bf69e..57cea268 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -94,7 +94,7 @@ async def _dispatch_one(self, event_entry: MinosRepositoryEntry) -> Optional[Min if event_entry.action is MinosRepositoryAction.DELETE: await self._submit_delete(event_entry) return - instance = self._build_instance(event_entry) + instance = await self._build_instance(event_entry) return await self._submit_instance(instance) async def _submit_delete(self, entry: MinosRepositoryEntry) -> NoReturn: @@ -103,14 +103,14 @@ async def _submit_delete(self, entry: MinosRepositoryEntry) -> NoReturn: async with connect.cursor() as cursor: await cursor.execute(_DELETE_ONE_SNAPSHOT_ENTRY_QUERY, params) - def _build_instance(self, event_entry: MinosRepositoryEntry) -> Aggregate: + async def _build_instance(self, event_entry: MinosRepositoryEntry) -> Aggregate: # noinspection PyTypeChecker cls: Type[Aggregate] = import_module(event_entry.aggregate_name) instance = cls.from_avro_bytes(event_entry.data, id=event_entry.aggregate_id, version=event_entry.version) - instance = self._update_if_exists(instance) + instance = await self._update_if_exists(instance) return instance - def _update_if_exists(self, new: Aggregate) -> Aggregate: + async def _update_if_exists(self, new: Aggregate) -> Aggregate: try: # noinspection PyTypeChecker previous = await self._select_one_aggregate(new.id, new.classname) From c8aaa48f83762b3ebc70e5bb5ca138cff28ce9f9 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 12:36:13 +0000 Subject: [PATCH 123/239] Restyled by black --- minos/networks/__init__.py | 4 +--- minos/networks/snapshots/__init__.py | 12 +++-------- minos/networks/snapshots/dispatchers.py | 8 ++------ minos/networks/snapshots/entries.py | 14 ++++++------- minos/networks/snapshots/services.py | 20 ++++++------------- .../test_snapshots/test_dispatchers.py | 4 +--- .../test_snapshots/test_services.py | 8 ++------ 7 files changed, 21 insertions(+), 49 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 24ac6b06..7f8639a9 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,9 +13,7 @@ MinosQueueDispatcher, MinosQueueService, ) -from .exceptions import ( - MinosNetworkException, -) +from .exceptions import MinosNetworkException from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotService, diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index b919d8da..e35f7d88 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -5,12 +5,6 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatchers import ( - MinosSnapshotDispatcher, -) -from .services import ( - MinosSnapshotService, -) -from .entries import ( - MinosSnapshotEntry, -) +from .dispatchers import MinosSnapshotDispatcher +from .services import MinosSnapshotService +from .entries import MinosSnapshotEntry diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 57cea268..951f9fb8 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NoReturn, @@ -25,9 +23,7 @@ import_module, ) -from .entries import ( - MinosSnapshotEntry, -) +from .entries import MinosSnapshotEntry class MinosSnapshotDispatcher(MinosSetup): diff --git a/minos/networks/snapshots/entries.py b/minos/networks/snapshots/entries.py index 86d5fa9d..4af55e74 100644 --- a/minos/networks/snapshots/entries.py +++ b/minos/networks/snapshots/entries.py @@ -6,21 +6,19 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations -from datetime import ( - datetime, -) +from datetime import datetime from typing import ( Iterable, Optional, - Union, Type, + Union, + Type, ) from minos.common import ( - Aggregate, import_module, + Aggregate, + import_module, ) diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 75cec27c..2969ea1f 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,20 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - Any, -) - -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) - -from .dispatchers import ( - MinosSnapshotDispatcher, -) +from typing import Any + +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig + +from .dispatchers import MinosSnapshotDispatcher class MinosSnapshotService(PeriodicService): diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index ba3515ee..48e113d9 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,9 +8,7 @@ import unittest -from minos.networks import ( - MinosSnapshotDispatcher, -) +from minos.networks import MinosSnapshotDispatcher class TestMinosSnapshotDispatcher(unittest.TestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 0deeaa3b..5601d019 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -8,12 +8,8 @@ import unittest -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.networks import ( - MinosSnapshotService, -) +from aiomisc.service.periodic import PeriodicService +from minos.networks import MinosSnapshotService class TestMinosSnapshotService(unittest.TestCase): From adc18ee91809f61176e5d9715d322f48b44a5820 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 12:36:13 +0000 Subject: [PATCH 124/239] Restyled by isort --- minos/networks/__init__.py | 6 ++++-- minos/networks/snapshots/__init__.py | 12 ++++++++--- minos/networks/snapshots/dispatchers.py | 8 ++++++-- minos/networks/snapshots/entries.py | 10 +++++++--- minos/networks/snapshots/services.py | 20 +++++++++++++------ .../test_snapshots/test_dispatchers.py | 4 +++- .../test_snapshots/test_services.py | 8 ++++++-- 7 files changed, 49 insertions(+), 19 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 7f8639a9..b90383e8 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,9 +13,11 @@ MinosQueueDispatcher, MinosQueueService, ) -from .exceptions import MinosNetworkException +from .exceptions import ( + MinosNetworkException, +) from .snapshots import ( MinosSnapshotDispatcher, - MinosSnapshotService, MinosSnapshotEntry, + MinosSnapshotService, ) diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index e35f7d88..ab7b0571 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -5,6 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatchers import MinosSnapshotDispatcher -from .services import MinosSnapshotService -from .entries import MinosSnapshotEntry +from .dispatchers import ( + MinosSnapshotDispatcher, +) +from .entries import ( + MinosSnapshotEntry, +) +from .services import ( + MinosSnapshotService, +) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 951f9fb8..57cea268 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( NoReturn, @@ -23,7 +25,9 @@ import_module, ) -from .entries import MinosSnapshotEntry +from .entries import ( + MinosSnapshotEntry, +) class MinosSnapshotDispatcher(MinosSetup): diff --git a/minos/networks/snapshots/entries.py b/minos/networks/snapshots/entries.py index 4af55e74..8fd771f1 100644 --- a/minos/networks/snapshots/entries.py +++ b/minos/networks/snapshots/entries.py @@ -6,14 +6,18 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) -from datetime import datetime +from datetime import ( + datetime, +) from typing import ( Iterable, Optional, - Union, Type, + Union, ) from minos.common import ( diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 2969ea1f..75cec27c 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,12 +5,20 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import Any - -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig - -from .dispatchers import MinosSnapshotDispatcher +from typing import ( + Any, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosSnapshotDispatcher, +) class MinosSnapshotService(PeriodicService): diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 48e113d9..ba3515ee 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,7 +8,9 @@ import unittest -from minos.networks import MinosSnapshotDispatcher +from minos.networks import ( + MinosSnapshotDispatcher, +) class TestMinosSnapshotDispatcher(unittest.TestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 5601d019..0deeaa3b 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -8,8 +8,12 @@ import unittest -from aiomisc.service.periodic import PeriodicService -from minos.networks import MinosSnapshotService +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.networks import ( + MinosSnapshotService, +) class TestMinosSnapshotService(unittest.TestCase): From 3693382ccd4bfdbe76c650a56b401799bcd1af56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 15:00:20 +0200 Subject: [PATCH 125/239] ISSUE #78 * Add tests for `minos.networks.MinosQueueService`. --- .../test_networks/test_broker/test_service.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tests/test_networks/test_broker/test_service.py diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py new file mode 100644 index 00000000..cfe92221 --- /dev/null +++ b/tests/test_networks/test_broker/test_service.py @@ -0,0 +1,56 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +import unittest +from unittest.mock import MagicMock + +from aiomisc.service.periodic import PeriodicService +from minos.common.testing import PostgresAsyncTestCase + +from minos.networks import MinosQueueService, MinosQueueDispatcher +from tests.utils import BASE_PATH + + +class TestMinosQueueService(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_is_instance(self): + service = MinosQueueService(interval=10) + self.assertIsInstance(service, PeriodicService) + + def test_dispatcher_empty(self): + service = MinosQueueService(interval=10) + self.assertIsNone(service.dispatcher) + + def test_dispatcher_config(self): + service = MinosQueueService(interval=10, config=self.config) + dispatcher = service.dispatcher + self.assertIsInstance(dispatcher, MinosQueueDispatcher) + self.assertFalse(dispatcher.already_setup) + + def test_dispatcher_config_context(self): + with self.config: + service = MinosQueueService(interval=10) + self.assertIsInstance(service.dispatcher, MinosQueueDispatcher) + + async def test_start(self): + with self.config: + service = MinosQueueService(interval=1, loop=None) + service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) + await service.start() + self.assertTrue(1, service.dispatcher.setup.call_count) + + async def test_callback(self): + with self.config: + service = MinosQueueService(interval=1, loop=None) + service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) + await service.start() + self.assertEqual(1, service.dispatcher.dispatch.call_count) + + +if __name__ == '__main__': + unittest.main() From dfb86be4d1721c556b45c7f3061f8004fecd7517 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 13:00:53 +0000 Subject: [PATCH 126/239] Restyled by black --- minos/networks/broker/services.py | 20 ++++++------------- .../test_networks/test_broker/test_service.py | 2 +- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index c4626664..645da90e 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -5,20 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) - -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) - -from .dispatchers import ( - MinosQueueDispatcher, -) +from __future__ import annotations + +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig + +from .dispatchers import MinosQueueDispatcher class MinosQueueService(PeriodicService): diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index cfe92221..ea6c8a67 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -52,5 +52,5 @@ async def test_callback(self): self.assertEqual(1, service.dispatcher.dispatch.call_count) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From 34d09c04c30f6b655b47c548bf573e8e31a59d16 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 13:00:54 +0000 Subject: [PATCH 127/239] Restyled by isort --- minos/networks/broker/services.py | 20 ++++++++++++----- .../test_networks/test_broker/test_service.py | 22 ++++++++++++++----- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index 645da90e..c4626664 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -5,12 +5,20 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations - -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig - -from .dispatchers import MinosQueueDispatcher +from __future__ import ( + annotations, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosQueueDispatcher, +) class MinosQueueService(PeriodicService): diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index ea6c8a67..b96cbba8 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -6,13 +6,23 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ import unittest -from unittest.mock import MagicMock +from unittest.mock import ( + MagicMock, +) -from aiomisc.service.periodic import PeriodicService -from minos.common.testing import PostgresAsyncTestCase - -from minos.networks import MinosQueueService, MinosQueueDispatcher -from tests.utils import BASE_PATH +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosQueueDispatcher, + MinosQueueService, +) +from tests.utils import ( + BASE_PATH, +) class TestMinosQueueService(PostgresAsyncTestCase): From 564c0af7a51f1454d1efdcbba82486c1bbfcace0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 15:02:48 +0200 Subject: [PATCH 128/239] ISSUE #? * Fix filename mismatching. --- tests/database_testcase.py | 2 +- tests/test_event_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/database_testcase.py b/tests/database_testcase.py index 5b573848..5d0f4096 100644 --- a/tests/database_testcase.py +++ b/tests/database_testcase.py @@ -48,7 +48,7 @@ async def asyncSetUp(self): @staticmethod def _broker_config(): - return MinosConfig(path="./tests/test_config.yaml") + return MinosConfig(path="./tests/test_config.yml") async def _database(self): conf = self._broker_config() diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 13036c9e..2b7ad25a 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -17,7 +17,7 @@ @pytest.fixture() def config(): - return MinosConfig(path='./tests/test_config.yaml') + return MinosConfig(path='./tests/test_config.yml') @pytest.fixture() From e7d77c46b059d2845864505daea3edbbe6ffed3e Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 13:06:31 +0000 Subject: [PATCH 129/239] Restyled by black --- minos/networks/__init__.py | 4 +- minos/networks/event.py | 57 +++++++++++-------- minos/networks/snapshots/__init__.py | 12 +--- minos/networks/snapshots/dispatchers.py | 8 +-- minos/networks/snapshots/entries.py | 8 +-- minos/networks/snapshots/services.py | 20 ++----- tests/database_testcase.py | 6 +- tests/test_event_manager.py | 17 +++--- .../test_snapshots/test_dispatchers.py | 4 +- .../test_snapshots/test_services.py | 8 +-- 10 files changed, 65 insertions(+), 79 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index b90383e8..2c5d2e47 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,9 +13,7 @@ MinosQueueDispatcher, MinosQueueService, ) -from .exceptions import ( - MinosNetworkException, -) +from .exceptions import MinosNetworkException from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotEntry, diff --git a/minos/networks/event.py b/minos/networks/event.py index d771c121..99d2780f 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -30,14 +30,18 @@ class MinosEventServer(Service): Consumer for the Broker ( at the moment only Kafka is supported ) """ + __slots__ = "_tasks", "_db_dsn", "_handlers", "_topics", "_kafka_conn_data", "_broker_group_name" def __init__(self, *, conf: MinosConfig, **kwargs: t.Any): self._tasks = set() # type: t.Set[asyncio.Task] - self._db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ - f"password={conf.events.queue.password} host={conf.events.queue.host}" - self._handler = {item.name: {'controller': item.controller, 'action': item.action} - for item in conf.events.items} + self._db_dsn = ( + f"dbname={conf.events.queue.database} user={conf.events.queue.user} " + f"password={conf.events.queue.password} host={conf.events.queue.host}" + ) + self._handler = { + item.name: {"controller": item.controller, "action": item.action} for item in conf.events.items + } self._topics = list(self._handler.keys()) self._kafka_conn_data = f"{conf.events.broker.host}:{conf.events.broker.port}" self._broker_group_name = f"event_{conf.service.name}" @@ -100,7 +104,6 @@ async def handle_single_message(self, msg): finally: pass - async def handle_message(self, consumer: t.Any): """Message consumer. @@ -113,16 +116,16 @@ async def handle_message(self, consumer: t.Any): async for msg in consumer: await self.handle_single_message(msg) - async def start(self) -> t.Any: self.start_event.set() log.debug("Event Consumer Manager: Started") # start the Service Event Consumer for Kafka - consumer = AIOKafkaConsumer(loop=self.loop, - group_id=self._broker_group_name, - auto_offset_reset="latest", - bootstrap_servers=self._kafka_conn_data, - ) + consumer = AIOKafkaConsumer( + loop=self.loop, + group_id=self._broker_group_name, + auto_offset_reset="latest", + bootstrap_servers=self._kafka_conn_data, + ) await consumer.start() consumer.subscribe(self._topics) @@ -130,10 +133,11 @@ async def start(self) -> t.Any: self.create_task(self.handle_message(consumer)) - async def event_handler_table_creation(conf: MinosConfig): - db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ - f"password={conf.events.queue.password} host={conf.events.queue.host}" + db_dsn = ( + f"dbname={conf.events.queue.database} user={conf.events.queue.user} " + f"password={conf.events.queue.password} host={conf.events.queue.host}" + ) async with aiopg.create_pool(db_dsn) as pool: async with pool.acquire() as connect: async with connect.cursor() as cur: @@ -158,14 +162,18 @@ class MinosEventHandlerPeriodicService(PeriodicService): Periodic Service Event Handler """ - __slots__ = "_db_dsn", "_handlers", "_event_items", "_topics" , "_conf" + + __slots__ = "_db_dsn", "_handlers", "_event_items", "_topics", "_conf" def __init__(self, *, conf: MinosConfig, **kwargs: t.Any): super().__init__(**kwargs) - self._db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ - f"password={conf.events.queue.password} host={conf.events.queue.host}" - self._handlers = {item.name: {'controller': item.controller, 'action': item.action} - for item in conf.events.items} + self._db_dsn = ( + f"dbname={conf.events.queue.database} user={conf.events.queue.user} " + f"password={conf.events.queue.password} host={conf.events.queue.host}" + ) + self._handlers = { + item.name: {"controller": item.controller, "action": item.action} for item in conf.events.items + } self._event_items = conf.events.items self._topics = list(self._handlers.keys()) self._conf = conf @@ -194,8 +202,9 @@ def get_event_handler(self, topic: str) -> t.Callable: class_method = getattr(instance_class, action) return class_method - raise MinosNetworkException(f"topic {topic} have no controller/action configured, " - f"please review th configuration file") + raise MinosNetworkException( + f"topic {topic} have no controller/action configured, " f"please review th configuration file" + ) async def event_queue_checker(self): """Event Queue Checker and dispatcher. @@ -209,8 +218,10 @@ async def event_queue_checker(self): Raises: Exception: An error occurred inserting record. """ - db_dsn = f"dbname={self._conf.events.queue.database} user={self._conf.events.queue.user} " \ - f"password={self._conf.events.queue.password} host={self._conf.events.queue.host}" + db_dsn = ( + f"dbname={self._conf.events.queue.database} user={self._conf.events.queue.user} " + f"password={self._conf.events.queue.password} host={self._conf.events.queue.host}" + ) async with aiopg.create_pool(db_dsn) as pool: async with pool.acquire() as connect: async with connect.cursor() as cur: diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index ab7b0571..29ac484a 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -5,12 +5,6 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatchers import ( - MinosSnapshotDispatcher, -) -from .entries import ( - MinosSnapshotEntry, -) -from .services import ( - MinosSnapshotService, -) +from .dispatchers import MinosSnapshotDispatcher +from .entries import MinosSnapshotEntry +from .services import MinosSnapshotService diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 57cea268..951f9fb8 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NoReturn, @@ -25,9 +23,7 @@ import_module, ) -from .entries import ( - MinosSnapshotEntry, -) +from .entries import MinosSnapshotEntry class MinosSnapshotDispatcher(MinosSetup): diff --git a/minos/networks/snapshots/entries.py b/minos/networks/snapshots/entries.py index 8fd771f1..fa5bc0c7 100644 --- a/minos/networks/snapshots/entries.py +++ b/minos/networks/snapshots/entries.py @@ -6,13 +6,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations -from datetime import ( - datetime, -) +from datetime import datetime from typing import ( Iterable, Optional, diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 75cec27c..2969ea1f 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,20 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - Any, -) - -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) - -from .dispatchers import ( - MinosSnapshotDispatcher, -) +from typing import Any + +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig + +from .dispatchers import MinosSnapshotDispatcher class MinosSnapshotService(PeriodicService): diff --git a/tests/database_testcase.py b/tests/database_testcase.py index 5d0f4096..fd87d0fb 100644 --- a/tests/database_testcase.py +++ b/tests/database_testcase.py @@ -52,8 +52,10 @@ def _broker_config(): async def _database(self): conf = self._broker_config() - db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ - f"password={conf.events.queue.password} host={conf.events.queue.host}" + db_dsn = ( + f"dbname={conf.events.queue.database} user={conf.events.queue.user} " + f"password={conf.events.queue.password} host={conf.events.queue.host}" + ) return await aiopg.connect(db_dsn) async def asyncTearDown(self): diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 2b7ad25a..4afbf1df 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -17,12 +17,16 @@ @pytest.fixture() def config(): - return MinosConfig(path='./tests/test_config.yml') + return MinosConfig(path="./tests/test_config.yml") @pytest.fixture() def services(config): - return [EventHandlerDatabaseInitializer(config=config), MinosEventServer(conf=config), MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config)] + return [ + EventHandlerDatabaseInitializer(config=config), + MinosEventServer(conf=config), + MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config), + ] class AggregateTest(Aggregate): @@ -69,7 +73,7 @@ async def test_get_event_handler(self): cls = m.get_event_handler(topic="TicketAdded") result = await cls(topic="TicketAdded", event=event_instance) - assert result == 'request_added' + assert result == "request_added" async def test_event_queue_checker(self): model = AggregateTest(test_id=1, test=2, id=1, version=1) @@ -105,12 +109,12 @@ async def test_event_queue_checker(self): async def test_producer_kafka(config, loop): - #basic_config( + # basic_config( # level=logging.INFO, # buffered=True, # log_format='color', # flush_interval=2 - #) + # ) kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" producer = AIOKafkaProducer(loop=loop, bootstrap_servers=kafka_conn_data) # Get cluster layout and topic/partition allocation @@ -131,6 +135,7 @@ async def test_producer_kafka(config, loop): await producer.stop() + """ async def test_consumer_kafka(config,loop): handler = {item.name: {'controller': item.controller, 'action': item.action} @@ -156,5 +161,3 @@ async def test_consumer_kafka(config,loop): await consumer.stop() """ - - diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index ba3515ee..48e113d9 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,9 +8,7 @@ import unittest -from minos.networks import ( - MinosSnapshotDispatcher, -) +from minos.networks import MinosSnapshotDispatcher class TestMinosSnapshotDispatcher(unittest.TestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 0deeaa3b..5601d019 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -8,12 +8,8 @@ import unittest -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.networks import ( - MinosSnapshotService, -) +from aiomisc.service.periodic import PeriodicService +from minos.networks import MinosSnapshotService class TestMinosSnapshotService(unittest.TestCase): From 04a9af534eabddb36ea1e7bc925d8ce4a9e68a3b Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 13:06:32 +0000 Subject: [PATCH 130/239] Restyled by isort --- minos/networks/__init__.py | 4 +- minos/networks/event.py | 37 ++++++++++++----- minos/networks/snapshots/__init__.py | 12 ++++-- minos/networks/snapshots/dispatchers.py | 8 +++- minos/networks/snapshots/entries.py | 8 +++- minos/networks/snapshots/services.py | 20 +++++++--- tests/database_testcase.py | 8 +++- tests/services/CqrsTestService.py | 4 +- tests/test_event_manager.py | 40 ++++++++++++++----- .../test_snapshots/test_dispatchers.py | 4 +- .../test_snapshots/test_services.py | 8 +++- 11 files changed, 111 insertions(+), 42 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 2c5d2e47..b90383e8 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,7 +13,9 @@ MinosQueueDispatcher, MinosQueueService, ) -from .exceptions import MinosNetworkException +from .exceptions import ( + MinosNetworkException, +) from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotEntry, diff --git a/minos/networks/event.py b/minos/networks/event.py index 99d2780f..37f3c435 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -6,21 +6,36 @@ # permission of Clariteia SL. import asyncio -import functools -import typing as t import datetime +import functools import inspect +import typing as t -from aiokafka import AIOKafkaConsumer -from aiomisc import Service -from aiomisc.service.periodic import PeriodicService import aiopg -from minos.common.configuration.config import MinosConfig -from minos.common.importlib import import_module -from minos.common.logs import log - -from minos.networks.exceptions import MinosNetworkException -from minos.common.broker import Event +from aiokafka import ( + AIOKafkaConsumer, +) +from aiomisc import ( + Service, +) +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common.broker import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.importlib import ( + import_module, +) +from minos.common.logs import ( + log, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) class MinosEventServer(Service): diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index 29ac484a..ab7b0571 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -5,6 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatchers import MinosSnapshotDispatcher -from .entries import MinosSnapshotEntry -from .services import MinosSnapshotService +from .dispatchers import ( + MinosSnapshotDispatcher, +) +from .entries import ( + MinosSnapshotEntry, +) +from .services import ( + MinosSnapshotService, +) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 951f9fb8..57cea268 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( NoReturn, @@ -23,7 +25,9 @@ import_module, ) -from .entries import MinosSnapshotEntry +from .entries import ( + MinosSnapshotEntry, +) class MinosSnapshotDispatcher(MinosSetup): diff --git a/minos/networks/snapshots/entries.py b/minos/networks/snapshots/entries.py index fa5bc0c7..8fd771f1 100644 --- a/minos/networks/snapshots/entries.py +++ b/minos/networks/snapshots/entries.py @@ -6,9 +6,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) -from datetime import datetime +from datetime import ( + datetime, +) from typing import ( Iterable, Optional, diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 2969ea1f..75cec27c 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,12 +5,20 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import Any - -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig - -from .dispatchers import MinosSnapshotDispatcher +from typing import ( + Any, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosSnapshotDispatcher, +) class MinosSnapshotService(PeriodicService): diff --git a/tests/database_testcase.py b/tests/database_testcase.py index fd87d0fb..a6d2148e 100644 --- a/tests/database_testcase.py +++ b/tests/database_testcase.py @@ -7,8 +7,12 @@ import unittest import aiopg -from minos.common.configuration.config import MinosConfig -from minos.networks.event import event_handler_table_creation +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.networks.event import ( + event_handler_table_creation, +) class EventHandlerPostgresAsyncTestCase(unittest.IsolatedAsyncioTestCase): diff --git a/tests/services/CqrsTestService.py b/tests/services/CqrsTestService.py index dfbe0293..11100f0d 100644 --- a/tests/services/CqrsTestService.py +++ b/tests/services/CqrsTestService.py @@ -1,4 +1,6 @@ -from minos.common.broker import Event +from minos.common.broker import ( + Event, +) class CqrsService(object): diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 4afbf1df..f0bc87c4 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -1,18 +1,36 @@ import asyncio import logging - -import pytest -import string -from aiokafka import AIOKafkaProducer, AIOKafkaConsumer import random +import string -from minos.common.logs import log -from aiomisc.log import basic_config -from minos.common.configuration.config import MinosConfig -from minos.common import Aggregate -from minos.common.broker import Event -from minos.networks.event import MinosEventServer, EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService -from tests.database_testcase import EventHandlerPostgresAsyncTestCase +import pytest +from aiokafka import ( + AIOKafkaConsumer, + AIOKafkaProducer, +) +from aiomisc.log import ( + basic_config, +) +from minos.common import ( + Aggregate, +) +from minos.common.broker import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.logs import ( + log, +) +from minos.networks.event import ( + EventHandlerDatabaseInitializer, + MinosEventHandlerPeriodicService, + MinosEventServer, +) +from tests.database_testcase import ( + EventHandlerPostgresAsyncTestCase, +) @pytest.fixture() diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 48e113d9..ba3515ee 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,7 +8,9 @@ import unittest -from minos.networks import MinosSnapshotDispatcher +from minos.networks import ( + MinosSnapshotDispatcher, +) class TestMinosSnapshotDispatcher(unittest.TestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 5601d019..0deeaa3b 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -8,8 +8,12 @@ import unittest -from aiomisc.service.periodic import PeriodicService -from minos.networks import MinosSnapshotService +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.networks import ( + MinosSnapshotService, +) class TestMinosSnapshotService(unittest.TestCase): From cfeb53e42209bb0613340692e3d887cc0618ed4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 15:12:57 +0200 Subject: [PATCH 131/239] ISSUE #51 * Add tests for `minos.networks.MinosSnapshotService`. --- .../test_snapshots/test_services.py | 48 ++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 0deeaa3b..54e02189 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -7,18 +7,54 @@ """ import unittest +from unittest.mock import MagicMock from aiomisc.service.periodic import ( PeriodicService, ) +from minos.common.testing import PostgresAsyncTestCase + from minos.networks import ( - MinosSnapshotService, + MinosSnapshotService, MinosSnapshotDispatcher, ) - - -class TestMinosSnapshotService(unittest.TestCase): - def test_type(self): - self.assertTrue(issubclass(MinosSnapshotService, PeriodicService)) +from tests.utils import BASE_PATH + + +class TestMinosSnapshotService(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_is_instance(self): + service = MinosSnapshotService(interval=10) + self.assertIsInstance(service, PeriodicService) + + def test_dispatcher_empty(self): + service = MinosSnapshotService(interval=10) + self.assertIsNone(service.dispatcher) + + def test_dispatcher_config(self): + service = MinosSnapshotService(interval=10, config=self.config) + dispatcher = service.dispatcher + self.assertIsInstance(dispatcher, MinosSnapshotDispatcher) + self.assertFalse(dispatcher.already_setup) + + def test_dispatcher_config_context(self): + with self.config: + service = MinosSnapshotService(interval=10) + self.assertIsInstance(service.dispatcher, MinosSnapshotDispatcher) + + async def test_start(self): + with self.config: + service = MinosSnapshotService(interval=1, loop=None) + service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) + await service.start() + self.assertTrue(1, service.dispatcher.setup.call_count) + + async def test_callback(self): + with self.config: + service = MinosSnapshotService(interval=1, loop=None) + service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) + await service.start() + self.assertEqual(1, service.dispatcher.dispatch.call_count) if __name__ == "__main__": From 0f143239f3c3d082c089495d791728267d4ee595 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 13:13:15 +0000 Subject: [PATCH 132/239] Restyled by black --- minos/networks/__init__.py | 4 +-- minos/networks/event.py | 32 +++++-------------- minos/networks/snapshots/__init__.py | 12 ++----- minos/networks/snapshots/dispatchers.py | 8 ++--- minos/networks/snapshots/entries.py | 8 ++--- minos/networks/snapshots/services.py | 20 ++++-------- tests/database_testcase.py | 8 ++--- tests/services/CqrsTestService.py | 4 +-- tests/test_event_manager.py | 24 ++++---------- .../test_snapshots/test_dispatchers.py | 4 +-- .../test_snapshots/test_services.py | 7 ++-- 11 files changed, 35 insertions(+), 96 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index b90383e8..2c5d2e47 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,9 +13,7 @@ MinosQueueDispatcher, MinosQueueService, ) -from .exceptions import ( - MinosNetworkException, -) +from .exceptions import MinosNetworkException from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotEntry, diff --git a/minos/networks/event.py b/minos/networks/event.py index 37f3c435..399fc9e6 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -12,30 +12,14 @@ import typing as t import aiopg -from aiokafka import ( - AIOKafkaConsumer, -) -from aiomisc import ( - Service, -) -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common.broker import ( - Event, -) -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.common.importlib import ( - import_module, -) -from minos.common.logs import ( - log, -) -from minos.networks.exceptions import ( - MinosNetworkException, -) +from aiokafka import AIOKafkaConsumer +from aiomisc import Service +from aiomisc.service.periodic import PeriodicService +from minos.common.broker import Event +from minos.common.configuration.config import MinosConfig +from minos.common.importlib import import_module +from minos.common.logs import log +from minos.networks.exceptions import MinosNetworkException class MinosEventServer(Service): diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index ab7b0571..29ac484a 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -5,12 +5,6 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatchers import ( - MinosSnapshotDispatcher, -) -from .entries import ( - MinosSnapshotEntry, -) -from .services import ( - MinosSnapshotService, -) +from .dispatchers import MinosSnapshotDispatcher +from .entries import MinosSnapshotEntry +from .services import MinosSnapshotService diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 57cea268..951f9fb8 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NoReturn, @@ -25,9 +23,7 @@ import_module, ) -from .entries import ( - MinosSnapshotEntry, -) +from .entries import MinosSnapshotEntry class MinosSnapshotDispatcher(MinosSetup): diff --git a/minos/networks/snapshots/entries.py b/minos/networks/snapshots/entries.py index 8fd771f1..fa5bc0c7 100644 --- a/minos/networks/snapshots/entries.py +++ b/minos/networks/snapshots/entries.py @@ -6,13 +6,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations -from datetime import ( - datetime, -) +from datetime import datetime from typing import ( Iterable, Optional, diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 75cec27c..2969ea1f 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,20 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - Any, -) - -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) - -from .dispatchers import ( - MinosSnapshotDispatcher, -) +from typing import Any + +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig + +from .dispatchers import MinosSnapshotDispatcher class MinosSnapshotService(PeriodicService): diff --git a/tests/database_testcase.py b/tests/database_testcase.py index a6d2148e..fd87d0fb 100644 --- a/tests/database_testcase.py +++ b/tests/database_testcase.py @@ -7,12 +7,8 @@ import unittest import aiopg -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.networks.event import ( - event_handler_table_creation, -) +from minos.common.configuration.config import MinosConfig +from minos.networks.event import event_handler_table_creation class EventHandlerPostgresAsyncTestCase(unittest.IsolatedAsyncioTestCase): diff --git a/tests/services/CqrsTestService.py b/tests/services/CqrsTestService.py index 11100f0d..dfbe0293 100644 --- a/tests/services/CqrsTestService.py +++ b/tests/services/CqrsTestService.py @@ -1,6 +1,4 @@ -from minos.common.broker import ( - Event, -) +from minos.common.broker import Event class CqrsService(object): diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index f0bc87c4..b3cbd37c 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -8,29 +8,17 @@ AIOKafkaConsumer, AIOKafkaProducer, ) -from aiomisc.log import ( - basic_config, -) -from minos.common import ( - Aggregate, -) -from minos.common.broker import ( - Event, -) -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.common.logs import ( - log, -) +from aiomisc.log import basic_config +from minos.common import Aggregate +from minos.common.broker import Event +from minos.common.configuration.config import MinosConfig +from minos.common.logs import log from minos.networks.event import ( EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService, MinosEventServer, ) -from tests.database_testcase import ( - EventHandlerPostgresAsyncTestCase, -) +from tests.database_testcase import EventHandlerPostgresAsyncTestCase @pytest.fixture() diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index ba3515ee..48e113d9 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,9 +8,7 @@ import unittest -from minos.networks import ( - MinosSnapshotDispatcher, -) +from minos.networks import MinosSnapshotDispatcher class TestMinosSnapshotDispatcher(unittest.TestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 54e02189..2eb7392a 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -9,13 +9,12 @@ import unittest from unittest.mock import MagicMock -from aiomisc.service.periodic import ( - PeriodicService, -) +from aiomisc.service.periodic import PeriodicService from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( - MinosSnapshotService, MinosSnapshotDispatcher, + MinosSnapshotService, + MinosSnapshotDispatcher, ) from tests.utils import BASE_PATH From 19f9955b7ee2daeba25991fd2b579a007c8b8e82 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 13:13:16 +0000 Subject: [PATCH 133/239] Restyled by isort --- minos/networks/__init__.py | 4 ++- minos/networks/event.py | 32 ++++++++++++++----- minos/networks/snapshots/__init__.py | 12 +++++-- minos/networks/snapshots/dispatchers.py | 8 +++-- minos/networks/snapshots/entries.py | 8 +++-- minos/networks/snapshots/services.py | 20 ++++++++---- tests/database_testcase.py | 8 +++-- tests/services/CqrsTestService.py | 4 ++- tests/test_event_manager.py | 24 ++++++++++---- .../test_snapshots/test_dispatchers.py | 4 ++- .../test_snapshots/test_services.py | 19 +++++++---- 11 files changed, 105 insertions(+), 38 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 2c5d2e47..b90383e8 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,7 +13,9 @@ MinosQueueDispatcher, MinosQueueService, ) -from .exceptions import MinosNetworkException +from .exceptions import ( + MinosNetworkException, +) from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotEntry, diff --git a/minos/networks/event.py b/minos/networks/event.py index 399fc9e6..37f3c435 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -12,14 +12,30 @@ import typing as t import aiopg -from aiokafka import AIOKafkaConsumer -from aiomisc import Service -from aiomisc.service.periodic import PeriodicService -from minos.common.broker import Event -from minos.common.configuration.config import MinosConfig -from minos.common.importlib import import_module -from minos.common.logs import log -from minos.networks.exceptions import MinosNetworkException +from aiokafka import ( + AIOKafkaConsumer, +) +from aiomisc import ( + Service, +) +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common.broker import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.importlib import ( + import_module, +) +from minos.common.logs import ( + log, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) class MinosEventServer(Service): diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index 29ac484a..ab7b0571 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -5,6 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatchers import MinosSnapshotDispatcher -from .entries import MinosSnapshotEntry -from .services import MinosSnapshotService +from .dispatchers import ( + MinosSnapshotDispatcher, +) +from .entries import ( + MinosSnapshotEntry, +) +from .services import ( + MinosSnapshotService, +) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 951f9fb8..57cea268 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( NoReturn, @@ -23,7 +25,9 @@ import_module, ) -from .entries import MinosSnapshotEntry +from .entries import ( + MinosSnapshotEntry, +) class MinosSnapshotDispatcher(MinosSetup): diff --git a/minos/networks/snapshots/entries.py b/minos/networks/snapshots/entries.py index fa5bc0c7..8fd771f1 100644 --- a/minos/networks/snapshots/entries.py +++ b/minos/networks/snapshots/entries.py @@ -6,9 +6,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) -from datetime import datetime +from datetime import ( + datetime, +) from typing import ( Iterable, Optional, diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 2969ea1f..75cec27c 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,12 +5,20 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import Any - -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig - -from .dispatchers import MinosSnapshotDispatcher +from typing import ( + Any, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosSnapshotDispatcher, +) class MinosSnapshotService(PeriodicService): diff --git a/tests/database_testcase.py b/tests/database_testcase.py index fd87d0fb..a6d2148e 100644 --- a/tests/database_testcase.py +++ b/tests/database_testcase.py @@ -7,8 +7,12 @@ import unittest import aiopg -from minos.common.configuration.config import MinosConfig -from minos.networks.event import event_handler_table_creation +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.networks.event import ( + event_handler_table_creation, +) class EventHandlerPostgresAsyncTestCase(unittest.IsolatedAsyncioTestCase): diff --git a/tests/services/CqrsTestService.py b/tests/services/CqrsTestService.py index dfbe0293..11100f0d 100644 --- a/tests/services/CqrsTestService.py +++ b/tests/services/CqrsTestService.py @@ -1,4 +1,6 @@ -from minos.common.broker import Event +from minos.common.broker import ( + Event, +) class CqrsService(object): diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index b3cbd37c..f0bc87c4 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -8,17 +8,29 @@ AIOKafkaConsumer, AIOKafkaProducer, ) -from aiomisc.log import basic_config -from minos.common import Aggregate -from minos.common.broker import Event -from minos.common.configuration.config import MinosConfig -from minos.common.logs import log +from aiomisc.log import ( + basic_config, +) +from minos.common import ( + Aggregate, +) +from minos.common.broker import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.logs import ( + log, +) from minos.networks.event import ( EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService, MinosEventServer, ) -from tests.database_testcase import EventHandlerPostgresAsyncTestCase +from tests.database_testcase import ( + EventHandlerPostgresAsyncTestCase, +) @pytest.fixture() diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 48e113d9..ba3515ee 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,7 +8,9 @@ import unittest -from minos.networks import MinosSnapshotDispatcher +from minos.networks import ( + MinosSnapshotDispatcher, +) class TestMinosSnapshotDispatcher(unittest.TestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 2eb7392a..bc932a4d 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -7,16 +7,23 @@ """ import unittest -from unittest.mock import MagicMock - -from aiomisc.service.periodic import PeriodicService -from minos.common.testing import PostgresAsyncTestCase +from unittest.mock import ( + MagicMock, +) +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( - MinosSnapshotService, MinosSnapshotDispatcher, + MinosSnapshotService, +) +from tests.utils import ( + BASE_PATH, ) -from tests.utils import BASE_PATH class TestMinosSnapshotService(PostgresAsyncTestCase): From 72bdfcc6518da4fdacc96987382baf33ccec6b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 15:43:28 +0200 Subject: [PATCH 134/239] ISSUE #? * Restyled with `black` + `isort`. --- minos/networks/event.py | 102 ++++++++++++++++++++---------- tests/services/CqrsTestService.py | 4 +- tests/test_event_manager.py | 58 +++++++++++------ 3 files changed, 110 insertions(+), 54 deletions(-) diff --git a/minos/networks/event.py b/minos/networks/event.py index d771c121..5e43b85d 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -6,21 +6,37 @@ # permission of Clariteia SL. import asyncio -import functools -import typing as t import datetime +import functools import inspect +import typing as t -from aiokafka import AIOKafkaConsumer -from aiomisc import Service -from aiomisc.service.periodic import PeriodicService import aiopg -from minos.common.configuration.config import MinosConfig -from minos.common.importlib import import_module -from minos.common.logs import log - -from minos.networks.exceptions import MinosNetworkException -from minos.common.broker import Event +from aiokafka import ( + AIOKafkaConsumer, +) +from aiomisc import ( + Service, +) +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common.broker import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.importlib import ( + import_module, +) +from minos.common.logs import ( + log, +) + +from minos.networks.exceptions import ( + MinosNetworkException, +) class MinosEventServer(Service): @@ -30,14 +46,18 @@ class MinosEventServer(Service): Consumer for the Broker ( at the moment only Kafka is supported ) """ + __slots__ = "_tasks", "_db_dsn", "_handlers", "_topics", "_kafka_conn_data", "_broker_group_name" def __init__(self, *, conf: MinosConfig, **kwargs: t.Any): self._tasks = set() # type: t.Set[asyncio.Task] - self._db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ - f"password={conf.events.queue.password} host={conf.events.queue.host}" - self._handler = {item.name: {'controller': item.controller, 'action': item.action} - for item in conf.events.items} + self._db_dsn = ( + f"dbname={conf.events.queue.database} user={conf.events.queue.user} " + f"password={conf.events.queue.password} host={conf.events.queue.host}" + ) + self._handler = { + item.name: {"controller": item.controller, "action": item.action} for item in conf.events.items + } self._topics = list(self._handler.keys()) self._kafka_conn_data = f"{conf.events.broker.host}:{conf.events.broker.port}" self._broker_group_name = f"event_{conf.service.name}" @@ -72,7 +92,12 @@ async def event_queue_add(self, topic: str, partition: int, binary: bytes): async with connect.cursor() as cur: await cur.execute( "INSERT INTO event_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", - (topic, partition, binary, datetime.datetime.now(),), + ( + topic, + partition, + binary, + datetime.datetime.now(), + ), ) queue_id = await cur.fetchone() @@ -100,7 +125,6 @@ async def handle_single_message(self, msg): finally: pass - async def handle_message(self, consumer: t.Any): """Message consumer. @@ -113,16 +137,16 @@ async def handle_message(self, consumer: t.Any): async for msg in consumer: await self.handle_single_message(msg) - async def start(self) -> t.Any: self.start_event.set() log.debug("Event Consumer Manager: Started") # start the Service Event Consumer for Kafka - consumer = AIOKafkaConsumer(loop=self.loop, - group_id=self._broker_group_name, - auto_offset_reset="latest", - bootstrap_servers=self._kafka_conn_data, - ) + consumer = AIOKafkaConsumer( + loop=self.loop, + group_id=self._broker_group_name, + auto_offset_reset="latest", + bootstrap_servers=self._kafka_conn_data, + ) await consumer.start() consumer.subscribe(self._topics) @@ -130,10 +154,11 @@ async def start(self) -> t.Any: self.create_task(self.handle_message(consumer)) - async def event_handler_table_creation(conf: MinosConfig): - db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ - f"password={conf.events.queue.password} host={conf.events.queue.host}" + db_dsn = ( + f"dbname={conf.events.queue.database} user={conf.events.queue.user} " + f"password={conf.events.queue.password} host={conf.events.queue.host}" + ) async with aiopg.create_pool(db_dsn) as pool: async with pool.acquire() as connect: async with connect.cursor() as cur: @@ -158,14 +183,18 @@ class MinosEventHandlerPeriodicService(PeriodicService): Periodic Service Event Handler """ - __slots__ = "_db_dsn", "_handlers", "_event_items", "_topics" , "_conf" + + __slots__ = "_db_dsn", "_handlers", "_event_items", "_topics", "_conf" def __init__(self, *, conf: MinosConfig, **kwargs: t.Any): super().__init__(**kwargs) - self._db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ - f"password={conf.events.queue.password} host={conf.events.queue.host}" - self._handlers = {item.name: {'controller': item.controller, 'action': item.action} - for item in conf.events.items} + self._db_dsn = ( + f"dbname={conf.events.queue.database} user={conf.events.queue.user} " + f"password={conf.events.queue.password} host={conf.events.queue.host}" + ) + self._handlers = { + item.name: {"controller": item.controller, "action": item.action} for item in conf.events.items + } self._event_items = conf.events.items self._topics = list(self._handlers.keys()) self._conf = conf @@ -194,8 +223,9 @@ def get_event_handler(self, topic: str) -> t.Callable: class_method = getattr(instance_class, action) return class_method - raise MinosNetworkException(f"topic {topic} have no controller/action configured, " - f"please review th configuration file") + raise MinosNetworkException( + f"topic {topic} have no controller/action configured, " f"please review th configuration file" + ) async def event_queue_checker(self): """Event Queue Checker and dispatcher. @@ -209,8 +239,10 @@ async def event_queue_checker(self): Raises: Exception: An error occurred inserting record. """ - db_dsn = f"dbname={self._conf.events.queue.database} user={self._conf.events.queue.user} " \ - f"password={self._conf.events.queue.password} host={self._conf.events.queue.host}" + db_dsn = ( + f"dbname={self._conf.events.queue.database} user={self._conf.events.queue.user} " + f"password={self._conf.events.queue.password} host={self._conf.events.queue.host}" + ) async with aiopg.create_pool(db_dsn) as pool: async with pool.acquire() as connect: async with connect.cursor() as cur: diff --git a/tests/services/CqrsTestService.py b/tests/services/CqrsTestService.py index dfbe0293..cfaf0693 100644 --- a/tests/services/CqrsTestService.py +++ b/tests/services/CqrsTestService.py @@ -1,4 +1,6 @@ -from minos.common.broker import Event +from minos.common import ( + Event, +) class CqrsService(object): diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 2b7ad25a..5b5d3638 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -1,28 +1,51 @@ import asyncio import logging - -import pytest -import string -from aiokafka import AIOKafkaProducer, AIOKafkaConsumer import random +import string -from minos.common.logs import log -from aiomisc.log import basic_config -from minos.common.configuration.config import MinosConfig -from minos.common import Aggregate -from minos.common.broker import Event -from minos.networks.event import MinosEventServer, EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService -from tests.database_testcase import EventHandlerPostgresAsyncTestCase +import pytest +from aiokafka import ( + AIOKafkaConsumer, + AIOKafkaProducer, +) +from aiomisc.log import ( + basic_config, +) +from minos.common import ( + Aggregate, +) +from minos.common.broker import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.logs import ( + log, +) + +from minos.networks.event import ( + EventHandlerDatabaseInitializer, + MinosEventHandlerPeriodicService, + MinosEventServer, +) +from tests.database_testcase import ( + EventHandlerPostgresAsyncTestCase, +) @pytest.fixture() def config(): - return MinosConfig(path='./tests/test_config.yml') + return MinosConfig(path="./tests/test_config.yml") @pytest.fixture() def services(config): - return [EventHandlerDatabaseInitializer(config=config), MinosEventServer(conf=config), MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config)] + return [ + EventHandlerDatabaseInitializer(config=config), + MinosEventServer(conf=config), + MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config), + ] class AggregateTest(Aggregate): @@ -69,7 +92,7 @@ async def test_get_event_handler(self): cls = m.get_event_handler(topic="TicketAdded") result = await cls(topic="TicketAdded", event=event_instance) - assert result == 'request_added' + assert result == "request_added" async def test_event_queue_checker(self): model = AggregateTest(test_id=1, test=2, id=1, version=1) @@ -105,12 +128,12 @@ async def test_event_queue_checker(self): async def test_producer_kafka(config, loop): - #basic_config( + # basic_config( # level=logging.INFO, # buffered=True, # log_format='color', # flush_interval=2 - #) + # ) kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" producer = AIOKafkaProducer(loop=loop, bootstrap_servers=kafka_conn_data) # Get cluster layout and topic/partition allocation @@ -131,6 +154,7 @@ async def test_producer_kafka(config, loop): await producer.stop() + """ async def test_consumer_kafka(config,loop): handler = {item.name: {'controller': item.controller, 'action': item.action} @@ -156,5 +180,3 @@ async def test_consumer_kafka(config,loop): await consumer.stop() """ - - From f582eeed143d0e1ad87257bb6610e6b1be19c9bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 15:46:55 +0200 Subject: [PATCH 135/239] ISSUE #? * Restyled with `black` + `isort`. --- minos/networks/event.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/minos/networks/event.py b/minos/networks/event.py index 5e43b85d..1c21e547 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -92,12 +92,7 @@ async def event_queue_add(self, topic: str, partition: int, binary: bytes): async with connect.cursor() as cur: await cur.execute( "INSERT INTO event_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", - ( - topic, - partition, - binary, - datetime.datetime.now(), - ), + (topic, partition, binary, datetime.datetime.now(),), ) queue_id = await cur.fetchone() From 72447d87fa827ad2852f50a2ac0fb40f9e4f80c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 15:47:58 +0200 Subject: [PATCH 136/239] ISSUE #? * Restyled with `black` + `isort`. --- minos/networks/event.py | 1 - 1 file changed, 1 deletion(-) diff --git a/minos/networks/event.py b/minos/networks/event.py index 1c21e547..37f3c435 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -33,7 +33,6 @@ from minos.common.logs import ( log, ) - from minos.networks.exceptions import ( MinosNetworkException, ) From 8416f239cfaabca733a827b89c838bef7cbb12d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 16:09:14 +0200 Subject: [PATCH 137/239] ISSUE #51 * Add tests for `minos.common.MinosSnapshotEntry`. --- .../test_snapshots/test_entries.py | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 tests/test_networks/test_snapshots/test_entries.py diff --git a/tests/test_networks/test_snapshots/test_entries.py b/tests/test_networks/test_snapshots/test_entries.py new file mode 100644 index 00000000..7b6028c0 --- /dev/null +++ b/tests/test_networks/test_snapshots/test_entries.py @@ -0,0 +1,79 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +import unittest +from datetime import datetime + +from minos.networks import ( + MinosSnapshotEntry, +) +from tests.aggregate_classes import Car + + +class TestMinosSnapshotEntry(unittest.TestCase): + def test_constructor(self): + entry = MinosSnapshotEntry(1234, "example.Car", 0, bytes("car", "utf-8")) + self.assertEqual(1234, entry.aggregate_id) + self.assertEqual("example.Car", entry.aggregate_name) + self.assertEqual(0, entry.version) + self.assertEqual(bytes("car", "utf-8"), entry.data) + self.assertEqual(None, entry.created_at) + self.assertEqual(None, entry.updated_at) + + def test_constructor_extended(self): + entry = MinosSnapshotEntry( + 1234, "example.Car", 0, bytes("car", "utf-8"), datetime(2020, 1, 10, 4, 23), datetime(2020, 1, 10, 4, 25) + ) + self.assertEqual(1234, entry.aggregate_id) + self.assertEqual("example.Car", entry.aggregate_name) + self.assertEqual(0, entry.version) + self.assertEqual(bytes("car", "utf-8"), entry.data) + self.assertEqual(datetime(2020, 1, 10, 4, 23), entry.created_at) + self.assertEqual(datetime(2020, 1, 10, 4, 25), entry.updated_at) + + def test_from_aggregate(self): + car = Car(1, 1, 3, "blue") + entry = MinosSnapshotEntry.from_aggregate(car) + self.assertEqual(car.id, entry.aggregate_id) + self.assertEqual(car.classname, entry.aggregate_name) + self.assertEqual(car.version, entry.version) + self.assertIsInstance(entry.data, bytes) + self.assertEqual(None, entry.created_at) + self.assertEqual(None, entry.updated_at) + + def test_equals(self): + a = MinosSnapshotEntry(1234, "example.Car", 0, bytes("car", "utf-8")) + b = MinosSnapshotEntry(1234, "example.Car", 0, bytes("car", "utf-8")) + self.assertEqual(a, b) + + def test_hash(self): + entry = MinosSnapshotEntry(1234, "example.Car", 0, bytes("car", "utf-8")) + self.assertIsInstance(hash(entry), int) + + def test_aggregate_cls(self): + car = Car(1, 1, 3, "blue") + entry = MinosSnapshotEntry.from_aggregate(car) + self.assertEqual(Car, entry.aggregate_cls) + + def test_aggregate(self): + car = Car(1, 1, 3, "blue") + entry = MinosSnapshotEntry.from_aggregate(car) + self.assertEqual(car, entry.aggregate) + + def test_repr(self): + entry = MinosSnapshotEntry( + 1234, "example.Car", 0, bytes("car", "utf-8"), datetime(2020, 1, 10, 4, 23), datetime(2020, 1, 10, 4, 25), + ) + expected = ( + "MinosSnapshotEntry(aggregate_id=1234, aggregate_name='example.Car', version=0, data=b'car', " + "created_at=datetime.datetime(2020, 1, 10, 4, 23), updated_at=datetime.datetime(2020, 1, 10, 4, 25))" + ) + self.assertEqual(expected, repr(entry)) + + +if __name__ == "__main__": + unittest.main() From 637fe9588bb00087c4de2071cbecbb7185077001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 16:11:53 +0200 Subject: [PATCH 138/239] ISSUE #51 * Run black + isort. --- tests/test_networks/test_snapshots/test_entries.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_networks/test_snapshots/test_entries.py b/tests/test_networks/test_snapshots/test_entries.py index 7b6028c0..7c752783 100644 --- a/tests/test_networks/test_snapshots/test_entries.py +++ b/tests/test_networks/test_snapshots/test_entries.py @@ -6,12 +6,16 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ import unittest -from datetime import datetime +from datetime import ( + datetime, +) from minos.networks import ( MinosSnapshotEntry, ) -from tests.aggregate_classes import Car +from tests.aggregate_classes import ( + Car, +) class TestMinosSnapshotEntry(unittest.TestCase): From c41924330b034d7983c12dcab40998cb696c9857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 16:15:59 +0200 Subject: [PATCH 139/239] ISSUE #51 * Add missing module. --- tests/aggregate_classes.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tests/aggregate_classes.py diff --git a/tests/aggregate_classes.py b/tests/aggregate_classes.py new file mode 100644 index 00000000..501537a6 --- /dev/null +++ b/tests/aggregate_classes.py @@ -0,0 +1,31 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from typing import ( + Optional, +) + +from minos.common import ( + Aggregate, + ModelRef, +) + + +class Owner(Aggregate): + """Aggregate ``Owner`` class for testing purposes.""" + + name: str + surname: str + age: Optional[int] + + +class Car(Aggregate): + """Aggregate ``Car`` class for testing purposes.""" + + doors: int + color: str + owner: Optional[list[ModelRef[Owner]]] From 55fcd4fed27517b2c70e02e41150229ddcba121c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 16:31:54 +0200 Subject: [PATCH 140/239] ISSUE #51 * Increase coverage. --- minos/networks/snapshots/dispatchers.py | 5 ++-- .../test_snapshots/test_dispatchers.py | 27 ++++++++++++++++++- .../test_snapshots/test_services.py | 12 +++++---- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 57cea268..3cbe7569 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -19,6 +19,7 @@ from minos.common import ( Aggregate, MinosConfig, + MinosConfigException, MinosRepositoryAction, MinosRepositoryEntry, MinosSetup, @@ -55,7 +56,7 @@ def __init__( self.offset = offset @classmethod - def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosSnapshotDispatcher]: + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> MinosSnapshotDispatcher: """Build a new Snapshot Dispatcher from config. :param args: Additional positional arguments. :param config: Config instance. If `None` is provided, default config is chosen. @@ -65,7 +66,7 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi if config is None: config = MinosConfig.get_default() if config is None: - return None + raise MinosConfigException("The config object must be setup.") # noinspection PyProtectedMember return cls(*args, **config.repository._asdict(), **kwargs) diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index ba3515ee..3e30dea0 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,15 +8,40 @@ import unittest +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) + from minos.networks import ( MinosSnapshotDispatcher, ) +from tests.utils import ( + BASE_PATH, +) + +class TestMinosSnapshotDispatcher(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" -class TestMinosSnapshotDispatcher(unittest.TestCase): def test_type(self): self.assertTrue(issubclass(MinosSnapshotDispatcher, object)) + def test_from_config(self): + dispatcher = MinosSnapshotDispatcher.from_config(config=self.config) + self.assertEqual(self.config.repository.host, dispatcher.host) + self.assertEqual(self.config.repository.port, dispatcher.port) + self.assertEqual(self.config.repository.database, dispatcher.database) + self.assertEqual(self.config.repository.user, dispatcher.user) + self.assertEqual(self.config.repository.password, dispatcher.password) + self.assertEqual(0, dispatcher.offset) + + def test_from_config_raises(self): + with self.assertRaises(MinosConfigException): + MinosSnapshotDispatcher.from_config() + if __name__ == "__main__": unittest.main() diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index bc932a4d..10010655 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -14,6 +14,7 @@ from aiomisc.service.periodic import ( PeriodicService, ) +from minos.common import MinosConfigException from minos.common.testing import ( PostgresAsyncTestCase, ) @@ -30,12 +31,13 @@ class TestMinosSnapshotService(PostgresAsyncTestCase): CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" def test_is_instance(self): - service = MinosSnapshotService(interval=10) - self.assertIsInstance(service, PeriodicService) + with self.config: + service = MinosSnapshotService(interval=10) + self.assertIsInstance(service, PeriodicService) - def test_dispatcher_empty(self): - service = MinosSnapshotService(interval=10) - self.assertIsNone(service.dispatcher) + def test_dispatcher_config_raises(self): + with self.assertRaises(MinosConfigException): + MinosSnapshotService(interval=10) def test_dispatcher_config(self): service = MinosSnapshotService(interval=10, config=self.config) From 9f7e1c3d6d3d03d260a4717d5c9859f7ee90e455 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 14:32:06 +0000 Subject: [PATCH 141/239] Restyled by black --- minos/networks/__init__.py | 4 +--- minos/networks/snapshots/__init__.py | 12 +++------- minos/networks/snapshots/dispatchers.py | 8 ++----- minos/networks/snapshots/entries.py | 8 ++----- minos/networks/snapshots/services.py | 20 +++++----------- tests/aggregate_classes.py | 4 +--- tests/database_testcase.py | 8 ++----- tests/test_event_manager.py | 24 +++++-------------- .../test_snapshots/test_dispatchers.py | 18 ++++---------- .../test_snapshots/test_entries.py | 12 +++------- .../test_snapshots/test_services.py | 16 ++++--------- 11 files changed, 35 insertions(+), 99 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index b90383e8..2c5d2e47 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,9 +13,7 @@ MinosQueueDispatcher, MinosQueueService, ) -from .exceptions import ( - MinosNetworkException, -) +from .exceptions import MinosNetworkException from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotEntry, diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index ab7b0571..29ac484a 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -5,12 +5,6 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatchers import ( - MinosSnapshotDispatcher, -) -from .entries import ( - MinosSnapshotEntry, -) -from .services import ( - MinosSnapshotService, -) +from .dispatchers import MinosSnapshotDispatcher +from .entries import MinosSnapshotEntry +from .services import MinosSnapshotService diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 3cbe7569..1009baf0 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NoReturn, @@ -26,9 +24,7 @@ import_module, ) -from .entries import ( - MinosSnapshotEntry, -) +from .entries import MinosSnapshotEntry class MinosSnapshotDispatcher(MinosSetup): diff --git a/minos/networks/snapshots/entries.py b/minos/networks/snapshots/entries.py index 8fd771f1..fa5bc0c7 100644 --- a/minos/networks/snapshots/entries.py +++ b/minos/networks/snapshots/entries.py @@ -6,13 +6,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations -from datetime import ( - datetime, -) +from datetime import datetime from typing import ( Iterable, Optional, diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 75cec27c..2969ea1f 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,20 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - Any, -) - -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) - -from .dispatchers import ( - MinosSnapshotDispatcher, -) +from typing import Any + +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig + +from .dispatchers import MinosSnapshotDispatcher class MinosSnapshotService(PeriodicService): diff --git a/tests/aggregate_classes.py b/tests/aggregate_classes.py index 501537a6..9ef7ea89 100644 --- a/tests/aggregate_classes.py +++ b/tests/aggregate_classes.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - Optional, -) +from typing import Optional from minos.common import ( Aggregate, diff --git a/tests/database_testcase.py b/tests/database_testcase.py index a6d2148e..fd87d0fb 100644 --- a/tests/database_testcase.py +++ b/tests/database_testcase.py @@ -7,12 +7,8 @@ import unittest import aiopg -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.networks.event import ( - event_handler_table_creation, -) +from minos.common.configuration.config import MinosConfig +from minos.networks.event import event_handler_table_creation class EventHandlerPostgresAsyncTestCase(unittest.IsolatedAsyncioTestCase): diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index f0bc87c4..b3cbd37c 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -8,29 +8,17 @@ AIOKafkaConsumer, AIOKafkaProducer, ) -from aiomisc.log import ( - basic_config, -) -from minos.common import ( - Aggregate, -) -from minos.common.broker import ( - Event, -) -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.common.logs import ( - log, -) +from aiomisc.log import basic_config +from minos.common import Aggregate +from minos.common.broker import Event +from minos.common.configuration.config import MinosConfig +from minos.common.logs import log from minos.networks.event import ( EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService, MinosEventServer, ) -from tests.database_testcase import ( - EventHandlerPostgresAsyncTestCase, -) +from tests.database_testcase import EventHandlerPostgresAsyncTestCase @pytest.fixture() diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 3e30dea0..35d32b75 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,19 +8,11 @@ import unittest -from minos.common import ( - MinosConfigException, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) - -from minos.networks import ( - MinosSnapshotDispatcher, -) -from tests.utils import ( - BASE_PATH, -) +from minos.common import MinosConfigException +from minos.common.testing import PostgresAsyncTestCase + +from minos.networks import MinosSnapshotDispatcher +from tests.utils import BASE_PATH class TestMinosSnapshotDispatcher(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_snapshots/test_entries.py b/tests/test_networks/test_snapshots/test_entries.py index 7c752783..b9e8defb 100644 --- a/tests/test_networks/test_snapshots/test_entries.py +++ b/tests/test_networks/test_snapshots/test_entries.py @@ -6,16 +6,10 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ import unittest -from datetime import ( - datetime, -) +from datetime import datetime -from minos.networks import ( - MinosSnapshotEntry, -) -from tests.aggregate_classes import ( - Car, -) +from minos.networks import MinosSnapshotEntry +from tests.aggregate_classes import Car class TestMinosSnapshotEntry(unittest.TestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 10010655..28879a42 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -7,24 +7,16 @@ """ import unittest -from unittest.mock import ( - MagicMock, -) +from unittest.mock import MagicMock -from aiomisc.service.periodic import ( - PeriodicService, -) +from aiomisc.service.periodic import PeriodicService from minos.common import MinosConfigException -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosSnapshotDispatcher, MinosSnapshotService, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class TestMinosSnapshotService(PostgresAsyncTestCase): From 17d212c12bfa16a3c3872d380c006d40a354d5cd Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 29 Apr 2021 14:32:07 +0000 Subject: [PATCH 142/239] Restyled by isort --- minos/networks/__init__.py | 4 +++- minos/networks/snapshots/__init__.py | 12 +++++++--- minos/networks/snapshots/dispatchers.py | 8 +++++-- minos/networks/snapshots/entries.py | 8 +++++-- minos/networks/snapshots/services.py | 20 +++++++++++----- tests/aggregate_classes.py | 4 +++- tests/database_testcase.py | 8 +++++-- tests/test_event_manager.py | 24 ++++++++++++++----- .../test_snapshots/test_dispatchers.py | 17 +++++++++---- .../test_snapshots/test_entries.py | 12 +++++++--- .../test_snapshots/test_services.py | 20 ++++++++++++---- 11 files changed, 101 insertions(+), 36 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 2c5d2e47..b90383e8 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,7 +13,9 @@ MinosQueueDispatcher, MinosQueueService, ) -from .exceptions import MinosNetworkException +from .exceptions import ( + MinosNetworkException, +) from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotEntry, diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py index 29ac484a..ab7b0571 100644 --- a/minos/networks/snapshots/__init__.py +++ b/minos/networks/snapshots/__init__.py @@ -5,6 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatchers import MinosSnapshotDispatcher -from .entries import MinosSnapshotEntry -from .services import MinosSnapshotService +from .dispatchers import ( + MinosSnapshotDispatcher, +) +from .entries import ( + MinosSnapshotEntry, +) +from .services import ( + MinosSnapshotService, +) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 1009baf0..3cbe7569 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( NoReturn, @@ -24,7 +26,9 @@ import_module, ) -from .entries import MinosSnapshotEntry +from .entries import ( + MinosSnapshotEntry, +) class MinosSnapshotDispatcher(MinosSetup): diff --git a/minos/networks/snapshots/entries.py b/minos/networks/snapshots/entries.py index fa5bc0c7..8fd771f1 100644 --- a/minos/networks/snapshots/entries.py +++ b/minos/networks/snapshots/entries.py @@ -6,9 +6,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) -from datetime import datetime +from datetime import ( + datetime, +) from typing import ( Iterable, Optional, diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 2969ea1f..75cec27c 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,12 +5,20 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import Any - -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig - -from .dispatchers import MinosSnapshotDispatcher +from typing import ( + Any, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosSnapshotDispatcher, +) class MinosSnapshotService(PeriodicService): diff --git a/tests/aggregate_classes.py b/tests/aggregate_classes.py index 9ef7ea89..501537a6 100644 --- a/tests/aggregate_classes.py +++ b/tests/aggregate_classes.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import Optional +from typing import ( + Optional, +) from minos.common import ( Aggregate, diff --git a/tests/database_testcase.py b/tests/database_testcase.py index fd87d0fb..a6d2148e 100644 --- a/tests/database_testcase.py +++ b/tests/database_testcase.py @@ -7,8 +7,12 @@ import unittest import aiopg -from minos.common.configuration.config import MinosConfig -from minos.networks.event import event_handler_table_creation +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.networks.event import ( + event_handler_table_creation, +) class EventHandlerPostgresAsyncTestCase(unittest.IsolatedAsyncioTestCase): diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index b3cbd37c..f0bc87c4 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -8,17 +8,29 @@ AIOKafkaConsumer, AIOKafkaProducer, ) -from aiomisc.log import basic_config -from minos.common import Aggregate -from minos.common.broker import Event -from minos.common.configuration.config import MinosConfig -from minos.common.logs import log +from aiomisc.log import ( + basic_config, +) +from minos.common import ( + Aggregate, +) +from minos.common.broker import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.logs import ( + log, +) from minos.networks.event import ( EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService, MinosEventServer, ) -from tests.database_testcase import EventHandlerPostgresAsyncTestCase +from tests.database_testcase import ( + EventHandlerPostgresAsyncTestCase, +) @pytest.fixture() diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 35d32b75..ab3edcad 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -8,11 +8,18 @@ import unittest -from minos.common import MinosConfigException -from minos.common.testing import PostgresAsyncTestCase - -from minos.networks import MinosSnapshotDispatcher -from tests.utils import BASE_PATH +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosSnapshotDispatcher, +) +from tests.utils import ( + BASE_PATH, +) class TestMinosSnapshotDispatcher(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_snapshots/test_entries.py b/tests/test_networks/test_snapshots/test_entries.py index b9e8defb..7c752783 100644 --- a/tests/test_networks/test_snapshots/test_entries.py +++ b/tests/test_networks/test_snapshots/test_entries.py @@ -6,10 +6,16 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ import unittest -from datetime import datetime +from datetime import ( + datetime, +) -from minos.networks import MinosSnapshotEntry -from tests.aggregate_classes import Car +from minos.networks import ( + MinosSnapshotEntry, +) +from tests.aggregate_classes import ( + Car, +) class TestMinosSnapshotEntry(unittest.TestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 28879a42..3c131d95 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -7,16 +7,26 @@ """ import unittest -from unittest.mock import MagicMock +from unittest.mock import ( + MagicMock, +) -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfigException -from minos.common.testing import PostgresAsyncTestCase +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosSnapshotDispatcher, MinosSnapshotService, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class TestMinosSnapshotService(PostgresAsyncTestCase): From fb93ffcf4a38bf7ab8cb48f0ead0422c34a44230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 16:49:15 +0200 Subject: [PATCH 143/239] ISSUE #51 * Increase coverage with dispatch and select. --- minos/networks/snapshots/dispatchers.py | 56 +++++++++++-------- .../test_snapshots/test_dispatchers.py | 41 +++++++++++++- 2 files changed, 73 insertions(+), 24 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 3cbe7569..015bdc41 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -12,7 +12,7 @@ from typing import ( NoReturn, Optional, - Type, + Type, Any, ) import aiopg @@ -74,9 +74,19 @@ async def _setup(self) -> NoReturn: await self._create_broker_table() async def _create_broker_table(self) -> NoReturn: - async with self._connection() as connect: - async with connect.cursor() as cur: - await cur.execute(_CREATE_TABLE_QUERY) + await self._submit_sql(_CREATE_TABLE_QUERY, fetch=False) + + # noinspection PyUnusedLocal + async def select(self, *args, **kwargs) -> list[MinosSnapshotEntry]: + """TODO + + :param args: TODO + :param kwargs: TODO + :return: + """ + response = await self._submit_sql(_SELECT_ALL_ENTRIES_QUERY) + entries = [MinosSnapshotEntry(*row) for row in response] + return entries async def dispatch(self) -> NoReturn: """TODO""" @@ -85,11 +95,8 @@ async def dispatch(self) -> NoReturn: @property async def _new_entries(self): - async with self._connection() as connect: - async with connect.cursor() as cursor: - await cursor.execute(_SELECT_EVENT_ENTRIES_QUERY, (self.offset,)) - async for raw in cursor: - yield MinosRepositoryEntry(*raw) + for raw in await self._submit_sql(_SELECT_EVENT_ENTRIES_QUERY, (self.offset,)): + yield MinosRepositoryEntry(*raw) async def _dispatch_one(self, event_entry: MinosRepositoryEntry) -> Optional[MinosSnapshotEntry]: if event_entry.action is MinosRepositoryAction.DELETE: @@ -100,9 +107,7 @@ async def _dispatch_one(self, event_entry: MinosRepositoryEntry) -> Optional[Min async def _submit_delete(self, entry: MinosRepositoryEntry) -> NoReturn: params = {"aggregate_id": entry.aggregate_id, "aggregate_name": entry.aggregate_name} - async with self._connection() as connect: - async with connect.cursor() as cursor: - await cursor.execute(_DELETE_ONE_SNAPSHOT_ENTRY_QUERY, params) + await self._submit_sql(_DELETE_ONE_SNAPSHOT_ENTRY_QUERY, params, fetch=False) async def _build_instance(self, event_entry: MinosRepositoryEntry) -> Aggregate: # noinspection PyTypeChecker @@ -124,13 +129,8 @@ async def _select_one_aggregate(self, aggregate_id: int, aggregate_name: str) -> return snapshot_entry.aggregate async def _select_one(self, aggregate_id: int, aggregate_name: str) -> MinosSnapshotEntry: - async with self._connection() as connect: - async with connect.cursor() as cursor: - await cursor.execute(_SELECT_ONE_SNAPSHOT_ENTRY_QUERY, (aggregate_id, aggregate_name)) - raw = await cursor.fetchone() - if raw is None: - raise Exception() - return MinosSnapshotEntry(aggregate_id, aggregate_name, *raw) + raw = (await self._submit_sql(_SELECT_ONE_SNAPSHOT_ENTRY_QUERY, (aggregate_id, aggregate_name)))[0] + return MinosSnapshotEntry(aggregate_id, aggregate_name, *raw) async def _submit_instance(self, aggregate: Aggregate) -> MinosSnapshotEntry: snapshot_entry = MinosSnapshotEntry.from_aggregate(aggregate) @@ -144,15 +144,20 @@ async def _submit_update_or_create(self, entry: MinosSnapshotEntry) -> MinosSnap "version": entry.version, "data": entry.data, } - async with self._connection() as connect: - async with connect.cursor() as cursor: - await cursor.execute(_INSERT_ONE_SNAPSHOT_ENTRY_QUERY, params) - response = await cursor.fetchone() + response = (await self._submit_sql(_INSERT_ONE_SNAPSHOT_ENTRY_QUERY, params))[0] entry.created_at, entry.updated_at = response return entry + async def _submit_sql(self, query: str, *args, fetch: bool = True, **kwargs) -> Optional[list[tuple[Any, ...]]]: + async with self._connection() as connect: + async with connect.cursor() as cursor: + await cursor.execute(query, *args, **kwargs) + if not fetch: + return + return await cursor.fetchall() + def _connection(self): return aiopg.connect( host=self.host, port=self.port, dbname=self.database, user=self.user, password=self.password @@ -171,6 +176,11 @@ def _connection(self): ); """.strip() +_SELECT_ALL_ENTRIES_QUERY = """ +SELECT aggregate_id, aggregate_name, version, data, created_at, updated_at +FROM snapshot; +""".strip() + _SELECT_EVENT_ENTRIES_QUERY = """ SELECT aggregate_id, aggregate_name, version, data, id, action FROM events diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 3e30dea0..5ecf890e 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -7,16 +7,22 @@ """ import unittest +from datetime import datetime from minos.common import ( MinosConfigException, + MinosRepositoryEntry, + PostgreSqlMinosRepository, ) from minos.common.testing import ( PostgresAsyncTestCase, ) from minos.networks import ( - MinosSnapshotDispatcher, + MinosSnapshotDispatcher, MinosSnapshotEntry, +) +from tests.aggregate_classes import ( + Car, ) from tests.utils import ( BASE_PATH, @@ -42,6 +48,39 @@ def test_from_config_raises(self): with self.assertRaises(MinosConfigException): MinosSnapshotDispatcher.from_config() + async def test_dispatch_select(self): + await self._populate() + with self.config: + dispatcher = MinosSnapshotDispatcher.from_config() + await dispatcher.setup() + await dispatcher.dispatch() + observed = await dispatcher.select() + + expected = [ + MinosSnapshotEntry.from_aggregate(Car(2, 2, 3, "blue")), + MinosSnapshotEntry.from_aggregate(Car(3, 1, 3, "blue")), + ] + self.assertEqual(len(expected), len(observed)) + for exp, obs in zip(expected, observed): + self.assertEqual(exp.aggregate, obs.aggregate) + self.assertIsInstance(obs.created_at, datetime) + self.assertIsInstance(obs.updated_at, datetime) + + async def _populate(self): + with self.config: + car = Car(1, 1, 3, "blue") + # noinspection PyTypeChecker + aggregate_name: str = car.classname + repository = PostgreSqlMinosRepository.from_config() + await repository.setup() + await repository.insert(MinosRepositoryEntry(1, aggregate_name, 1, car.avro_bytes)) + await repository.update(MinosRepositoryEntry(1, aggregate_name, 2, car.avro_bytes)) + await repository.insert(MinosRepositoryEntry(2, aggregate_name, 1, car.avro_bytes)) + await repository.update(MinosRepositoryEntry(1, aggregate_name, 3, car.avro_bytes)) + await repository.delete(MinosRepositoryEntry(1, aggregate_name, 4)) + await repository.update(MinosRepositoryEntry(2, aggregate_name, 2, car.avro_bytes)) + await repository.insert(MinosRepositoryEntry(3, aggregate_name, 1, car.avro_bytes)) + if __name__ == "__main__": unittest.main() From a2d9ca29652c64327cb67db44f1f670eb90c32fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 16:50:16 +0200 Subject: [PATCH 144/239] ISSUE #51 * Run black + isort. --- minos/networks/snapshots/dispatchers.py | 3 ++- tests/test_networks/test_snapshots/test_dispatchers.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 015bdc41..903cd356 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -10,9 +10,10 @@ ) from typing import ( + Any, NoReturn, Optional, - Type, Any, + Type, ) import aiopg diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 5ecf890e..0fd61c9e 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -7,7 +7,9 @@ """ import unittest -from datetime import datetime +from datetime import ( + datetime, +) from minos.common import ( MinosConfigException, @@ -19,7 +21,8 @@ ) from minos.networks import ( - MinosSnapshotDispatcher, MinosSnapshotEntry, + MinosSnapshotDispatcher, + MinosSnapshotEntry, ) from tests.aggregate_classes import ( Car, From 980e9ea62bb1caa0eece6c0841551f72b9d14a14 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 29 Apr 2021 17:06:36 +0200 Subject: [PATCH 145/239] Ato delete table and produce kafka data EventHandler Refactor code and add more tests #87 --- tests/database_testcase.py | 3 --- tests/test_event_manager.py | 7 ++++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/database_testcase.py b/tests/database_testcase.py index 5d0f4096..aeb60de2 100644 --- a/tests/database_testcase.py +++ b/tests/database_testcase.py @@ -57,11 +57,8 @@ async def _database(self): return await aiopg.connect(db_dsn) async def asyncTearDown(self): - pass - """ database = await self._database() async with database as connection: async with connection.cursor() as cursor: template = "DROP TABLE IF EXISTS event_queue" await cursor.execute(template) - """ diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 5b5d3638..6b7b2fe5 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -126,7 +126,7 @@ async def test_event_queue_checker(self): assert records[0] == 0 -async def test_producer_kafka(config, loop): +async def kafka_producer(config, loop): # basic_config( # level=logging.INFO, @@ -155,8 +155,9 @@ async def test_producer_kafka(config, loop): await producer.stop() -""" + async def test_consumer_kafka(config,loop): + await kafka_producer(config, loop) handler = {item.name: {'controller': item.controller, 'action': item.action} for item in config.events.items} topics = list(handler.keys()) @@ -179,4 +180,4 @@ async def test_consumer_kafka(config,loop): await m.handle_single_message(msg) await consumer.stop() -""" + From f8b0959b7c8fe8544065f6cdc581a79c2d8256d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Thu, 29 Apr 2021 17:17:14 +0200 Subject: [PATCH 146/239] ISSUE #51 * Improve scalability using generators instead of fetchall(). --- minos/networks/snapshots/dispatchers.py | 31 +++++++++++++++---------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 903cd356..862cb365 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -10,7 +10,7 @@ ) from typing import ( - Any, + Generator, NoReturn, Optional, Type, @@ -75,7 +75,7 @@ async def _setup(self) -> NoReturn: await self._create_broker_table() async def _create_broker_table(self) -> NoReturn: - await self._submit_sql(_CREATE_TABLE_QUERY, fetch=False) + await self._submit_sql(_CREATE_TABLE_QUERY) # noinspection PyUnusedLocal async def select(self, *args, **kwargs) -> list[MinosSnapshotEntry]: @@ -85,8 +85,8 @@ async def select(self, *args, **kwargs) -> list[MinosSnapshotEntry]: :param kwargs: TODO :return: """ - response = await self._submit_sql(_SELECT_ALL_ENTRIES_QUERY) - entries = [MinosSnapshotEntry(*row) for row in response] + response = self._submit_and_iter_sql(_SELECT_ALL_ENTRIES_QUERY) + entries = [MinosSnapshotEntry(*row) async for row in response] return entries async def dispatch(self) -> NoReturn: @@ -96,7 +96,7 @@ async def dispatch(self) -> NoReturn: @property async def _new_entries(self): - for raw in await self._submit_sql(_SELECT_EVENT_ENTRIES_QUERY, (self.offset,)): + async for raw in self._submit_and_iter_sql(_SELECT_EVENT_ENTRIES_QUERY, (self.offset,)): yield MinosRepositoryEntry(*raw) async def _dispatch_one(self, event_entry: MinosRepositoryEntry) -> Optional[MinosSnapshotEntry]: @@ -108,7 +108,7 @@ async def _dispatch_one(self, event_entry: MinosRepositoryEntry) -> Optional[Min async def _submit_delete(self, entry: MinosRepositoryEntry) -> NoReturn: params = {"aggregate_id": entry.aggregate_id, "aggregate_name": entry.aggregate_name} - await self._submit_sql(_DELETE_ONE_SNAPSHOT_ENTRY_QUERY, params, fetch=False) + await self._submit_sql(_DELETE_ONE_SNAPSHOT_ENTRY_QUERY, params) async def _build_instance(self, event_entry: MinosRepositoryEntry) -> Aggregate: # noinspection PyTypeChecker @@ -130,7 +130,7 @@ async def _select_one_aggregate(self, aggregate_id: int, aggregate_name: str) -> return snapshot_entry.aggregate async def _select_one(self, aggregate_id: int, aggregate_name: str) -> MinosSnapshotEntry: - raw = (await self._submit_sql(_SELECT_ONE_SNAPSHOT_ENTRY_QUERY, (aggregate_id, aggregate_name)))[0] + raw = await self._submit_and_fetchone_sql(_SELECT_ONE_SNAPSHOT_ENTRY_QUERY, (aggregate_id, aggregate_name)) return MinosSnapshotEntry(aggregate_id, aggregate_name, *raw) async def _submit_instance(self, aggregate: Aggregate) -> MinosSnapshotEntry: @@ -145,19 +145,26 @@ async def _submit_update_or_create(self, entry: MinosSnapshotEntry) -> MinosSnap "version": entry.version, "data": entry.data, } - response = (await self._submit_sql(_INSERT_ONE_SNAPSHOT_ENTRY_QUERY, params))[0] + response = await self._submit_and_fetchone_sql(_INSERT_ONE_SNAPSHOT_ENTRY_QUERY, params) entry.created_at, entry.updated_at = response return entry - async def _submit_sql(self, query: str, *args, fetch: bool = True, **kwargs) -> Optional[list[tuple[Any, ...]]]: + async def _submit_and_fetchone_sql(self, *args, **kwargs) -> tuple: + return await self._submit_and_iter_sql(*args, **kwargs).__anext__() + + async def _submit_and_iter_sql(self, query: str, *args, **kwargs) -> Generator[tuple, None, None]: + async with self._connection() as connect: + async with connect.cursor() as cursor: + await cursor.execute(query, *args, **kwargs) + async for row in cursor: + yield row + + async def _submit_sql(self, query: str, *args, **kwargs) -> NoReturn: async with self._connection() as connect: async with connect.cursor() as cursor: await cursor.execute(query, *args, **kwargs) - if not fetch: - return - return await cursor.fetchall() def _connection(self): return aiopg.connect( From 7a37e30ab82146f268fe2d26c907262ec1b1b883 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 29 Apr 2021 17:20:02 +0200 Subject: [PATCH 147/239] EventHandler tests EventHandler Refactor code and add more tests #87 --- minos/networks/event.py | 5 ++--- tests/test_event_manager.py | 7 ++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/minos/networks/event.py b/minos/networks/event.py index 37f3c435..08fd2115 100644 --- a/minos/networks/event.py +++ b/minos/networks/event.py @@ -115,7 +115,8 @@ async def handle_single_message(self, msg): # check if the event binary string is well formatted try: Event.from_avro_bytes(msg.value) - await self.event_queue_add(msg.topic, msg.partition, msg.value) + affected_rows, id = await self.event_queue_add(msg.topic, msg.partition, msg.value) + return affected_rows, id finally: pass @@ -251,8 +252,6 @@ async def event_queue_checker(self): event_instance = Event.from_avro_bytes(row[3]) await reply_on(topic=row[1], event=event_instance) call_ok = True - except: - call_ok = False finally: if call_ok: # Delete from database If the event was sent successfully to Kafka. diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 6b7b2fe5..d5cd61a7 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -175,9 +175,10 @@ async def test_consumer_kafka(config,loop): await consumer.start() consumer.subscribe(topics) - for i in range(0, 2): - msg = await consumer.getone() - await m.handle_single_message(msg) + msg = await consumer.getone() + affected_rows, id = await m.handle_single_message(msg) + assert affected_rows > 0 + assert id > 0 await consumer.stop() From ac1e6260ec3da15b17f0d7ad94cb68dafe3822c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Fri, 30 Apr 2021 08:13:03 +0200 Subject: [PATCH 148/239] ISSUE #51 * Start using `PostgreSqlMinosRepository` instead of directly querying the database table. --- minos/networks/snapshots/dispatchers.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 862cb365..ede6b389 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -10,6 +10,7 @@ ) from typing import ( + Any, Generator, NoReturn, Optional, @@ -24,6 +25,7 @@ MinosRepositoryAction, MinosRepositoryEntry, MinosSetup, + PostgreSqlMinosRepository, import_module, ) @@ -43,6 +45,7 @@ def __init__( database: str = None, user: str = None, password: str = None, + repository: dict[str, Any] = None, offset: int = 0, **kwargs ): @@ -56,6 +59,8 @@ def __init__( self.offset = offset + self.repository = PostgreSqlMinosRepository(**repository) + @classmethod def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> MinosSnapshotDispatcher: """Build a new Snapshot Dispatcher from config. @@ -69,7 +74,7 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> MinosSnapsh if config is None: raise MinosConfigException("The config object must be setup.") # noinspection PyProtectedMember - return cls(*args, **config.repository._asdict(), **kwargs) + return cls(*args, **config.repository._asdict(), repository=config.repository._asdict(), **kwargs) async def _setup(self) -> NoReturn: await self._create_broker_table() @@ -96,8 +101,8 @@ async def dispatch(self) -> NoReturn: @property async def _new_entries(self): - async for raw in self._submit_and_iter_sql(_SELECT_EVENT_ENTRIES_QUERY, (self.offset,)): - yield MinosRepositoryEntry(*raw) + for entry in await self.repository.select(id_ge=self.offset): + yield entry async def _dispatch_one(self, event_entry: MinosRepositoryEntry) -> Optional[MinosSnapshotEntry]: if event_entry.action is MinosRepositoryAction.DELETE: @@ -189,12 +194,6 @@ def _connection(self): FROM snapshot; """.strip() -_SELECT_EVENT_ENTRIES_QUERY = """ -SELECT aggregate_id, aggregate_name, version, data, id, action -FROM events -WHERE id >= %s; -""".strip() - _DELETE_ONE_SNAPSHOT_ENTRY_QUERY = """ DELETE FROM snapshot WHERE aggregate_id = %(aggregate_id)s and aggregate_name = %(aggregate_name)s; From d3721b82a367b1068c83f1bcc64d44df8c6ac5e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Fri, 30 Apr 2021 09:29:12 +0200 Subject: [PATCH 149/239] ISSUE #51 * Add missing docstring. --- minos/networks/snapshots/dispatchers.py | 15 +++++++++------ minos/networks/snapshots/entries.py | 13 ++++++++----- minos/networks/snapshots/services.py | 15 ++++++--------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index ede6b389..08de6181 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -35,7 +35,7 @@ class MinosSnapshotDispatcher(MinosSetup): - """TODO""" + """Minos Snapshot Dispatcher class.""" def __init__( self, @@ -84,18 +84,21 @@ async def _create_broker_table(self) -> NoReturn: # noinspection PyUnusedLocal async def select(self, *args, **kwargs) -> list[MinosSnapshotEntry]: - """TODO + """Select a sequence of ``MinosSnapshotEntry`` objects. - :param args: TODO - :param kwargs: TODO - :return: + :param args: Additional positional arguments. + :param kwargs: Additional named arguments. + :return: A sequence of ``MinosSnapshotEntry`` objects. """ response = self._submit_and_iter_sql(_SELECT_ALL_ENTRIES_QUERY) entries = [MinosSnapshotEntry(*row) async for row in response] return entries async def dispatch(self) -> NoReturn: - """TODO""" + """Perform a dispatching step, based on the sequence of non already processed ``MinosRepositoryEntry`` objects. + + :return: This method does not return anything. + """ async for entry in self._new_entries: await self._dispatch_one(entry) diff --git a/minos/networks/snapshots/entries.py b/minos/networks/snapshots/entries.py index 8fd771f1..973b351e 100644 --- a/minos/networks/snapshots/entries.py +++ b/minos/networks/snapshots/entries.py @@ -27,7 +27,10 @@ class MinosSnapshotEntry(object): - """TODO""" + """Minos Snapshot Entry class. + + Is the python object representation of a row in the ``snapshot`` storage system. + """ __slots__ = "aggregate_id", "aggregate_name", "version", "data", "created_at", "updated_at" @@ -64,9 +67,9 @@ def from_aggregate(cls, aggregate: Aggregate) -> MinosSnapshotEntry: @property def aggregate(self) -> Aggregate: - """TODO + """Rebuild the stored ``Aggregate`` object instance from the internal state. - :return: TODO + :return: A ``Aggregate`` instance. """ cls = self.aggregate_cls instance = cls.from_avro_bytes(self.data, id=self.aggregate_id, version=self.version) @@ -74,9 +77,9 @@ def aggregate(self) -> Aggregate: @property def aggregate_cls(self) -> Type[Aggregate]: - """TODO + """Load the concrete ``Aggregate`` class. - :return: TODO + :return: A ``Type`` object. """ # noinspection PyTypeChecker return import_module(self.aggregate_name) diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 75cec27c..0fcb9a89 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -5,9 +5,6 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - Any, -) from aiomisc.service.periodic import ( PeriodicService, @@ -22,23 +19,23 @@ class MinosSnapshotService(PeriodicService): - """TODO""" + """Minos Snapshot Service class.""" def __init__(self, config: MinosConfig = None, **kwargs): super().__init__(**kwargs) self.dispatcher = MinosSnapshotDispatcher.from_config(config=config) async def start(self) -> None: - """TODO + """Start the service execution. - :return: TODO + :return: This method does not return anything. """ await super().start() await self.dispatcher.setup() - async def callback(self) -> Any: - """TODO + async def callback(self) -> None: + """Callback implementation to be executed periodically. - :return: TODO + :return: This method does not return anything. """ await self.dispatcher.dispatch() From b0ed988affabb86bcb9d98e7a5268f887b1b2773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Fri, 30 Apr 2021 09:40:50 +0200 Subject: [PATCH 150/239] ISSUE #51 * Refactor select to return an async iterator. --- minos/networks/snapshots/dispatchers.py | 11 +++++------ .../test_networks/test_snapshots/test_dispatchers.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 08de6181..75dd23b6 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -11,7 +11,7 @@ from typing import ( Any, - Generator, + AsyncIterator, NoReturn, Optional, Type, @@ -83,16 +83,15 @@ async def _create_broker_table(self) -> NoReturn: await self._submit_sql(_CREATE_TABLE_QUERY) # noinspection PyUnusedLocal - async def select(self, *args, **kwargs) -> list[MinosSnapshotEntry]: + async def select(self, *args, **kwargs) -> AsyncIterator[MinosSnapshotEntry]: """Select a sequence of ``MinosSnapshotEntry`` objects. :param args: Additional positional arguments. :param kwargs: Additional named arguments. :return: A sequence of ``MinosSnapshotEntry`` objects. """ - response = self._submit_and_iter_sql(_SELECT_ALL_ENTRIES_QUERY) - entries = [MinosSnapshotEntry(*row) async for row in response] - return entries + async for row in self._submit_and_iter_sql(_SELECT_ALL_ENTRIES_QUERY): + yield MinosSnapshotEntry(*row) async def dispatch(self) -> NoReturn: """Perform a dispatching step, based on the sequence of non already processed ``MinosRepositoryEntry`` objects. @@ -162,7 +161,7 @@ async def _submit_update_or_create(self, entry: MinosSnapshotEntry) -> MinosSnap async def _submit_and_fetchone_sql(self, *args, **kwargs) -> tuple: return await self._submit_and_iter_sql(*args, **kwargs).__anext__() - async def _submit_and_iter_sql(self, query: str, *args, **kwargs) -> Generator[tuple, None, None]: + async def _submit_and_iter_sql(self, query: str, *args, **kwargs) -> AsyncIterator[tuple]: async with self._connection() as connect: async with connect.cursor() as cursor: await cursor.execute(query, *args, **kwargs) diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index a717d733..43d32df5 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -56,7 +56,7 @@ async def test_dispatch_select(self): dispatcher = MinosSnapshotDispatcher.from_config() await dispatcher.setup() await dispatcher.dispatch() - observed = await dispatcher.select() + observed = [v async for v in dispatcher.select()] expected = [ MinosSnapshotEntry.from_aggregate(Car(2, 2, 3, "blue")), From cc16ad6e922a25ec034b04e4ba9c4762839976b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Fri, 30 Apr 2021 17:05:59 +0200 Subject: [PATCH 151/239] ISSUE #88 * Add `MinosSnapshotException` and `MinosPreviousVersionSnapshotException`. * Skip `aggregate_events` with lower `version` than current one. --- minos/networks/__init__.py | 2 ++ minos/networks/exceptions.py | 18 +++++++++++++++++- minos/networks/snapshots/dispatchers.py | 18 +++++++++++++++--- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index b90383e8..a92183b6 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -15,6 +15,8 @@ ) from .exceptions import ( MinosNetworkException, + MinosSnapshotException, + MinosPreviousVersionSnapshotException, ) from .snapshots import ( MinosSnapshotDispatcher, diff --git a/minos/networks/exceptions.py b/minos/networks/exceptions.py index 63f2ea7f..e29a8a7d 100644 --- a/minos/networks/exceptions.py +++ b/minos/networks/exceptions.py @@ -6,9 +6,25 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ from minos.common import ( - MinosException, + MinosException, Aggregate, ) class MinosNetworkException(MinosException): """Base network exception.""" + + +class MinosSnapshotException(MinosException): + """Base snapshot exception""" + + +class MinosPreviousVersionSnapshotException(MinosSnapshotException): + """Exception to be raised when current version is newer than the one to be processed.""" + + def __init__(self, previous: Aggregate, new: Aggregate): + self.previous = previous + self.new = new + super().__init__( + f"Version for {repr(previous.classname)} aggregate must be " + f"greater than {previous.version}. Obtained: {new.version}" + ) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 75dd23b6..41804142 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -29,6 +29,9 @@ import_module, ) +from ..exceptions import ( + MinosPreviousVersionSnapshotException, +) from .entries import ( MinosSnapshotEntry, ) @@ -99,7 +102,10 @@ async def dispatch(self) -> NoReturn: :return: This method does not return anything. """ async for entry in self._new_entries: - await self._dispatch_one(entry) + try: + await self._dispatch_one(entry) + except MinosPreviousVersionSnapshotException: + continue @property async def _new_entries(self): @@ -125,13 +131,19 @@ async def _build_instance(self, event_entry: MinosRepositoryEntry) -> Aggregate: return instance async def _update_if_exists(self, new: Aggregate) -> Aggregate: + # noinspection PyBroadException try: # noinspection PyTypeChecker previous = await self._select_one_aggregate(new.id, new.classname) - new._fields = previous.fields | new.fields - finally: + except Exception: return new + if previous.version >= new.version: + raise MinosPreviousVersionSnapshotException(previous, new) + + new._fields = previous.fields | new.fields + return new + async def _select_one_aggregate(self, aggregate_id: int, aggregate_name: str) -> Aggregate: snapshot_entry = await self._select_one(aggregate_id, aggregate_name) return snapshot_entry.aggregate From 43bb97d25494cd555aef24178f0e3de5a2c49655 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 30 Apr 2021 15:14:42 +0000 Subject: [PATCH 152/239] Restyled by black --- minos/networks/exceptions.py | 3 ++- minos/networks/snapshots/dispatchers.py | 12 +++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/minos/networks/exceptions.py b/minos/networks/exceptions.py index e29a8a7d..4e0260d2 100644 --- a/minos/networks/exceptions.py +++ b/minos/networks/exceptions.py @@ -6,7 +6,8 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ from minos.common import ( - MinosException, Aggregate, + MinosException, + Aggregate, ) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 41804142..1bc5cc39 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( Any, @@ -29,12 +27,8 @@ import_module, ) -from ..exceptions import ( - MinosPreviousVersionSnapshotException, -) -from .entries import ( - MinosSnapshotEntry, -) +from ..exceptions import MinosPreviousVersionSnapshotException +from .entries import MinosSnapshotEntry class MinosSnapshotDispatcher(MinosSetup): From b4855b4b0055371c066ca95f57b33948a701dbd5 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 30 Apr 2021 15:15:17 +0000 Subject: [PATCH 153/239] Restyled by isort --- minos/networks/__init__.py | 2 +- minos/networks/exceptions.py | 2 +- minos/networks/snapshots/dispatchers.py | 12 +++++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index a92183b6..7e5ea5d7 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -15,8 +15,8 @@ ) from .exceptions import ( MinosNetworkException, - MinosSnapshotException, MinosPreviousVersionSnapshotException, + MinosSnapshotException, ) from .snapshots import ( MinosSnapshotDispatcher, diff --git a/minos/networks/exceptions.py b/minos/networks/exceptions.py index 4e0260d2..5253c616 100644 --- a/minos/networks/exceptions.py +++ b/minos/networks/exceptions.py @@ -6,8 +6,8 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ from minos.common import ( - MinosException, Aggregate, + MinosException, ) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 1bc5cc39..41804142 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( Any, @@ -27,8 +29,12 @@ import_module, ) -from ..exceptions import MinosPreviousVersionSnapshotException -from .entries import MinosSnapshotEntry +from ..exceptions import ( + MinosPreviousVersionSnapshotException, +) +from .entries import ( + MinosSnapshotEntry, +) class MinosSnapshotDispatcher(MinosSetup): From 00c490bdea83a536a54182f4d5febdf4747157f7 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 30 Apr 2021 17:58:18 +0200 Subject: [PATCH 154/239] EventHandler Refactor Code and tests EventHandler Refactor code and add more tests #87 --- minos/networks/__init__.py | 6 + minos/networks/event.py | 262 ------------------ minos/networks/event/__init__.py | 20 ++ minos/networks/event/abc.py | 63 +++++ minos/networks/event/dispatcher.py | 143 ++++++++++ minos/networks/event/event_server.py | 143 ++++++++++ minos/networks/event/services.py | 76 +++++ tests/database_testcase.py | 64 ----- tests/test_event_manager.py | 184 ------------ tests/test_networks/test_event/__init__.py | 7 + .../test_event/test_dispatcher.py | 102 +++++++ .../test_event/test_event_server.py | 128 +++++++++ .../test_event/test_event_services.py | 44 +++ 13 files changed, 732 insertions(+), 510 deletions(-) delete mode 100644 minos/networks/event.py create mode 100644 minos/networks/event/__init__.py create mode 100644 minos/networks/event/abc.py create mode 100644 minos/networks/event/dispatcher.py create mode 100644 minos/networks/event/event_server.py create mode 100644 minos/networks/event/services.py delete mode 100644 tests/database_testcase.py delete mode 100644 tests/test_event_manager.py create mode 100644 tests/test_networks/test_event/__init__.py create mode 100644 tests/test_networks/test_event/test_dispatcher.py create mode 100644 tests/test_networks/test_event/test_event_server.py create mode 100644 tests/test_networks/test_event/test_event_services.py diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 01742ffe..be6002ca 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -16,3 +16,9 @@ from .exceptions import ( MinosNetworkException, ) +from .event import ( + MinosEventServer, + MinosEventHandler, + MinosEventServerService, + MinosEventPeriodicService +) diff --git a/minos/networks/event.py b/minos/networks/event.py deleted file mode 100644 index 08fd2115..00000000 --- a/minos/networks/event.py +++ /dev/null @@ -1,262 +0,0 @@ -# Copyright (C) 2020 Clariteia SL -# -# This file is part of minos framework. -# -# Minos framework can not be copied and/or distributed without the express -# permission of Clariteia SL. - -import asyncio -import datetime -import functools -import inspect -import typing as t - -import aiopg -from aiokafka import ( - AIOKafkaConsumer, -) -from aiomisc import ( - Service, -) -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common.broker import ( - Event, -) -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.common.importlib import ( - import_module, -) -from minos.common.logs import ( - log, -) -from minos.networks.exceptions import ( - MinosNetworkException, -) - - -class MinosEventServer(Service): - """ - Event Manager - - Consumer for the Broker ( at the moment only Kafka is supported ) - - """ - - __slots__ = "_tasks", "_db_dsn", "_handlers", "_topics", "_kafka_conn_data", "_broker_group_name" - - def __init__(self, *, conf: MinosConfig, **kwargs: t.Any): - self._tasks = set() # type: t.Set[asyncio.Task] - self._db_dsn = ( - f"dbname={conf.events.queue.database} user={conf.events.queue.user} " - f"password={conf.events.queue.password} host={conf.events.queue.host}" - ) - self._handler = { - item.name: {"controller": item.controller, "action": item.action} for item in conf.events.items - } - self._topics = list(self._handler.keys()) - self._kafka_conn_data = f"{conf.events.broker.host}:{conf.events.broker.port}" - self._broker_group_name = f"event_{conf.service.name}" - super().__init__(**kwargs) - - def create_task(self, coro: t.Awaitable[t.Any]): - task = self.loop.create_task(coro) - self._tasks.add(task) - task.add_done_callback(self._tasks.remove) - - async def event_queue_add(self, topic: str, partition: int, binary: bytes): - """Insert row to event_queue table. - - Retrieves number of affected rows and row ID. - - Args: - topic: Kafka topic. Example: "TicketAdded" - partition: Kafka partition number. - binary: Event Model in bytes. - - Returns: - Affected rows and queue ID. - - Example: 1, 12 - - Raises: - Exception: An error occurred inserting record. - """ - - async with aiopg.create_pool(self._db_dsn) as pool: - async with pool.acquire() as connect: - async with connect.cursor() as cur: - await cur.execute( - "INSERT INTO event_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", - (topic, partition, binary, datetime.datetime.now(),), - ) - - queue_id = await cur.fetchone() - affected_rows = cur.rowcount - - return affected_rows, queue_id[0] - - async def handle_single_message(self, msg): - """Handle Kafka messages. - - Evaluate if the binary of message is an Event instance. - Add Event instance to the event_queue table. - - Args: - msg: Kafka message. - - Raises: - Exception: An error occurred inserting record. - """ - # the handler receive a message and store in the queue database - # check if the event binary string is well formatted - try: - Event.from_avro_bytes(msg.value) - affected_rows, id = await self.event_queue_add(msg.topic, msg.partition, msg.value) - return affected_rows, id - finally: - pass - - async def handle_message(self, consumer: t.Any): - """Message consumer. - - It consumes the messages and sends them for processing. - - Args: - consumer: Kafka Consumer instance (at the moment only Kafka consumer is supported). - """ - - async for msg in consumer: - await self.handle_single_message(msg) - - async def start(self) -> t.Any: - self.start_event.set() - log.debug("Event Consumer Manager: Started") - # start the Service Event Consumer for Kafka - consumer = AIOKafkaConsumer( - loop=self.loop, - group_id=self._broker_group_name, - auto_offset_reset="latest", - bootstrap_servers=self._kafka_conn_data, - ) - - await consumer.start() - consumer.subscribe(self._topics) - - self.create_task(self.handle_message(consumer)) - - -async def event_handler_table_creation(conf: MinosConfig): - db_dsn = ( - f"dbname={conf.events.queue.database} user={conf.events.queue.user} " - f"password={conf.events.queue.password} host={conf.events.queue.host}" - ) - async with aiopg.create_pool(db_dsn) as pool: - async with pool.acquire() as connect: - async with connect.cursor() as cur: - await cur.execute( - 'CREATE TABLE IF NOT EXISTS "event_queue" ("id" SERIAL NOT NULL PRIMARY KEY, ' - '"topic" VARCHAR(255) NOT NULL, "partition_id" INTEGER , "binary_data" BYTEA NOT NULL, "creation_date" TIMESTAMP NOT NULL);' - ) - - -class EventHandlerDatabaseInitializer(Service): - async def start(self): - # Send signal to entrypoint for continue running - self.start_event.set() - - await event_handler_table_creation(conf=self.config) - - await self.stop(self) - - -class MinosEventHandlerPeriodicService(PeriodicService): - """ - Periodic Service Event Handler - - """ - - __slots__ = "_db_dsn", "_handlers", "_event_items", "_topics", "_conf" - - def __init__(self, *, conf: MinosConfig, **kwargs: t.Any): - super().__init__(**kwargs) - self._db_dsn = ( - f"dbname={conf.events.queue.database} user={conf.events.queue.user} " - f"password={conf.events.queue.password} host={conf.events.queue.host}" - ) - self._handlers = { - item.name: {"controller": item.controller, "action": item.action} for item in conf.events.items - } - self._event_items = conf.events.items - self._topics = list(self._handlers.keys()) - self._conf = conf - - def get_event_handler(self, topic: str) -> t.Callable: - - """Get Event instance to call. - - Gets the instance of the class and method to call. - - Args: - topic: Kafka topic. Example: "TicketAdded" - - Raises: - MinosNetworkException: topic TicketAdded have no controller/action configured, please review th configuration file. - """ - for event in self._event_items: - if event.name == topic: - # the topic exist, get the controller and the action - controller = event.controller - action = event.action - - object_class = import_module(controller) - log.debug(object_class()) - instance_class = object_class() - class_method = getattr(instance_class, action) - - return class_method - raise MinosNetworkException( - f"topic {topic} have no controller/action configured, " f"please review th configuration file" - ) - - async def event_queue_checker(self): - """Event Queue Checker and dispatcher. - - It is in charge of querying the database and calling the action according to the topic. - - 1. Get periodically 10 records (or as many as defined in config > queue > records). - 2. Instantiate the action (asynchronous) by passing it the model. - 3. If the invoked function terminates successfully, remove the event from the database. - - Raises: - Exception: An error occurred inserting record. - """ - db_dsn = ( - f"dbname={self._conf.events.queue.database} user={self._conf.events.queue.user} " - f"password={self._conf.events.queue.password} host={self._conf.events.queue.host}" - ) - async with aiopg.create_pool(db_dsn) as pool: - async with pool.acquire() as connect: - async with connect.cursor() as cur: - await cur.execute( - "SELECT * FROM event_queue ORDER BY creation_date ASC LIMIT %d;" - % (self._conf.events.queue.records), - ) - async for row in cur: - call_ok = False - try: - reply_on = self.get_event_handler(topic=row[1]) - event_instance = Event.from_avro_bytes(row[3]) - await reply_on(topic=row[1], event=event_instance) - call_ok = True - finally: - if call_ok: - # Delete from database If the event was sent successfully to Kafka. - async with connect.cursor() as cur2: - await cur2.execute("DELETE FROM event_queue WHERE id=%d;" % row[0]) - - async def callback(self): - await self.event_queue_checker() diff --git a/minos/networks/event/__init__.py b/minos/networks/event/__init__.py new file mode 100644 index 00000000..ffadcde9 --- /dev/null +++ b/minos/networks/event/__init__.py @@ -0,0 +1,20 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from .dispatcher import ( + MinosEventHandler, +) + +from .event_server import ( + MinosEventServer, +) + +from .services import ( + MinosEventServerService, + MinosEventPeriodicService +) diff --git a/minos/networks/event/abc.py b/minos/networks/event/abc.py new file mode 100644 index 00000000..e130e7b7 --- /dev/null +++ b/minos/networks/event/abc.py @@ -0,0 +1,63 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from abc import ( + ABC, +) +from datetime import ( + datetime, +) +from typing import ( + NoReturn, +) + +import aiopg +from minos.common import ( + MinosSetup, +) + + +class MinosEventSetup(MinosSetup): + """Minos Broker Setup Class""" + + def __init__( + self, + *args, + host: str = None, + port: int = None, + database: str = None, + user: str = None, + password: str = None, + **kwargs + ): + super().__init__(*args, **kwargs) + + self.host = host + self.port = port + self.database = database + self.user = user + self.password = password + + async def _setup(self) -> NoReturn: + await self._create_event_queue_table() + + async def _create_event_queue_table(self) -> NoReturn: + async with self._connection() as connect: + async with connect.cursor() as cur: + await cur.execute( + 'CREATE TABLE IF NOT EXISTS "event_queue" (' + '"id" BIGSERIAL NOT NULL PRIMARY KEY, ' + '"topic" VARCHAR(255) NOT NULL, ' + '"partition_id" INTEGER,' + '"binary_data" BYTEA NOT NULL, ' + '"creation_date" TIMESTAMP NOT NULL);' + ) + + def _connection(self): + return aiopg.connect( + host=self.host, port=self.port, dbname=self.database, user=self.user, password=self.password, + ) diff --git a/minos/networks/event/dispatcher.py b/minos/networks/event/dispatcher.py new file mode 100644 index 00000000..a07f9128 --- /dev/null +++ b/minos/networks/event/dispatcher.py @@ -0,0 +1,143 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from __future__ import ( + annotations, +) + +from typing import ( + NamedTuple, + NoReturn, + Optional, + Callable, + Any, +) + +import aiopg + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common.broker import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.importlib import ( + import_module, +) +from minos.common.logs import ( + log, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) +from .abc import ( + MinosEventSetup, +) + + +class MinosEventHandler(MinosEventSetup): + """ + Event Handler + + """ + + __slots__ = "_db_dsn", "_handlers", "_event_items", "_topics", "_conf" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(**kwargs, **config.events.queue._asdict()) + self._db_dsn = ( + f"dbname={config.events.queue.database} user={config.events.queue.user} " + f"password={config.events.queue.password} host={config.events.queue.host}" + ) + self._handlers = { + item.name: {"controller": item.controller, "action": item.action} for item in config.events.items + } + self._event_items = config.events.items + self._topics = list(self._handlers.keys()) + self._conf = config + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosEventHandler]: + """Build a new repository from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + return None + # noinspection PyProtectedMember + return cls(*args, config=config, **kwargs) + + def get_event_handler(self, topic: str) -> Callable: + + """Get Event instance to call. + + Gets the instance of the class and method to call. + + Args: + topic: Kafka topic. Example: "TicketAdded" + + Raises: + MinosNetworkException: topic TicketAdded have no controller/action configured, please review th configuration file. + """ + for event in self._event_items: + if event.name == topic: + # the topic exist, get the controller and the action + controller = event.controller + action = event.action + + object_class = import_module(controller) + log.debug(object_class()) + instance_class = object_class() + class_method = getattr(instance_class, action) + + return class_method + raise MinosNetworkException( + f"topic {topic} have no controller/action configured, " f"please review th configuration file" + ) + + async def event_queue_checker(self) -> NoReturn: + """Event Queue Checker and dispatcher. + + It is in charge of querying the database and calling the action according to the topic. + + 1. Get periodically 10 records (or as many as defined in config > queue > records). + 2. Instantiate the action (asynchronous) by passing it the model. + 3. If the invoked function terminates successfully, remove the event from the database. + + Raises: + Exception: An error occurred inserting record. + """ + db_dsn = ( + f"dbname={self._conf.events.queue.database} user={self._conf.events.queue.user} " + f"password={self._conf.events.queue.password} host={self._conf.events.queue.host}" + ) + async with aiopg.create_pool(db_dsn) as pool: + async with pool.acquire() as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT * FROM event_queue ORDER BY creation_date ASC LIMIT %d;" + % (self._conf.events.queue.records), + ) + async for row in cur: + call_ok = False + try: + reply_on = self.get_event_handler(topic=row[1]) + event_instance = Event.from_avro_bytes(row[3]) + await reply_on(topic=row[1], event=event_instance) + call_ok = True + finally: + if call_ok: + # Delete from database If the event was sent successfully to Kafka. + async with connect.cursor() as cur2: + await cur2.execute("DELETE FROM event_queue WHERE id=%d;" % row[0]) diff --git a/minos/networks/event/event_server.py b/minos/networks/event/event_server.py new file mode 100644 index 00000000..948a00a4 --- /dev/null +++ b/minos/networks/event/event_server.py @@ -0,0 +1,143 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from __future__ import ( + annotations, +) + +import asyncio +import datetime +from typing import ( + NoReturn, + Optional, + Awaitable, + Any, +) + +import aiopg + +from minos.common.broker import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.logs import ( + log, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) +from .abc import ( + MinosEventSetup, +) + + +class MinosEventServer(MinosEventSetup): + """ + Event Manager + + Consumer for the Broker ( at the moment only Kafka is supported ) + + """ + + __slots__ = "_tasks", "_db_dsn", "_handlers", "_topics", "_kafka_conn_data", "_broker_group_name" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(**kwargs, **config.events.queue._asdict()) + self._tasks = set() # type: t.Set[asyncio.Task] + self._db_dsn = ( + f"dbname={config.events.queue.database} user={config.events.queue.user} " + f"password={config.events.queue.password} host={config.events.queue.host}" + ) + self._handler = { + item.name: {"controller": item.controller, "action": item.action} for item in config.events.items + } + self._topics = list(self._handler.keys()) + self._kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" + self._broker_group_name = f"event_{config.service.name}" + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosEventServer]: + """Build a new repository from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + return None + # noinspection PyProtectedMember + return cls(*args, config=config, **kwargs) + + async def event_queue_add(self, topic: str, partition: int, binary: bytes): + """Insert row to event_queue table. + + Retrieves number of affected rows and row ID. + + Args: + topic: Kafka topic. Example: "TicketAdded" + partition: Kafka partition number. + binary: Event Model in bytes. + + Returns: + Affected rows and queue ID. + + Example: 1, 12 + + Raises: + Exception: An error occurred inserting record. + """ + + async with aiopg.create_pool(self._db_dsn) as pool: + async with pool.acquire() as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO event_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + (topic, partition, binary, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + return affected_rows, queue_id[0] + + async def handle_single_message(self, msg): + """Handle Kafka messages. + + Evaluate if the binary of message is an Event instance. + Add Event instance to the event_queue table. + + Args: + msg: Kafka message. + + Raises: + Exception: An error occurred inserting record. + """ + # the handler receive a message and store in the queue database + # check if the event binary string is well formatted + try: + Event.from_avro_bytes(msg.value) + affected_rows, id = await self.event_queue_add(msg.topic, msg.partition, msg.value) + return affected_rows, id + finally: + pass + + async def handle_message(self, consumer: Any): + """Message consumer. + + It consumes the messages and sends them for processing. + + Args: + consumer: Kafka Consumer instance (at the moment only Kafka consumer is supported). + """ + + async for msg in consumer: + await self.handle_single_message(msg) + diff --git a/minos/networks/event/services.py b/minos/networks/event/services.py new file mode 100644 index 00000000..d120d95e --- /dev/null +++ b/minos/networks/event/services.py @@ -0,0 +1,76 @@ +from aiomisc.service.periodic import ( + Service, + PeriodicService, +) +from minos.common import ( + MinosConfig, +) +from .dispatcher import ( + MinosEventHandler, +) +from .event_server import ( + MinosEventServer +) +from aiokafka import ( + AIOKafkaConsumer, +) +from typing import ( + Awaitable, + Any, +) + + +class MinosEventServerService(Service): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosEventServer.from_config(config=config) + + def create_task(self, coro: Awaitable[Any]): + task = self.loop.create_task(coro) + self.dispatcher._tasks.add(task) + task.add_done_callback(self.dispatcher._tasks.remove) + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await self.dispatcher.setup() + + self.start_event.set() + # start the Service Event Consumer for Kafka + consumer = AIOKafkaConsumer( + group_id=self.dispatcher._broker_group_name, + auto_offset_reset="latest", + bootstrap_servers=self.dispatcher._kafka_conn_data, + ) + + await consumer.start() + consumer.subscribe(self.dispatcher._topics) + + self.create_task(self.dispatcher.handle_message(consumer)) + + +class MinosEventPeriodicService(PeriodicService): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosEventHandler.from_config(config=config) + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await super().start() + await self.dispatcher.setup() + + async def callback(self) -> None: + """Method to be called periodically by the internal ``aiomisc`` logic. + + :return:This method does not return anything. + """ + await self.dispatcher.event_queue_checker() diff --git a/tests/database_testcase.py b/tests/database_testcase.py deleted file mode 100644 index aeb60de2..00000000 --- a/tests/database_testcase.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Copyright (C) 2021 Clariteia SL -This file is part of minos framework. -Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. -""" -import os -import unittest - -import aiopg -from minos.common.configuration.config import MinosConfig -from minos.networks.event import event_handler_table_creation - - -class EventHandlerPostgresAsyncTestCase(unittest.IsolatedAsyncioTestCase): - def setUp(self) -> None: - self._meta_kwargs = { - "host": os.getenv("POSTGRES_HOST", "localhost"), - "port": os.getenv("POSTGRES_PORT", 5432), - "database": os.getenv("POSTGRES_DATABASE", "postgres"), - "user": os.getenv("POSTGRES_USER", "postgres"), - "password": os.getenv("POSTGRES_PASSWORD", ""), - } - - self.kwargs = self._meta_kwargs | { - "database": "broker_db", - "user": "broker", - "password": "br0k3r", - } - - async def asyncSetUp(self): - await event_handler_table_creation(self._broker_config()) - """ - async with aiopg.connect(**self._meta_kwargs) as connection: - async with connection.cursor() as cursor: - - template = "DROP DATABASE IF EXISTS {database};" - await cursor.execute(template.format(**self.kwargs)) - - template = "DROP ROLE IF EXISTS {user};" - await cursor.execute(template.format(**self.kwargs)) - - template = "CREATE ROLE {user} WITH CREATEDB LOGIN ENCRYPTED PASSWORD {password!r};" - await cursor.execute(template.format(**self.kwargs)) - - template = "CREATE DATABASE {database} WITH OWNER = {user};" - await cursor.execute(template.format(**self.kwargs)) - """ - - @staticmethod - def _broker_config(): - return MinosConfig(path="./tests/test_config.yml") - - async def _database(self): - conf = self._broker_config() - db_dsn = f"dbname={conf.events.queue.database} user={conf.events.queue.user} " \ - f"password={conf.events.queue.password} host={conf.events.queue.host}" - return await aiopg.connect(db_dsn) - - async def asyncTearDown(self): - database = await self._database() - async with database as connection: - async with connection.cursor() as cursor: - template = "DROP TABLE IF EXISTS event_queue" - await cursor.execute(template) diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py deleted file mode 100644 index d5cd61a7..00000000 --- a/tests/test_event_manager.py +++ /dev/null @@ -1,184 +0,0 @@ -import asyncio -import logging -import random -import string - -import pytest -from aiokafka import ( - AIOKafkaConsumer, - AIOKafkaProducer, -) -from aiomisc.log import ( - basic_config, -) -from minos.common import ( - Aggregate, -) -from minos.common.broker import ( - Event, -) -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.common.logs import ( - log, -) - -from minos.networks.event import ( - EventHandlerDatabaseInitializer, - MinosEventHandlerPeriodicService, - MinosEventServer, -) -from tests.database_testcase import ( - EventHandlerPostgresAsyncTestCase, -) - - -@pytest.fixture() -def config(): - return MinosConfig(path="./tests/test_config.yml") - - -@pytest.fixture() -def services(config): - return [ - EventHandlerDatabaseInitializer(config=config), - MinosEventServer(conf=config), - MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config), - ] - - -class AggregateTest(Aggregate): - test: int - - -class TestPostgreSqlMinosEventHandler(EventHandlerPostgresAsyncTestCase): - async def test_database_connection(self): - database = await self._database() - async with database as connect: - assert database.closed == 0 - - async def test_if_queue_table_exists(self): - database = await self._database() - async with database as connect: - async with connect.cursor() as cur: - - await cur.execute( - "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'event_queue';" - ) - ret = [] - async for row in cur: - ret.append(row) - - assert ret == [(1,)] - - async def test_event_queue_add(self): - model = AggregateTest(test_id=1, test=2, id=1, version=1) - event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) - bin_data = event_instance.avro_bytes - Event.from_avro_bytes(bin_data) - - m = MinosEventServer(conf=self._broker_config()) - affected_rows, id = await m.event_queue_add(topic=event_instance.topic, partition=0, binary=bin_data) - - assert affected_rows == 1 - assert id > 0 - - async def test_get_event_handler(self): - model = AggregateTest(test_id=1, test=2, id=1, version=1) - event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) - m = MinosEventHandlerPeriodicService(conf=self._broker_config(), interval=0.5) - - cls = m.get_event_handler(topic="TicketAdded") - result = await cls(topic="TicketAdded", event=event_instance) - - assert result == "request_added" - - async def test_event_queue_checker(self): - model = AggregateTest(test_id=1, test=2, id=1, version=1) - event_instance = Event(topic="TicketAdded", model=model.classname, items=[]) - bin_data = event_instance.avro_bytes - Event.from_avro_bytes(bin_data) - - m = MinosEventServer(conf=self._broker_config()) - affected_rows, id = await m.event_queue_add(topic=event_instance.topic, partition=0, binary=bin_data) - - assert affected_rows == 1 - assert id > 0 - - database = await self._database() - async with database as connect: - async with connect.cursor() as cur: - await cur.execute("SELECT COUNT(*) FROM event_queue WHERE id=%d" % (id)) - records = await cur.fetchone() - - assert records[0] == 1 - - m = MinosEventHandlerPeriodicService(conf=self._broker_config(), interval=0.5) - await m.event_queue_checker() - - database = await self._database() - async with database as connect: - async with connect.cursor() as cur: - await cur.execute("SELECT COUNT(*) FROM event_queue WHERE id=%d" % (id)) - records = await cur.fetchone() - - assert records[0] == 0 - - -async def kafka_producer(config, loop): - - # basic_config( - # level=logging.INFO, - # buffered=True, - # log_format='color', - # flush_interval=2 - # ) - kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" - producer = AIOKafkaProducer(loop=loop, bootstrap_servers=kafka_conn_data) - # Get cluster layout and topic/partition allocation - await producer.start() - # Produce messages - - model = AggregateTest(test_id=1, test=2, id=1, version=1) - event_instance = Event(topic="TicketAdded", model=model.classname, items=[model]) - bin_data = event_instance.avro_bytes - - model2 = AggregateTest(test_id=2, test=2, id=1, version=1) - event_instance_2 = Event(topic="TicketDeleted", model=model2.classname, items=[model2]) - bin_data2 = event_instance_2.avro_bytes - - for i in range(0, 10): - await producer.send_and_wait(event_instance.topic, bin_data) - await producer.send_and_wait(event_instance_2.topic, bin_data2) - - await producer.stop() - - - -async def test_consumer_kafka(config,loop): - await kafka_producer(config, loop) - handler = {item.name: {'controller': item.controller, 'action': item.action} - for item in config.events.items} - topics = list(handler.keys()) - kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" - broker_group_name = f"event_{config.service.name}" - - m = MinosEventServer(conf=config) - consumer = AIOKafkaConsumer( - loop=loop, - group_id=broker_group_name, - auto_offset_reset="latest", - bootstrap_servers=kafka_conn_data, consumer_timeout_ms=500 - ) - - await consumer.start() - consumer.subscribe(topics) - - msg = await consumer.getone() - affected_rows, id = await m.handle_single_message(msg) - assert affected_rows > 0 - assert id > 0 - - await consumer.stop() - diff --git a/tests/test_networks/test_event/__init__.py b/tests/test_networks/test_event/__init__.py new file mode 100644 index 00000000..fed3716d --- /dev/null +++ b/tests/test_networks/test_event/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" diff --git a/tests/test_networks/test_event/test_dispatcher.py b/tests/test_networks/test_event/test_dispatcher.py new file mode 100644 index 00000000..f3397429 --- /dev/null +++ b/tests/test_networks/test_event/test_dispatcher.py @@ -0,0 +1,102 @@ + +import datetime +import aiopg + +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosEventHandler, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) +from minos.common import ( + Event +) +from minos.networks.exceptions import ( + MinosNetworkException, +) + + +class TestEventDispatcher(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosEventHandler.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosEventHandler) + + async def test_if_queue_table_exists(self): + event_handler = MinosEventHandler.from_config(config=self.config) + await event_handler.setup() + + async with aiopg.connect(**self.events_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'event_queue';" + ) + ret = [] + async for row in cur: + ret.append(row) + + assert ret == [(1,)] + + async def test_get_event_handler(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) + m = MinosEventHandler.from_config(config=self.config) + + cls = m.get_event_handler(topic="TicketAdded") + result = await cls(topic="TicketAdded", event=event_instance) + + assert result == "request_added" + + async def test_non_implemented_action(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="NotExisting", model=model.classname, items=[]) + m = MinosEventHandler.from_config(config=self.config) + + with self.assertRaises(MinosNetworkException) as context: + cls = m.get_event_handler(topic=event_instance.topic) + await cls(topic=event_instance.topic, event=event_instance) + + self.assertTrue("topic NotExisting have no controller/action configured, please review th configuration file" in str(context.exception)) + + async def test_none_config(self): + event_handler = MinosEventHandler.from_config(config=None) + + self.assertIsNone(event_handler) + + async def test_event_queue_checker(self): + event_handler = MinosEventHandler.from_config(config=self.config) + await event_handler.setup() + + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TicketAdded", model=model.classname, items=[]) + bin_data = event_instance.avro_bytes + Event.from_avro_bytes(bin_data) + + async with aiopg.connect(**self.events_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO event_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + (event_instance.topic, 0, bin_data, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + assert affected_rows == 1 + assert queue_id[0] > 0 + + + # Must get the record, call on_reply function and delete the record from DB + await event_handler.event_queue_checker() + + async with aiopg.connect(**self.events_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM event_queue WHERE id=%d" % (queue_id)) + records = await cur.fetchone() + + assert records[0] == 0 diff --git a/tests/test_networks/test_event/test_event_server.py b/tests/test_networks/test_event/test_event_server.py new file mode 100644 index 00000000..25e28bda --- /dev/null +++ b/tests/test_networks/test_event/test_event_server.py @@ -0,0 +1,128 @@ +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosEventServer, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) +from minos.common import ( + Event +) +from aiokafka import ( + AIOKafkaConsumer, + AIOKafkaProducer, +) + + +async def kafka_producer(config): + + # basic_config( + # level=logging.INFO, + # buffered=True, + # log_format='color', + # flush_interval=2 + # ) + kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" + producer = AIOKafkaProducer(bootstrap_servers=kafka_conn_data) + # Get cluster layout and topic/partition allocation + await producer.start() + # Produce messages + + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TicketAdded", model=model.classname, items=[model]) + bin_data = event_instance.avro_bytes + + model2 = NaiveAggregate(test_id=2, test=2, id=1, version=1) + event_instance_2 = Event(topic="TicketDeleted", model=model2.classname, items=[model2]) + bin_data2 = event_instance_2.avro_bytes + + for i in range(0, 10): + await producer.send_and_wait(event_instance.topic, bin_data) + await producer.send_and_wait(event_instance_2.topic, bin_data2) + + await producer.stop() + + +class TestEventServer(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosEventServer.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosEventServer) + + async def test_none_config(self): + event_server = MinosEventServer.from_config(config=None) + + self.assertIsNone(event_server) + + async def test_event_queue_add(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) + bin_data = event_instance.avro_bytes + Event.from_avro_bytes(bin_data) + + event_server = MinosEventServer.from_config(config=self.config) + await event_server.setup() + + affected_rows, id = await event_server.event_queue_add(topic=event_instance.topic, partition=0, binary=bin_data) + + assert affected_rows == 1 + assert id > 0 + + async def test_consumer_kafka(self): + await kafka_producer(self.config) + handler = {item.name: {'controller': item.controller, 'action': item.action} + for item in self.config.events.items} + topics = list(handler.keys()) + kafka_conn_data = f"{self.config.events.broker.host}:{self.config.events.broker.port}" + broker_group_name = f"event_{self.config.service.name}" + + event_server = MinosEventServer.from_config(config=self.config) + await event_server.setup() + + consumer = AIOKafkaConsumer( + loop=None, + group_id=broker_group_name, + auto_offset_reset="latest", + bootstrap_servers=kafka_conn_data, consumer_timeout_ms=500 + ) + + await consumer.start() + consumer.subscribe(topics) + + msg = await consumer.getone() + affected_rows, id = await event_server.handle_single_message(msg) + assert affected_rows > 0 + assert id > 0 + + await consumer.stop() + + """ + async def test_handle_message(self): + await kafka_producer(self.config) + handler = {item.name: {'controller': item.controller, 'action': item.action} + for item in self.config.events.items} + topics = list(handler.keys()) + kafka_conn_data = f"{self.config.events.broker.host}:{self.config.events.broker.port}" + broker_group_name = f"event_{self.config.service.name}" + + event_server = MinosEventServer.from_config(config=self.config) + await event_server.setup() + + consumer = AIOKafkaConsumer( + loop=None, + group_id=broker_group_name, + auto_offset_reset="latest", + bootstrap_servers=kafka_conn_data, consumer_timeout_ms=500 + ) + + await consumer.start() + consumer.subscribe(topics) + + await event_server.handle_message(consumer) + + await consumer.stop() + """ diff --git a/tests/test_networks/test_event/test_event_services.py b/tests/test_networks/test_event/test_event_services.py new file mode 100644 index 00000000..b4ed0537 --- /dev/null +++ b/tests/test_networks/test_event/test_event_services.py @@ -0,0 +1,44 @@ +import unittest +from unittest.mock import ( + MagicMock, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.networks import ( + MinosEventServerService, + MinosEventPeriodicService +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from tests.utils import ( + BASE_PATH, +) + + +""" +@pytest.fixture() +def services(config): + return [ + MinosEventServerService(config=config), + MinosEventPeriodicService(interval=0.5, delay=0, config=config), + ] +""" + +class TestMinosQueueService(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_start(self): + with self.config: + service = MinosEventPeriodicService(interval=1, loop=None) + service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) + await service.start() + self.assertTrue(1, service.dispatcher.setup.call_count) + + async def test_callback(self): + with self.config: + service = MinosEventPeriodicService(interval=1, loop=None) + service.dispatcher.event_queue_checker = MagicMock(side_effect=service.dispatcher.event_queue_checker) + await service.start() + self.assertEqual(1, service.dispatcher.event_queue_checker.call_count) From 637023bde85d708940bd0f0aad5e8e6f2a394c40 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 30 Apr 2021 18:10:52 +0200 Subject: [PATCH 155/239] Update test_event_server.py --- tests/test_networks/test_event/test_event_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_networks/test_event/test_event_server.py b/tests/test_networks/test_event/test_event_server.py index 25e28bda..148e64af 100644 --- a/tests/test_networks/test_event/test_event_server.py +++ b/tests/test_networks/test_event/test_event_server.py @@ -72,6 +72,7 @@ async def test_event_queue_add(self): assert affected_rows == 1 assert id > 0 + """ async def test_consumer_kafka(self): await kafka_producer(self.config) handler = {item.name: {'controller': item.controller, 'action': item.action} @@ -100,7 +101,7 @@ async def test_consumer_kafka(self): await consumer.stop() - """ + async def test_handle_message(self): await kafka_producer(self.config) handler = {item.name: {'controller': item.controller, 'action': item.action} From f1eb34f95c493ebde45a0904e88473dffd21520c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 10:46:51 +0200 Subject: [PATCH 156/239] ISSUE #88 * Add tests for exceptions. --- minos/networks/exceptions.py | 2 +- tests/test_networks/test_exceptions.py | 37 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/test_networks/test_exceptions.py diff --git a/minos/networks/exceptions.py b/minos/networks/exceptions.py index 5253c616..593d3924 100644 --- a/minos/networks/exceptions.py +++ b/minos/networks/exceptions.py @@ -15,7 +15,7 @@ class MinosNetworkException(MinosException): """Base network exception.""" -class MinosSnapshotException(MinosException): +class MinosSnapshotException(MinosNetworkException): """Base snapshot exception""" diff --git a/tests/test_networks/test_exceptions.py b/tests/test_networks/test_exceptions.py new file mode 100644 index 00000000..6ec84119 --- /dev/null +++ b/tests/test_networks/test_exceptions.py @@ -0,0 +1,37 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +import unittest + +from minos.common import MinosException +from minos.networks import MinosNetworkException, MinosSnapshotException, MinosPreviousVersionSnapshotException +from tests.aggregate_classes import Car + + +class TestExceptions(unittest.TestCase): + def test_type(self): + self.assertTrue(issubclass(MinosNetworkException, MinosException)) + + def test_snapshot(self): + self.assertTrue(issubclass(MinosSnapshotException, MinosNetworkException)) + + def test_snapshot_previous_version(self): + self.assertTrue(issubclass(MinosPreviousVersionSnapshotException, MinosSnapshotException)) + + def test_snapshot_previous_version_repr(self): + previous = Car(1, 2, 3, "blue") + new = Car(1, 1, 5, "blue") + exception = MinosPreviousVersionSnapshotException(previous, new) + expected = ( + "MinosPreviousVersionSnapshotException(message=\"Version for 'tests.aggregate_classes.Car' " + "aggregate must be greater than 2. Obtained: 1\")" + ) + self.assertEqual(expected, repr(exception)) + + +if __name__ == "__main__": + unittest.main() From 3c9d45bd377ab48007e468ef4fc18969ba78d4cb Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 3 May 2021 08:47:01 +0000 Subject: [PATCH 157/239] Restyled by black --- minos/networks/snapshots/dispatchers.py | 12 +++--------- tests/test_networks/test_exceptions.py | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 41804142..1bc5cc39 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( Any, @@ -29,12 +27,8 @@ import_module, ) -from ..exceptions import ( - MinosPreviousVersionSnapshotException, -) -from .entries import ( - MinosSnapshotEntry, -) +from ..exceptions import MinosPreviousVersionSnapshotException +from .entries import MinosSnapshotEntry class MinosSnapshotDispatcher(MinosSetup): diff --git a/tests/test_networks/test_exceptions.py b/tests/test_networks/test_exceptions.py index 6ec84119..bf7b7a5a 100644 --- a/tests/test_networks/test_exceptions.py +++ b/tests/test_networks/test_exceptions.py @@ -28,7 +28,7 @@ def test_snapshot_previous_version_repr(self): exception = MinosPreviousVersionSnapshotException(previous, new) expected = ( "MinosPreviousVersionSnapshotException(message=\"Version for 'tests.aggregate_classes.Car' " - "aggregate must be greater than 2. Obtained: 1\")" + 'aggregate must be greater than 2. Obtained: 1")' ) self.assertEqual(expected, repr(exception)) From dbcde2fda51ba1a9cf6782122b5ccb0bccae1fe7 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 3 May 2021 08:47:02 +0000 Subject: [PATCH 158/239] Restyled by isort --- minos/networks/snapshots/dispatchers.py | 12 +++++++++--- tests/test_networks/test_exceptions.py | 14 +++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 1bc5cc39..41804142 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( Any, @@ -27,8 +29,12 @@ import_module, ) -from ..exceptions import MinosPreviousVersionSnapshotException -from .entries import MinosSnapshotEntry +from ..exceptions import ( + MinosPreviousVersionSnapshotException, +) +from .entries import ( + MinosSnapshotEntry, +) class MinosSnapshotDispatcher(MinosSetup): diff --git a/tests/test_networks/test_exceptions.py b/tests/test_networks/test_exceptions.py index bf7b7a5a..590d28f3 100644 --- a/tests/test_networks/test_exceptions.py +++ b/tests/test_networks/test_exceptions.py @@ -7,9 +7,17 @@ """ import unittest -from minos.common import MinosException -from minos.networks import MinosNetworkException, MinosSnapshotException, MinosPreviousVersionSnapshotException -from tests.aggregate_classes import Car +from minos.common import ( + MinosException, +) +from minos.networks import ( + MinosNetworkException, + MinosPreviousVersionSnapshotException, + MinosSnapshotException, +) +from tests.aggregate_classes import ( + Car, +) class TestExceptions(unittest.TestCase): From 699a43d96c1008a3b50a6135b2366cf15634f350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 11:40:11 +0200 Subject: [PATCH 159/239] ISSUE #94 * Increase `minos.common` version up to `0.0.5`. * Apply changes to be compatible with the `minos.common==0.0.5` library. --- minos/networks/broker/abc.py | 70 ++++++------------- minos/networks/broker/dispatchers.py | 4 +- minos/networks/broker/services.py | 10 +++ minos/networks/snapshots/dispatchers.py | 65 ++++------------- minos/networks/snapshots/services.py | 10 +++ requirements_dev.txt | 2 +- .../test_networks/test_broker/test_command.py | 5 +- .../test_networks/test_broker/test_events.py | 5 +- .../test_networks/test_broker/test_service.py | 3 + .../test_snapshots/test_services.py | 3 + 10 files changed, 73 insertions(+), 104 deletions(-) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index d83c2082..979345af 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -15,54 +15,28 @@ NoReturn, ) -import aiopg from minos.common import ( MinosBaseBroker, - MinosSetup, + PostgreSqlMinosDatabase, ) -class MinosBrokerSetup(MinosSetup): +class MinosBrokerSetup(PostgreSqlMinosDatabase): """Minos Broker Setup Class""" - def __init__( - self, - *args, - host: str = None, - port: int = None, - database: str = None, - user: str = None, - password: str = None, - **kwargs - ): - super().__init__(*args, **kwargs) - - self.host = host - self.port = port - self.database = database - self.user = user - self.password = password - async def _setup(self) -> NoReturn: await self._create_broker_table() async def _create_broker_table(self) -> NoReturn: - async with self._connection() as connect: - async with connect.cursor() as cur: - await cur.execute( - 'CREATE TABLE IF NOT EXISTS "producer_queue" (' - '"id" BIGSERIAL NOT NULL PRIMARY KEY, ' - '"topic" VARCHAR(255) NOT NULL, ' - '"model" BYTEA NOT NULL, ' - '"retry" INTEGER NOT NULL, ' - '"action" VARCHAR(255) NOT NULL, ' - '"creation_date" TIMESTAMP NOT NULL, ' - '"update_date" TIMESTAMP NOT NULL);' - ) - - def _connection(self): - return aiopg.connect( - host=self.host, port=self.port, dbname=self.database, user=self.user, password=self.password, + await self.submit_query( + 'CREATE TABLE IF NOT EXISTS "producer_queue" (' + '"id" BIGSERIAL NOT NULL PRIMARY KEY, ' + '"topic" VARCHAR(255) NOT NULL, ' + '"model" BYTEA NOT NULL, ' + '"retry" INTEGER NOT NULL, ' + '"action" VARCHAR(255) NOT NULL, ' + '"creation_date" TIMESTAMP NOT NULL, ' + '"update_date" TIMESTAMP NOT NULL);' ) @@ -76,16 +50,16 @@ def __init__(self, topic: str, *args, **kwargs): MinosBrokerSetup.__init__(self, *args, **kwargs) async def _send_bytes(self, topic: str, raw: bytes) -> (int, int): - async with self._connection() as connect: - async with connect.cursor() as cur: - await cur.execute( - "INSERT INTO producer_queue (" - "topic, model, retry, action, creation_date, update_date" - ") VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", - (topic, raw, 0, self.ACTION, datetime.now(), datetime.now()), - ) - - queue_id = await cur.fetchone() - affected_rows = cur.rowcount + pool = await self.pool + with await pool.cursor() as cur: + await cur.execute( + "INSERT INTO producer_queue (" + "topic, model, retry, action, creation_date, update_date" + ") VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", + (topic, raw, 0, self.ACTION, datetime.now(), datetime.now()), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount return affected_rows, queue_id[0] diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index 1cc729f0..a024aff4 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -18,6 +18,7 @@ from aiokafka import ( AIOKafkaProducer, ) + from minos.common import ( MinosConfig, ) @@ -58,7 +59,8 @@ async def dispatch(self) -> NoReturn: :return: This method does not return anything. """ - async with self._connection() as connect: + pool = await self.pool + async with pool.acquire() as connect: async with connect.cursor() as cur: # noinspection SqlRedundantOrderingDirection await cur.execute( diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index c4626664..b81dbd8c 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -12,6 +12,7 @@ from aiomisc.service.periodic import ( PeriodicService, ) + from minos.common import ( MinosConfig, ) @@ -42,3 +43,12 @@ async def callback(self) -> None: :return:This method does not return anything. """ await self.dispatcher.dispatch() + + async def stop(self, err: Exception = None) -> None: + """Stop the service execution. + + :param err: Optional exception that stopped the execution. + :return: This method does not return anything. + """ + await super().stop(err) + await self.dispatcher.destroy() diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 75dd23b6..7326068e 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -17,14 +17,13 @@ Type, ) -import aiopg from minos.common import ( Aggregate, MinosConfig, MinosConfigException, MinosRepositoryAction, MinosRepositoryEntry, - MinosSetup, + PostgreSqlMinosDatabase, PostgreSqlMinosRepository, import_module, ) @@ -34,33 +33,20 @@ ) -class MinosSnapshotDispatcher(MinosSetup): +class MinosSnapshotDispatcher(PostgreSqlMinosDatabase): """Minos Snapshot Dispatcher class.""" - def __init__( - self, - *args, - host: str = None, - port: int = None, - database: str = None, - user: str = None, - password: str = None, - repository: dict[str, Any] = None, - offset: int = 0, - **kwargs - ): + def __init__(self, *args, repository: dict[str, Any] = None, offset: int = 0, **kwargs): super().__init__(*args, **kwargs) - self.host = host - self.port = port - self.database = database - self.user = user - self.password = password - self.offset = offset self.repository = PostgreSqlMinosRepository(**repository) + async def _destroy(self) -> NoReturn: + await super()._destroy() + await self.repository.destroy() + @classmethod def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> MinosSnapshotDispatcher: """Build a new Snapshot Dispatcher from config. @@ -80,7 +66,7 @@ async def _setup(self) -> NoReturn: await self._create_broker_table() async def _create_broker_table(self) -> NoReturn: - await self._submit_sql(_CREATE_TABLE_QUERY) + await self.submit_query(_CREATE_TABLE_QUERY) # noinspection PyUnusedLocal async def select(self, *args, **kwargs) -> AsyncIterator[MinosSnapshotEntry]: @@ -90,7 +76,7 @@ async def select(self, *args, **kwargs) -> AsyncIterator[MinosSnapshotEntry]: :param kwargs: Additional named arguments. :return: A sequence of ``MinosSnapshotEntry`` objects. """ - async for row in self._submit_and_iter_sql(_SELECT_ALL_ENTRIES_QUERY): + async for row in self.submit_query_and_iter(_SELECT_ALL_ENTRIES_QUERY): yield MinosSnapshotEntry(*row) async def dispatch(self) -> NoReturn: @@ -98,14 +84,9 @@ async def dispatch(self) -> NoReturn: :return: This method does not return anything. """ - async for entry in self._new_entries: + async for entry in self.repository.select(id_ge=self.offset): await self._dispatch_one(entry) - @property - async def _new_entries(self): - for entry in await self.repository.select(id_ge=self.offset): - yield entry - async def _dispatch_one(self, event_entry: MinosRepositoryEntry) -> Optional[MinosSnapshotEntry]: if event_entry.action is MinosRepositoryAction.DELETE: await self._submit_delete(event_entry) @@ -115,7 +96,7 @@ async def _dispatch_one(self, event_entry: MinosRepositoryEntry) -> Optional[Min async def _submit_delete(self, entry: MinosRepositoryEntry) -> NoReturn: params = {"aggregate_id": entry.aggregate_id, "aggregate_name": entry.aggregate_name} - await self._submit_sql(_DELETE_ONE_SNAPSHOT_ENTRY_QUERY, params) + await self.submit_query(_DELETE_ONE_SNAPSHOT_ENTRY_QUERY, params) async def _build_instance(self, event_entry: MinosRepositoryEntry) -> Aggregate: # noinspection PyTypeChecker @@ -137,7 +118,7 @@ async def _select_one_aggregate(self, aggregate_id: int, aggregate_name: str) -> return snapshot_entry.aggregate async def _select_one(self, aggregate_id: int, aggregate_name: str) -> MinosSnapshotEntry: - raw = await self._submit_and_fetchone_sql(_SELECT_ONE_SNAPSHOT_ENTRY_QUERY, (aggregate_id, aggregate_name)) + raw = await self.submit_query_and_fetchone(_SELECT_ONE_SNAPSHOT_ENTRY_QUERY, (aggregate_id, aggregate_name)) return MinosSnapshotEntry(aggregate_id, aggregate_name, *raw) async def _submit_instance(self, aggregate: Aggregate) -> MinosSnapshotEntry: @@ -152,32 +133,12 @@ async def _submit_update_or_create(self, entry: MinosSnapshotEntry) -> MinosSnap "version": entry.version, "data": entry.data, } - response = await self._submit_and_fetchone_sql(_INSERT_ONE_SNAPSHOT_ENTRY_QUERY, params) + response = await self.submit_query_and_fetchone(_INSERT_ONE_SNAPSHOT_ENTRY_QUERY, params) entry.created_at, entry.updated_at = response return entry - async def _submit_and_fetchone_sql(self, *args, **kwargs) -> tuple: - return await self._submit_and_iter_sql(*args, **kwargs).__anext__() - - async def _submit_and_iter_sql(self, query: str, *args, **kwargs) -> AsyncIterator[tuple]: - async with self._connection() as connect: - async with connect.cursor() as cursor: - await cursor.execute(query, *args, **kwargs) - async for row in cursor: - yield row - - async def _submit_sql(self, query: str, *args, **kwargs) -> NoReturn: - async with self._connection() as connect: - async with connect.cursor() as cursor: - await cursor.execute(query, *args, **kwargs) - - def _connection(self): - return aiopg.connect( - host=self.host, port=self.port, dbname=self.database, user=self.user, password=self.password - ) - _CREATE_TABLE_QUERY = """ CREATE TABLE IF NOT EXISTS snapshot ( diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 0fcb9a89..b6c9483f 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -9,6 +9,7 @@ from aiomisc.service.periodic import ( PeriodicService, ) + from minos.common import ( MinosConfig, ) @@ -39,3 +40,12 @@ async def callback(self) -> None: :return: This method does not return anything. """ await self.dispatcher.dispatch() + + async def stop(self, err: Exception = None) -> None: + """Stop the service execution. + + :param err: Optional exception that stopped the execution. + :return: This method does not return anything. + """ + await super().stop(err) + await self.dispatcher.destroy() diff --git a/requirements_dev.txt b/requirements_dev.txt index 213a82e1..05df0b5b 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -32,7 +32,7 @@ keyring==23.0.1 lmdb==1.1.1 MarkupSafe==1.1.1 mccabe==0.6.1 -minos-microservice-common==0.0.4 +minos-microservice-common==0.0.5 more-itertools==8.7.0 mypy-extensions==0.4.3 orjson==3.5.1 diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 3fa800af..38393e53 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -8,6 +8,7 @@ import unittest import aiopg + from minos.common import ( MinosConfig, ) @@ -69,7 +70,9 @@ async def test_if_commands_retry_was_incremented(self): affected_rows_2, queue_id_2 = await broker.send_one(item) config = MinosConfig( - path=BASE_PATH / "wrong_test_config.yml", events_queue_database="test_db", events_queue_user="test_user" + path=BASE_PATH / "wrong_test_config.yml", + events_queue_database=self.config.events.queue.database, + events_queue_user=self.config.events.queue.user, ) await MinosQueueDispatcher.from_config(config=config).dispatch() diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index 0519c064..0fb75116 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -1,6 +1,7 @@ import unittest import aiopg + from minos.common import ( MinosConfig, ) @@ -76,7 +77,9 @@ async def test_if_events_retry_was_incremented(self): affected_rows_2, queue_id_2 = await broker.send_one(item) config = MinosConfig( - path=BASE_PATH / "wrong_test_config.yml", events_queue_database="test_db", events_queue_user="test_user" + path=BASE_PATH / "wrong_test_config.yml", + events_queue_database=self.config.commands.queue.database, + events_queue_user=self.config.commands.queue.user, ) await MinosQueueDispatcher.from_config(config=config).dispatch() diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index b96cbba8..b08de5df 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -13,6 +13,7 @@ from aiomisc.service.periodic import ( PeriodicService, ) + from minos.common.testing import ( PostgresAsyncTestCase, ) @@ -53,6 +54,7 @@ async def test_start(self): service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) await service.start() self.assertTrue(1, service.dispatcher.setup.call_count) + await service.stop() async def test_callback(self): with self.config: @@ -60,6 +62,7 @@ async def test_callback(self): service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) await service.start() self.assertEqual(1, service.dispatcher.dispatch.call_count) + await service.stop() if __name__ == "__main__": diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 3c131d95..f3cc81bb 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -14,6 +14,7 @@ from aiomisc.service.periodic import ( PeriodicService, ) + from minos.common import ( MinosConfigException, ) @@ -58,6 +59,7 @@ async def test_start(self): service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) await service.start() self.assertTrue(1, service.dispatcher.setup.call_count) + await service.stop() async def test_callback(self): with self.config: @@ -65,6 +67,7 @@ async def test_callback(self): service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) await service.start() self.assertEqual(1, service.dispatcher.dispatch.call_count) + await service.stop() if __name__ == "__main__": From 2493817481b0bf0b9621de4dd009e0d6bddd04b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 12:04:59 +0200 Subject: [PATCH 160/239] ISSUE #94 * Raise exception instead of returning `None`. --- minos/networks/broker/dispatchers.py | 3 ++- tests/test_networks/test_broker/test_service.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index a024aff4..946faaaa 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -21,6 +21,7 @@ from minos.common import ( MinosConfig, + MinosConfigException, ) from .abc import ( @@ -50,7 +51,7 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi if config is None: config = MinosConfig.get_default() if config is None: - return None + raise MinosConfigException("The config object must be setup.") # noinspection PyProtectedMember return cls(*args, **config.events._asdict(), **kwargs) diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index b08de5df..e63d4f25 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -14,6 +14,9 @@ PeriodicService, ) +from minos.common import ( + MinosConfigException, +) from minos.common.testing import ( PostgresAsyncTestCase, ) @@ -34,8 +37,8 @@ def test_is_instance(self): self.assertIsInstance(service, PeriodicService) def test_dispatcher_empty(self): - service = MinosQueueService(interval=10) - self.assertIsNone(service.dispatcher) + with self.assertRaises(MinosConfigException): + MinosQueueService(interval=10) def test_dispatcher_config(self): service = MinosQueueService(interval=10, config=self.config) From cec9c30dd479a377cfc93c60d6345f1a9615418f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 12:08:58 +0200 Subject: [PATCH 161/239] ISSUE #94 * Fix stupid bug. --- tests/test_networks/test_broker/test_service.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index e63d4f25..9dfc51e1 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -33,8 +33,9 @@ class TestMinosQueueService(PostgresAsyncTestCase): CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" def test_is_instance(self): - service = MinosQueueService(interval=10) - self.assertIsInstance(service, PeriodicService) + with self.config: + service = MinosQueueService(interval=10) + self.assertIsInstance(service, PeriodicService) def test_dispatcher_empty(self): with self.assertRaises(MinosConfigException): From 024d21e004bf4b8940b7adcc86e7f431f1ed3649 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 3 May 2021 10:20:07 +0000 Subject: [PATCH 162/239] Restyled by black --- minos/networks/broker/abc.py | 12 +++-------- minos/networks/broker/dispatchers.py | 12 +++-------- minos/networks/broker/services.py | 16 ++++----------- minos/networks/snapshots/dispatchers.py | 8 ++------ minos/networks/snapshots/services.py | 12 +++-------- .../test_networks/test_broker/test_command.py | 8 ++------ .../test_networks/test_broker/test_events.py | 8 ++------ .../test_networks/test_broker/test_service.py | 20 +++++-------------- .../test_snapshots/test_services.py | 20 +++++-------------- 9 files changed, 29 insertions(+), 87 deletions(-) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index 979345af..51bf5c9d 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,15 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ( - ABC, -) -from datetime import ( - datetime, -) -from typing import ( - NoReturn, -) +from abc import ABC +from datetime import datetime +from typing import NoReturn from minos.common import ( MinosBaseBroker, diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index 946faaaa..40aadaa9 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NamedTuple, @@ -15,18 +13,14 @@ Optional, ) -from aiokafka import ( - AIOKafkaProducer, -) +from aiokafka import AIOKafkaProducer from minos.common import ( MinosConfig, MinosConfigException, ) -from .abc import ( - MinosBrokerSetup, -) +from .abc import MinosBrokerSetup class MinosQueueDispatcher(MinosBrokerSetup): diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index b81dbd8c..2f72bd56 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -5,21 +5,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations -from aiomisc.service.periodic import ( - PeriodicService, -) +from aiomisc.service.periodic import PeriodicService -from minos.common import ( - MinosConfig, -) +from minos.common import MinosConfig -from .dispatchers import ( - MinosQueueDispatcher, -) +from .dispatchers import MinosQueueDispatcher class MinosQueueService(PeriodicService): diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 7326068e..6a41e452 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( Any, @@ -28,9 +26,7 @@ import_module, ) -from .entries import ( - MinosSnapshotEntry, -) +from .entries import MinosSnapshotEntry class MinosSnapshotDispatcher(PostgreSqlMinosDatabase): diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index b6c9483f..84dbe392 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -6,17 +6,11 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from aiomisc.service.periodic import ( - PeriodicService, -) +from aiomisc.service.periodic import PeriodicService -from minos.common import ( - MinosConfig, -) +from minos.common import MinosConfig -from .dispatchers import ( - MinosSnapshotDispatcher, -) +from .dispatchers import MinosSnapshotDispatcher class MinosSnapshotService(PeriodicService): diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 38393e53..aabda441 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -9,12 +9,8 @@ import aiopg -from minos.common import ( - MinosConfig, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import MinosConfig +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index 0fb75116..5869eef9 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -2,12 +2,8 @@ import aiopg -from minos.common import ( - MinosConfig, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import MinosConfig +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosEventBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index 9dfc51e1..e7e964c4 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -6,27 +6,17 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ import unittest -from unittest.mock import ( - MagicMock, -) +from unittest.mock import MagicMock -from aiomisc.service.periodic import ( - PeriodicService, -) +from aiomisc.service.periodic import PeriodicService -from minos.common import ( - MinosConfigException, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import MinosConfigException +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosQueueDispatcher, MinosQueueService, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class TestMinosQueueService(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index f3cc81bb..fcd0d018 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -7,27 +7,17 @@ """ import unittest -from unittest.mock import ( - MagicMock, -) +from unittest.mock import MagicMock -from aiomisc.service.periodic import ( - PeriodicService, -) +from aiomisc.service.periodic import PeriodicService -from minos.common import ( - MinosConfigException, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import MinosConfigException +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosSnapshotDispatcher, MinosSnapshotService, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class TestMinosSnapshotService(PostgresAsyncTestCase): From b3494f24f9aafe636e01d0ae18c211ab7cb7b879 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 3 May 2021 10:20:15 +0000 Subject: [PATCH 163/239] Restyled by isort --- minos/networks/broker/abc.py | 12 ++++++++--- minos/networks/broker/dispatchers.py | 13 ++++++++---- minos/networks/broker/services.py | 21 ++++++++++++------- minos/networks/snapshots/dispatchers.py | 8 +++++-- minos/networks/snapshots/services.py | 15 ++++++++----- .../test_networks/test_broker/test_command.py | 9 +++++--- .../test_networks/test_broker/test_events.py | 9 +++++--- .../test_networks/test_broker/test_service.py | 21 +++++++++++++------ .../test_snapshots/test_services.py | 21 +++++++++++++------ 9 files changed, 90 insertions(+), 39 deletions(-) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index 51bf5c9d..979345af 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,9 +5,15 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ABC -from datetime import datetime -from typing import NoReturn +from abc import ( + ABC, +) +from datetime import ( + datetime, +) +from typing import ( + NoReturn, +) from minos.common import ( MinosBaseBroker, diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index 40aadaa9..e53e0037 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( NamedTuple, @@ -13,14 +15,17 @@ Optional, ) -from aiokafka import AIOKafkaProducer - +from aiokafka import ( + AIOKafkaProducer, +) from minos.common import ( MinosConfig, MinosConfigException, ) -from .abc import MinosBrokerSetup +from .abc import ( + MinosBrokerSetup, +) class MinosQueueDispatcher(MinosBrokerSetup): diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index 2f72bd56..b6b767bd 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -5,13 +5,20 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations - -from aiomisc.service.periodic import PeriodicService - -from minos.common import MinosConfig - -from .dispatchers import MinosQueueDispatcher +from __future__ import ( + annotations, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosQueueDispatcher, +) class MinosQueueService(PeriodicService): diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 6a41e452..7326068e 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( Any, @@ -26,7 +28,9 @@ import_module, ) -from .entries import MinosSnapshotEntry +from .entries import ( + MinosSnapshotEntry, +) class MinosSnapshotDispatcher(PostgreSqlMinosDatabase): diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 84dbe392..511193c6 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -6,11 +6,16 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from aiomisc.service.periodic import PeriodicService - -from minos.common import MinosConfig - -from .dispatchers import MinosSnapshotDispatcher +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosSnapshotDispatcher, +) class MinosSnapshotService(PeriodicService): diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index aabda441..700124eb 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -8,9 +8,12 @@ import unittest import aiopg - -from minos.common import MinosConfig -from minos.common.testing import PostgresAsyncTestCase +from minos.common import ( + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index 5869eef9..1b764074 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -1,9 +1,12 @@ import unittest import aiopg - -from minos.common import MinosConfig -from minos.common.testing import PostgresAsyncTestCase +from minos.common import ( + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosEventBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index e7e964c4..5aa825b3 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -6,17 +6,26 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ import unittest -from unittest.mock import MagicMock - -from aiomisc.service.periodic import PeriodicService +from unittest.mock import ( + MagicMock, +) -from minos.common import MinosConfigException -from minos.common.testing import PostgresAsyncTestCase +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosQueueDispatcher, MinosQueueService, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class TestMinosQueueService(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index fcd0d018..ec5857cd 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -7,17 +7,26 @@ """ import unittest -from unittest.mock import MagicMock - -from aiomisc.service.periodic import PeriodicService +from unittest.mock import ( + MagicMock, +) -from minos.common import MinosConfigException -from minos.common.testing import PostgresAsyncTestCase +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosSnapshotDispatcher, MinosSnapshotService, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class TestMinosSnapshotService(PostgresAsyncTestCase): From 1cbe6be46af57a3c7a805a144cebfe60b1df7a22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 12:25:33 +0200 Subject: [PATCH 164/239] ISSUE #94 * Minor change. --- tests/test_event_manager.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index f0bc87c4..6892cb5f 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -1,16 +1,14 @@ -import asyncio -import logging -import random -import string +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" import pytest from aiokafka import ( - AIOKafkaConsumer, AIOKafkaProducer, ) -from aiomisc.log import ( - basic_config, -) from minos.common import ( Aggregate, ) @@ -20,9 +18,6 @@ from minos.common.configuration.config import ( MinosConfig, ) -from minos.common.logs import ( - log, -) from minos.networks.event import ( EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService, @@ -33,12 +28,12 @@ ) -@pytest.fixture() +@pytest.fixture(scope="module") def config(): return MinosConfig(path="./tests/test_config.yml") -@pytest.fixture() +@pytest.fixture(scope="module") def services(config): return [ EventHandlerDatabaseInitializer(config=config), @@ -61,7 +56,6 @@ async def test_if_queue_table_exists(self): database = await self._database() async with database as connect: async with connect.cursor() as cur: - await cur.execute( "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'event_queue';" ) From 9918be08c44e3d8be2a826e04cf6081e1b65c7d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 12:44:30 +0200 Subject: [PATCH 165/239] ISSUE #94 * Refactor dispatch method (2). --- .idea/dataSources.xml | 12 ++++++++ minos/networks/broker/dispatchers.py | 43 ++++++++++++++-------------- 2 files changed, 33 insertions(+), 22 deletions(-) create mode 100644 .idea/dataSources.xml diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml new file mode 100644 index 00000000..3f7ec5fa --- /dev/null +++ b/.idea/dataSources.xml @@ -0,0 +1,12 @@ + + + + + postgresql + true + org.postgresql.Driver + jdbc:postgresql://localhost:5432/order_db + $ProjectFileDir$ + + + \ No newline at end of file diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index e53e0037..bded85e7 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -59,28 +59,22 @@ async def dispatch(self) -> NoReturn: :return: This method does not return anything. """ - pool = await self.pool - async with pool.acquire() as connect: - async with connect.cursor() as cur: - # noinspection SqlRedundantOrderingDirection - await cur.execute( - "SELECT * FROM producer_queue WHERE retry <= %d ORDER BY creation_date ASC LIMIT %d;" - % (self.retry, self.records), - ) - async for row in cur: - # noinspection PyBroadException - try: - published = await self.publish(topic=row[1], message=row[2]) - except Exception: - published = False - finally: - async with connect.cursor() as cur2: - if published: - # Delete from database If the event was sent successfully to Kafka. - await cur2.execute("DELETE FROM producer_queue WHERE id=%d;" % row[0]) - else: - # Update queue retry column. Increase by 1. - await cur2.execute("UPDATE producer_queue SET retry = retry + 1 WHERE id=%d;" % row[0]) + params = (self.retry, self.records) + async for row in self.submit_query_and_iter(_SELECT_NON_PROCESSED_ROWS_QUERY, params): + await self._dispatch_one(row) + + async def _dispatch_one(self, row: tuple) -> NoReturn: + # noinspection PyBroadException + try: + published = await self.publish(topic=row[1], message=row[2]) + except Exception: + published = False + + await self._update_queue_state(row, published) + + async def _update_queue_state(self, row: tuple, published: bool): + update_query = _DELETE_PROCESSED_QUERY if published else _UPDATE_NON_PROCESSED_QUERY + await self.submit_query(update_query, (row[0],)) async def publish(self, topic: str, message: bytes) -> bool: """Publish a new item in in the broker (kafka). @@ -104,3 +98,8 @@ async def publish(self, topic: str, message: bytes) -> bool: await producer.stop() return flag + + +_SELECT_NON_PROCESSED_ROWS_QUERY = "SELECT * FROM producer_queue WHERE retry <= %s ORDER BY creation_date ASC LIMIT %s;" +_DELETE_PROCESSED_QUERY = "DELETE FROM producer_queue WHERE id=%s;" +_UPDATE_NON_PROCESSED_QUERY = "UPDATE producer_queue SET retry = retry + 1 WHERE id=%s;" From 7cda40bbbc022592b4722f1abb62bdfd9c1f5fc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 12:45:36 +0200 Subject: [PATCH 166/239] ISSUE #94 * Drop file that mustn't be commited. --- .idea/dataSources.xml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 .idea/dataSources.xml diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml deleted file mode 100644 index 3f7ec5fa..00000000 --- a/.idea/dataSources.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - postgresql - true - org.postgresql.Driver - jdbc:postgresql://localhost:5432/order_db - $ProjectFileDir$ - - - \ No newline at end of file From 7ed747c4f1240793002c29befc7f6ce14c4d46c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 12:47:27 +0200 Subject: [PATCH 167/239] ISSUE #94 * Revert unwanted changes. --- tests/test_event_manager.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 6892cb5f..f0bc87c4 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -1,14 +1,16 @@ -""" -Copyright (C) 2021 Clariteia SL - -This file is part of minos framework. +import asyncio +import logging +import random +import string -Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. -""" import pytest from aiokafka import ( + AIOKafkaConsumer, AIOKafkaProducer, ) +from aiomisc.log import ( + basic_config, +) from minos.common import ( Aggregate, ) @@ -18,6 +20,9 @@ from minos.common.configuration.config import ( MinosConfig, ) +from minos.common.logs import ( + log, +) from minos.networks.event import ( EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService, @@ -28,12 +33,12 @@ ) -@pytest.fixture(scope="module") +@pytest.fixture() def config(): return MinosConfig(path="./tests/test_config.yml") -@pytest.fixture(scope="module") +@pytest.fixture() def services(config): return [ EventHandlerDatabaseInitializer(config=config), @@ -56,6 +61,7 @@ async def test_if_queue_table_exists(self): database = await self._database() async with database as connect: async with connect.cursor() as cur: + await cur.execute( "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'event_queue';" ) From 9d45517eff843f0c74e137769198baf8bc410e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 12:53:25 +0200 Subject: [PATCH 168/239] Revert "ISSUE #94 * Revert unwanted changes." This reverts commit 7ed747c4 --- tests/test_event_manager.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index f0bc87c4..6892cb5f 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -1,16 +1,14 @@ -import asyncio -import logging -import random -import string +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" import pytest from aiokafka import ( - AIOKafkaConsumer, AIOKafkaProducer, ) -from aiomisc.log import ( - basic_config, -) from minos.common import ( Aggregate, ) @@ -20,9 +18,6 @@ from minos.common.configuration.config import ( MinosConfig, ) -from minos.common.logs import ( - log, -) from minos.networks.event import ( EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService, @@ -33,12 +28,12 @@ ) -@pytest.fixture() +@pytest.fixture(scope="module") def config(): return MinosConfig(path="./tests/test_config.yml") -@pytest.fixture() +@pytest.fixture(scope="module") def services(config): return [ EventHandlerDatabaseInitializer(config=config), @@ -61,7 +56,6 @@ async def test_if_queue_table_exists(self): database = await self._database() async with database as connect: async with connect.cursor() as cur: - await cur.execute( "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'event_queue';" ) From 55c9524d28af7f5c34e1c15faf6e17a9e3e09031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 12:58:51 +0200 Subject: [PATCH 169/239] ISSUE #94 * Minor refactor on publish method. --- minos/networks/broker/dispatchers.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index bded85e7..d8541806 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -65,11 +65,7 @@ async def dispatch(self) -> NoReturn: async def _dispatch_one(self, row: tuple) -> NoReturn: # noinspection PyBroadException - try: - published = await self.publish(topic=row[1], message=row[2]) - except Exception: - published = False - + published = await self.publish(topic=row[1], message=row[2]) await self._update_queue_state(row, published) async def _update_queue_state(self, row: tuple, published: bool): @@ -84,20 +80,20 @@ async def publish(self, topic: str, message: bytes) -> bool: :return: A boolean flag, ``True`` when the message is properly published or ``False`` otherwise. """ producer = AIOKafkaProducer(bootstrap_servers=f"{self.broker.host}:{self.broker.port}") - # Get cluster layout and initial topic/partition leadership information - await producer.start() # noinspection PyBroadException try: + # Get cluster layout and initial topic/partition leadership information + await producer.start() # Produce message await producer.send_and_wait(topic, message) - flag = True - except Exception: - flag = False - finally: # Wait for all pending messages to be delivered or expire. await producer.stop() - return flag + published = True + except Exception: + published = False + + return published _SELECT_NON_PROCESSED_ROWS_QUERY = "SELECT * FROM producer_queue WHERE retry <= %s ORDER BY creation_date ASC LIMIT %s;" From ab353faf6fd0b1ea5db43348974b66078b6473a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 13:14:17 +0200 Subject: [PATCH 170/239] ISSUE #94 * Refactor `send` method. --- minos/networks/broker/abc.py | 44 +++++++++---------- minos/networks/broker/commands.py | 2 +- minos/networks/broker/events.py | 2 +- .../test_networks/test_broker/test_command.py | 15 +++---- .../test_networks/test_broker/test_events.py | 15 +++---- 5 files changed, 34 insertions(+), 44 deletions(-) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index 979345af..b2c01d9e 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -28,16 +28,7 @@ async def _setup(self) -> NoReturn: await self._create_broker_table() async def _create_broker_table(self) -> NoReturn: - await self.submit_query( - 'CREATE TABLE IF NOT EXISTS "producer_queue" (' - '"id" BIGSERIAL NOT NULL PRIMARY KEY, ' - '"topic" VARCHAR(255) NOT NULL, ' - '"model" BYTEA NOT NULL, ' - '"retry" INTEGER NOT NULL, ' - '"action" VARCHAR(255) NOT NULL, ' - '"creation_date" TIMESTAMP NOT NULL, ' - '"update_date" TIMESTAMP NOT NULL);' - ) + await self.submit_query(_CREATE_TABLE_QUERY) class MinosBroker(MinosBaseBroker, MinosBrokerSetup, ABC): @@ -49,17 +40,26 @@ def __init__(self, topic: str, *args, **kwargs): MinosBaseBroker.__init__(self, topic) MinosBrokerSetup.__init__(self, *args, **kwargs) - async def _send_bytes(self, topic: str, raw: bytes) -> (int, int): - pool = await self.pool - with await pool.cursor() as cur: - await cur.execute( - "INSERT INTO producer_queue (" - "topic, model, retry, action, creation_date, update_date" - ") VALUES (%s, %s, %s, %s, %s, %s) RETURNING id;", - (topic, raw, 0, self.ACTION, datetime.now(), datetime.now()), - ) + async def _send_bytes(self, topic: str, raw: bytes) -> int: + params = (topic, raw, 0, self.ACTION, datetime.now(), datetime.now()) + raw = await self.submit_query_and_fetchone(_INSERT_ENTRY_QUERY, params) + return raw[0] - queue_id = await cur.fetchone() - affected_rows = cur.rowcount - return affected_rows, queue_id[0] +_CREATE_TABLE_QUERY = """ +CREATE TABLE IF NOT EXISTS producer_queue ( + id BIGSERIAL NOT NULL PRIMARY KEY, + topic VARCHAR(255) NOT NULL, + model BYTEA NOT NULL, + retry INTEGER NOT NULL, + action VARCHAR(255) NOT NULL, + creation_date TIMESTAMP NOT NULL, + update_date TIMESTAMP NOT NULL +); +""".strip() + +_INSERT_ENTRY_QUERY = """ +INSERT INTO producer_queue (topic, model, retry, action, creation_date, update_date) +VALUES (%s, %s, %s, %s, %s, %s) +RETURNING id; +""".strip() diff --git a/minos/networks/broker/commands.py b/minos/networks/broker/commands.py index 4cd1bd98..29fe5e86 100644 --- a/minos/networks/broker/commands.py +++ b/minos/networks/broker/commands.py @@ -49,7 +49,7 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi # noinspection PyProtectedMember return cls(*args, **config.commands.queue._asdict(), **kwargs) - async def send(self, items: list[Aggregate]) -> NoReturn: + async def send(self, items: list[Aggregate]) -> int: """Send a list of ``Aggregate`` instances. :param items: A list of aggregates. diff --git a/minos/networks/broker/events.py b/minos/networks/broker/events.py index 64de7705..b19b587a 100644 --- a/minos/networks/broker/events.py +++ b/minos/networks/broker/events.py @@ -44,7 +44,7 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi # noinspection PyProtectedMember return cls(*args, **config.events.queue._asdict(), **kwargs) - async def send(self, items: list[Aggregate]) -> (int, int): + async def send(self, items: list[Aggregate]) -> int: """Send a list of ``Aggregate`` instances. :param items: A list of aggregates. diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 700124eb..15780838 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -33,8 +33,7 @@ async def test_commands_broker_insertion(self): item = NaiveAggregate(test_id=1, test=2, id=1, version=1) - affected_rows, queue_id = await broker.send_one(item) - assert affected_rows == 1 + queue_id = await broker.send_one(item) assert queue_id > 0 async def test_if_commands_was_deleted(self): @@ -43,8 +42,8 @@ async def test_if_commands_was_deleted(self): item = NaiveAggregate(test_id=1, test=2, id=1, version=1) - affected_rows_1, queue_id_1 = await broker.send_one(item) - affected_rows_2, queue_id_2 = await broker.send_one(item) + queue_id_1 = await broker.send_one(item) + queue_id_2 = await broker.send_one(item) await MinosQueueDispatcher.from_config(config=self.config).dispatch() @@ -53,9 +52,7 @@ async def test_if_commands_was_deleted(self): await cursor.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "CommandBroker-Delete") records = await cursor.fetchone() - assert affected_rows_1 == 1 assert queue_id_1 > 0 - assert affected_rows_2 == 1 assert queue_id_2 > 0 assert records[0] == 0 @@ -65,8 +62,8 @@ async def test_if_commands_retry_was_incremented(self): item = NaiveAggregate(test_id=1, test=2, id=1, version=1) - affected_rows_1, queue_id_1 = await broker.send_one(item) - affected_rows_2, queue_id_2 = await broker.send_one(item) + queue_id_1 = await broker.send_one(item) + queue_id_2 = await broker.send_one(item) config = MinosConfig( path=BASE_PATH / "wrong_test_config.yml", @@ -86,9 +83,7 @@ async def test_if_commands_retry_was_incremented(self): await cursor.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_2) retry_2 = await cursor.fetchone() - assert affected_rows_1 == 1 assert queue_id_1 > 0 - assert affected_rows_2 == 1 assert queue_id_2 > 0 assert records[0] == 2 assert retry_1[0] > 0 diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index 1b764074..5cbcd721 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -40,9 +40,8 @@ async def test_events_broker_insertion(self): await broker.setup() item = NaiveAggregate(test_id=1, test=2, id=1, version=1) - affected_rows, queue_id = await broker.send_one(item) + queue_id = await broker.send_one(item) - assert affected_rows == 1 assert queue_id > 0 async def test_if_events_was_deleted(self): @@ -50,8 +49,8 @@ async def test_if_events_was_deleted(self): await broker.setup() item = NaiveAggregate(test_id=1, test=2, id=1, version=1) - affected_rows_1, queue_id_1 = await broker.send_one(item) - affected_rows_2, queue_id_2 = await broker.send_one(item) + queue_id_1 = await broker.send_one(item) + queue_id_2 = await broker.send_one(item) await MinosQueueDispatcher.from_config(config=self.config).dispatch() @@ -60,9 +59,7 @@ async def test_if_events_was_deleted(self): await cursor.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "EventBroker-Delete") records = await cursor.fetchone() - assert affected_rows_1 == 1 assert queue_id_1 > 0 - assert affected_rows_2 == 1 assert queue_id_2 > 0 assert records[0] == 0 @@ -72,8 +69,8 @@ async def test_if_events_retry_was_incremented(self): item = NaiveAggregate(test_id=1, test=2, id=1, version=1) - affected_rows_1, queue_id_1 = await broker.send_one(item) - affected_rows_2, queue_id_2 = await broker.send_one(item) + queue_id_1 = await broker.send_one(item) + queue_id_2 = await broker.send_one(item) config = MinosConfig( path=BASE_PATH / "wrong_test_config.yml", @@ -94,9 +91,7 @@ async def test_if_events_retry_was_incremented(self): await cursor.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_2) retry_2 = await cursor.fetchone() - assert affected_rows_1 == 1 assert queue_id_1 > 0 - assert affected_rows_2 == 1 assert queue_id_2 > 0 assert records[0] == 2 assert retry_1[0] > 0 From 630714586b127303642b9f81a426fefd178eda91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 13:24:08 +0200 Subject: [PATCH 171/239] ISSUE #94 * Minor change. --- minos/networks/broker/dispatchers.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index d8541806..de99d406 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -12,7 +12,7 @@ from typing import ( NamedTuple, NoReturn, - Optional, + Optional, AsyncIterator, ) from aiokafka import ( @@ -59,9 +59,20 @@ async def dispatch(self) -> NoReturn: :return: This method does not return anything. """ + async for row in self.select(): + await self._dispatch_one(row) + + # noinspection PyUnusedLocal + async def select(self, *args, **kwargs) -> AsyncIterator[tuple]: + """Select a sequence of ``MinosSnapshotEntry`` objects. + + :param args: Additional positional arguments. + :param kwargs: Additional named arguments. + :return: A sequence of ``MinosSnapshotEntry`` objects. + """ params = (self.retry, self.records) async for row in self.submit_query_and_iter(_SELECT_NON_PROCESSED_ROWS_QUERY, params): - await self._dispatch_one(row) + yield row async def _dispatch_one(self, row: tuple) -> NoReturn: # noinspection PyBroadException From 4fd6daf8519a7c9eebbc3b10574e1994a24d21b6 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 3 May 2021 11:24:18 +0000 Subject: [PATCH 172/239] Restyled by black --- minos/networks/broker/abc.py | 12 +++-------- minos/networks/broker/commands.py | 8 ++------ minos/networks/broker/dispatchers.py | 15 +++++--------- minos/networks/broker/events.py | 12 +++-------- minos/networks/broker/services.py | 20 ++++++------------- minos/networks/snapshots/dispatchers.py | 8 ++------ minos/networks/snapshots/services.py | 14 ++++--------- tests/test_event_manager.py | 20 +++++-------------- .../test_networks/test_broker/test_command.py | 8 ++------ .../test_networks/test_broker/test_events.py | 8 ++------ .../test_networks/test_broker/test_service.py | 20 +++++-------------- .../test_snapshots/test_services.py | 20 +++++-------------- 12 files changed, 44 insertions(+), 121 deletions(-) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index b2c01d9e..ff832667 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,15 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ( - ABC, -) -from datetime import ( - datetime, -) -from typing import ( - NoReturn, -) +from abc import ABC +from datetime import datetime +from typing import NoReturn from minos.common import ( MinosBaseBroker, diff --git a/minos/networks/broker/commands.py b/minos/networks/broker/commands.py index 29fe5e86..77730e80 100644 --- a/minos/networks/broker/commands.py +++ b/minos/networks/broker/commands.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NoReturn, @@ -20,9 +18,7 @@ MinosConfig, ) -from .abc import ( - MinosBroker, -) +from .abc import MinosBroker class MinosCommandBroker(MinosBroker): diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index de99d406..528466fb 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -5,27 +5,22 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NamedTuple, NoReturn, - Optional, AsyncIterator, + Optional, + AsyncIterator, ) -from aiokafka import ( - AIOKafkaProducer, -) +from aiokafka import AIOKafkaProducer from minos.common import ( MinosConfig, MinosConfigException, ) -from .abc import ( - MinosBrokerSetup, -) +from .abc import MinosBrokerSetup class MinosQueueDispatcher(MinosBrokerSetup): diff --git a/minos/networks/broker/events.py b/minos/networks/broker/events.py index b19b587a..f9034980 100644 --- a/minos/networks/broker/events.py +++ b/minos/networks/broker/events.py @@ -5,13 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations -from typing import ( - Optional, -) +from typing import Optional from minos.common import ( Aggregate, @@ -19,9 +15,7 @@ MinosConfig, ) -from .abc import ( - MinosBroker, -) +from .abc import MinosBroker class MinosEventBroker(MinosBroker): diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index b6b767bd..edfae788 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -5,20 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) - -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) - -from .dispatchers import ( - MinosQueueDispatcher, -) +from __future__ import annotations + +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig + +from .dispatchers import MinosQueueDispatcher class MinosQueueService(PeriodicService): diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 7326068e..6a41e452 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( Any, @@ -28,9 +26,7 @@ import_module, ) -from .entries import ( - MinosSnapshotEntry, -) +from .entries import MinosSnapshotEntry class MinosSnapshotDispatcher(PostgreSqlMinosDatabase): diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 511193c6..b40cba06 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -6,16 +6,10 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) - -from .dispatchers import ( - MinosSnapshotDispatcher, -) +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig + +from .dispatchers import MinosSnapshotDispatcher class MinosSnapshotService(PeriodicService): diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 6892cb5f..ce9a8c4c 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -6,26 +6,16 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ import pytest -from aiokafka import ( - AIOKafkaProducer, -) -from minos.common import ( - Aggregate, -) -from minos.common.broker import ( - Event, -) -from minos.common.configuration.config import ( - MinosConfig, -) +from aiokafka import AIOKafkaProducer +from minos.common import Aggregate +from minos.common.broker import Event +from minos.common.configuration.config import MinosConfig from minos.networks.event import ( EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService, MinosEventServer, ) -from tests.database_testcase import ( - EventHandlerPostgresAsyncTestCase, -) +from tests.database_testcase import EventHandlerPostgresAsyncTestCase @pytest.fixture(scope="module") diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 15780838..8bde9b72 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -8,12 +8,8 @@ import unittest import aiopg -from minos.common import ( - MinosConfig, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import MinosConfig +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index 5cbcd721..14f6443d 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -1,12 +1,8 @@ import unittest import aiopg -from minos.common import ( - MinosConfig, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import MinosConfig +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosEventBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index 5aa825b3..2b63ed7b 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -6,26 +6,16 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ import unittest -from unittest.mock import ( - MagicMock, -) +from unittest.mock import MagicMock -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfigException, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfigException +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosQueueDispatcher, MinosQueueService, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class TestMinosQueueService(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index ec5857cd..5b79c481 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -7,26 +7,16 @@ """ import unittest -from unittest.mock import ( - MagicMock, -) +from unittest.mock import MagicMock -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfigException, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfigException +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosSnapshotDispatcher, MinosSnapshotService, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class TestMinosSnapshotService(PostgresAsyncTestCase): From 055fd302637c0467001b1bd710b27c0b712eb2b9 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 3 May 2021 11:24:19 +0000 Subject: [PATCH 173/239] Restyled by isort --- minos/networks/broker/abc.py | 12 ++++++++--- minos/networks/broker/commands.py | 8 ++++++-- minos/networks/broker/dispatchers.py | 14 +++++++++---- minos/networks/broker/events.py | 12 ++++++++--- minos/networks/broker/services.py | 20 +++++++++++++------ minos/networks/snapshots/dispatchers.py | 8 ++++++-- minos/networks/snapshots/services.py | 14 +++++++++---- tests/test_event_manager.py | 20 ++++++++++++++----- .../test_networks/test_broker/test_command.py | 8 ++++++-- .../test_networks/test_broker/test_events.py | 8 ++++++-- .../test_networks/test_broker/test_service.py | 20 ++++++++++++++----- .../test_snapshots/test_services.py | 20 ++++++++++++++----- 12 files changed, 121 insertions(+), 43 deletions(-) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index ff832667..b2c01d9e 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,9 +5,15 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ABC -from datetime import datetime -from typing import NoReturn +from abc import ( + ABC, +) +from datetime import ( + datetime, +) +from typing import ( + NoReturn, +) from minos.common import ( MinosBaseBroker, diff --git a/minos/networks/broker/commands.py b/minos/networks/broker/commands.py index 77730e80..29fe5e86 100644 --- a/minos/networks/broker/commands.py +++ b/minos/networks/broker/commands.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( NoReturn, @@ -18,7 +20,9 @@ MinosConfig, ) -from .abc import MinosBroker +from .abc import ( + MinosBroker, +) class MinosCommandBroker(MinosBroker): diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index 528466fb..49e39234 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -5,22 +5,28 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( + AsyncIterator, NamedTuple, NoReturn, Optional, - AsyncIterator, ) -from aiokafka import AIOKafkaProducer +from aiokafka import ( + AIOKafkaProducer, +) from minos.common import ( MinosConfig, MinosConfigException, ) -from .abc import MinosBrokerSetup +from .abc import ( + MinosBrokerSetup, +) class MinosQueueDispatcher(MinosBrokerSetup): diff --git a/minos/networks/broker/events.py b/minos/networks/broker/events.py index f9034980..b19b587a 100644 --- a/minos/networks/broker/events.py +++ b/minos/networks/broker/events.py @@ -5,9 +5,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) -from typing import Optional +from typing import ( + Optional, +) from minos.common import ( Aggregate, @@ -15,7 +19,9 @@ MinosConfig, ) -from .abc import MinosBroker +from .abc import ( + MinosBroker, +) class MinosEventBroker(MinosBroker): diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index edfae788..b6b767bd 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -5,12 +5,20 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations - -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig - -from .dispatchers import MinosQueueDispatcher +from __future__ import ( + annotations, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosQueueDispatcher, +) class MinosQueueService(PeriodicService): diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 6a41e452..7326068e 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( Any, @@ -26,7 +28,9 @@ import_module, ) -from .entries import MinosSnapshotEntry +from .entries import ( + MinosSnapshotEntry, +) class MinosSnapshotDispatcher(PostgreSqlMinosDatabase): diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index b40cba06..511193c6 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -6,10 +6,16 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig - -from .dispatchers import MinosSnapshotDispatcher +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosSnapshotDispatcher, +) class MinosSnapshotService(PeriodicService): diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index ce9a8c4c..6892cb5f 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -6,16 +6,26 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ import pytest -from aiokafka import AIOKafkaProducer -from minos.common import Aggregate -from minos.common.broker import Event -from minos.common.configuration.config import MinosConfig +from aiokafka import ( + AIOKafkaProducer, +) +from minos.common import ( + Aggregate, +) +from minos.common.broker import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) from minos.networks.event import ( EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService, MinosEventServer, ) -from tests.database_testcase import EventHandlerPostgresAsyncTestCase +from tests.database_testcase import ( + EventHandlerPostgresAsyncTestCase, +) @pytest.fixture(scope="module") diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 8bde9b72..15780838 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -8,8 +8,12 @@ import unittest import aiopg -from minos.common import MinosConfig -from minos.common.testing import PostgresAsyncTestCase +from minos.common import ( + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index 14f6443d..5cbcd721 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -1,8 +1,12 @@ import unittest import aiopg -from minos.common import MinosConfig -from minos.common.testing import PostgresAsyncTestCase +from minos.common import ( + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosEventBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index 2b63ed7b..5aa825b3 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -6,16 +6,26 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ import unittest -from unittest.mock import MagicMock +from unittest.mock import ( + MagicMock, +) -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfigException -from minos.common.testing import PostgresAsyncTestCase +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosQueueDispatcher, MinosQueueService, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class TestMinosQueueService(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 5b79c481..ec5857cd 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -7,16 +7,26 @@ """ import unittest -from unittest.mock import MagicMock +from unittest.mock import ( + MagicMock, +) -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfigException -from minos.common.testing import PostgresAsyncTestCase +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosSnapshotDispatcher, MinosSnapshotService, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class TestMinosSnapshotService(PostgresAsyncTestCase): From 4341493a1db8b06ef09688be393f8b1266dbc514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 13:36:59 +0200 Subject: [PATCH 174/239] ISSUE #94 * Minor change (2). --- minos/networks/broker/dispatchers.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index de99d406..025c0307 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -107,6 +107,21 @@ async def publish(self, topic: str, message: bytes) -> bool: return published -_SELECT_NON_PROCESSED_ROWS_QUERY = "SELECT * FROM producer_queue WHERE retry <= %s ORDER BY creation_date ASC LIMIT %s;" -_DELETE_PROCESSED_QUERY = "DELETE FROM producer_queue WHERE id=%s;" -_UPDATE_NON_PROCESSED_QUERY = "UPDATE producer_queue SET retry = retry + 1 WHERE id=%s;" +_SELECT_NON_PROCESSED_ROWS_QUERY = """ +SELECT * +FROM producer_queue +WHERE retry <= %s +ORDER BY creation_date +LIMIT %s; +""".strip() + +_DELETE_PROCESSED_QUERY = """ +DELETE FROM producer_queue +WHERE id = %s; +""".strip() + +_UPDATE_NON_PROCESSED_QUERY = """ +UPDATE producer_queue + SET retry = retry + 1 +WHERE id = %s; +""".strip() From ac8cd143653b9dc64f31612e36badb28830eeefc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 13:42:10 +0200 Subject: [PATCH 175/239] ISSUE #94 * Minor change (3). --- minos/networks/broker/dispatchers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index 91477ad6..2b283733 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -55,6 +55,11 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi # noinspection PyProtectedMember return cls(*args, **config.events._asdict(), **kwargs) + async def _destroy(self) -> NoReturn: + await super()._destroy() + if self._pool is not None: + self._pool.terminate() + async def dispatch(self) -> NoReturn: """Dispatch the items in the publishing queue. From b6b5afc8935cd675b3b82831886ecaec9280ece4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 13:47:06 +0200 Subject: [PATCH 176/239] ISSUE #94 * Minor change (4). --- tests/test_event_manager.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 6892cb5f..4319fa8f 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -28,18 +28,18 @@ ) -@pytest.fixture(scope="module") -def config(): - return MinosConfig(path="./tests/test_config.yml") - - -@pytest.fixture(scope="module") -def services(config): - return [ - EventHandlerDatabaseInitializer(config=config), - MinosEventServer(conf=config), - MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config), - ] +# @pytest.fixture(scope="module") +# def config(): +# return MinosConfig(path="./tests/test_config.yml") +# +# +# @pytest.fixture(scope="module") +# def services(config): +# return [ +# EventHandlerDatabaseInitializer(config=config), +# MinosEventServer(conf=config), +# MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config), +# ] class AggregateTest(Aggregate): From 31e2d3529fe0e3157a73409b35f66cc286e8853f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 13:49:56 +0200 Subject: [PATCH 177/239] Revert "ISSUE #94 * Minor change (4)." This reverts commit b6b5afc8 --- tests/test_event_manager.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 4319fa8f..6892cb5f 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -28,18 +28,18 @@ ) -# @pytest.fixture(scope="module") -# def config(): -# return MinosConfig(path="./tests/test_config.yml") -# -# -# @pytest.fixture(scope="module") -# def services(config): -# return [ -# EventHandlerDatabaseInitializer(config=config), -# MinosEventServer(conf=config), -# MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config), -# ] +@pytest.fixture(scope="module") +def config(): + return MinosConfig(path="./tests/test_config.yml") + + +@pytest.fixture(scope="module") +def services(config): + return [ + EventHandlerDatabaseInitializer(config=config), + MinosEventServer(conf=config), + MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config), + ] class AggregateTest(Aggregate): From 596816cc9b666a3dcca5f0e4dac0dc6001b9f1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 13:56:02 +0200 Subject: [PATCH 178/239] ISSUE #94 * Minor change (5). --- .idea/minos_microservice_networks.iml | 3 +++ minos/networks/broker/dispatchers.py | 5 ----- tests/test_networks/test_broker/test_service.py | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.idea/minos_microservice_networks.iml b/.idea/minos_microservice_networks.iml index de5fcde3..1b9c4c49 100644 --- a/.idea/minos_microservice_networks.iml +++ b/.idea/minos_microservice_networks.iml @@ -3,6 +3,9 @@ + + + diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index 2b283733..91477ad6 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -55,11 +55,6 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi # noinspection PyProtectedMember return cls(*args, **config.events._asdict(), **kwargs) - async def _destroy(self) -> NoReturn: - await super()._destroy() - if self._pool is not None: - self._pool.terminate() - async def dispatch(self) -> NoReturn: """Dispatch the items in the publishing queue. diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index 5aa825b3..0dd88af8 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -53,7 +53,7 @@ def test_dispatcher_config_context(self): async def test_start(self): with self.config: - service = MinosQueueService(interval=1, loop=None) + service = MinosQueueService(interval=10, loop=None) service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) await service.start() self.assertTrue(1, service.dispatcher.setup.call_count) @@ -61,7 +61,7 @@ async def test_start(self): async def test_callback(self): with self.config: - service = MinosQueueService(interval=1, loop=None) + service = MinosQueueService(interval=10, loop=None) service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) await service.start() self.assertEqual(1, service.dispatcher.dispatch.call_count) From 6f57c20d8d73a12d8818cb191318b7c1a12a5167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 14:01:48 +0200 Subject: [PATCH 179/239] ISSUE #94 * Minor change (6). --- minos/networks/broker/services.py | 2 +- minos/networks/snapshots/services.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index b6b767bd..4d8f14cc 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -49,5 +49,5 @@ async def stop(self, err: Exception = None) -> None: :param err: Optional exception that stopped the execution. :return: This method does not return anything. """ - await super().stop(err) await self.dispatcher.destroy() + await super().stop(err) diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 511193c6..2443a1bf 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -46,5 +46,5 @@ async def stop(self, err: Exception = None) -> None: :param err: Optional exception that stopped the execution. :return: This method does not return anything. """ - await super().stop(err) await self.dispatcher.destroy() + await super().stop(err) From 6822b56d178f7bd7e10ac446fa9006332c87463b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 14:31:56 +0200 Subject: [PATCH 180/239] ISSUE #94 * Increase coverage. --- tests/test_networks/test_broker/test_dispatcher.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py index 8a15dbf2..9356b37e 100644 --- a/tests/test_networks/test_broker/test_dispatcher.py +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -1,5 +1,8 @@ import unittest +from minos.common import ( + MinosConfigException, +) from minos.common.testing import ( PostgresAsyncTestCase, ) @@ -18,6 +21,14 @@ def test_from_config(self): dispatcher = MinosQueueDispatcher.from_config(config=self.config) self.assertIsInstance(dispatcher, MinosQueueDispatcher) + def test_from_config_raises(self): + with self.assertRaises(MinosConfigException): + MinosQueueDispatcher.from_config() + + async def test_select(self): + dispatcher = MinosQueueDispatcher.from_config(config=self.config) + self.assertEqual([], [v async for v in dispatcher.select()]) + async def test_send_to_kafka_ok(self): dispatcher = MinosQueueDispatcher.from_config(config=self.config) response = await dispatcher.publish(topic="TestKafkaSend", message=bytes()) From 7e26b08ac67c8ae79411d31641592ca983f1d354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 14:32:56 +0200 Subject: [PATCH 181/239] ISSUE #94 * Fix bug on test. --- tests/test_networks/test_broker/test_dispatcher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py index 9356b37e..be51f22d 100644 --- a/tests/test_networks/test_broker/test_dispatcher.py +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -27,6 +27,7 @@ def test_from_config_raises(self): async def test_select(self): dispatcher = MinosQueueDispatcher.from_config(config=self.config) + await dispatcher.setup() self.assertEqual([], [v async for v in dispatcher.select()]) async def test_send_to_kafka_ok(self): From 288b1c15d7a88951acb298c4bfb6c377f6d9a871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 14:39:00 +0200 Subject: [PATCH 182/239] ISSUE #94 * Minor change. --- .../test_networks/test_broker/test_service.py | 22 +++++++++---------- .../test_snapshots/test_services.py | 22 +++++++++---------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index 0dd88af8..3416b989 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -52,20 +52,18 @@ def test_dispatcher_config_context(self): self.assertIsInstance(service.dispatcher, MinosQueueDispatcher) async def test_start(self): - with self.config: - service = MinosQueueService(interval=10, loop=None) - service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) - await service.start() - self.assertTrue(1, service.dispatcher.setup.call_count) - await service.stop() + service = MinosQueueService(interval=10, loop=None, config=self.config) + service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) + await service.start() + self.assertTrue(1, service.dispatcher.setup.call_count) + await service.stop() async def test_callback(self): - with self.config: - service = MinosQueueService(interval=10, loop=None) - service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) - await service.start() - self.assertEqual(1, service.dispatcher.dispatch.call_count) - await service.stop() + service = MinosQueueService(interval=10, loop=None, config=self.config) + service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) + await service.start() + self.assertEqual(1, service.dispatcher.dispatch.call_count) + await service.stop() if __name__ == "__main__": diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index ec5857cd..c7f9fd63 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -53,20 +53,18 @@ def test_dispatcher_config_context(self): self.assertIsInstance(service.dispatcher, MinosSnapshotDispatcher) async def test_start(self): - with self.config: - service = MinosSnapshotService(interval=1, loop=None) - service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) - await service.start() - self.assertTrue(1, service.dispatcher.setup.call_count) - await service.stop() + service = MinosSnapshotService(interval=1, loop=None, config=self.config) + service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) + await service.start() + self.assertTrue(1, service.dispatcher.setup.call_count) + await service.stop() async def test_callback(self): - with self.config: - service = MinosSnapshotService(interval=1, loop=None) - service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) - await service.start() - self.assertEqual(1, service.dispatcher.dispatch.call_count) - await service.stop() + service = MinosSnapshotService(interval=1, loop=None, config=self.config) + service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) + await service.start() + self.assertEqual(1, service.dispatcher.dispatch.call_count) + await service.stop() if __name__ == "__main__": From 3cec259505621ac6757a29cbdb74e9ccef2a4418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 15:12:13 +0200 Subject: [PATCH 183/239] ISSUE #94 * Revert unwanted changes. --- .idea/minos_microservice_networks.iml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.idea/minos_microservice_networks.iml b/.idea/minos_microservice_networks.iml index 1b9c4c49..765f3db6 100644 --- a/.idea/minos_microservice_networks.iml +++ b/.idea/minos_microservice_networks.iml @@ -3,9 +3,6 @@ - - - @@ -17,4 +14,4 @@ - \ No newline at end of file + From 95d1335d187f2e1876896ec2c1f067c91a545474 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 15:16:21 +0200 Subject: [PATCH 184/239] ISSUE #94 * Transform code. --- minos/networks/snapshots/dispatchers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 7326068e..d0781458 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -76,7 +76,9 @@ async def select(self, *args, **kwargs) -> AsyncIterator[MinosSnapshotEntry]: :param kwargs: Additional named arguments. :return: A sequence of ``MinosSnapshotEntry`` objects. """ - async for row in self.submit_query_and_iter(_SELECT_ALL_ENTRIES_QUERY): + # FIXME: Don't 'materialize' the full query, iterate over the generator. + entries = tuple(v async for v in self.submit_query_and_iter(_SELECT_ALL_ENTRIES_QUERY)) + for row in entries: yield MinosSnapshotEntry(*row) async def dispatch(self) -> NoReturn: From 508fa7310d41b01a8ccc2d8f20a9f333778b628f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 15:21:13 +0200 Subject: [PATCH 185/239] Revert "ISSUE #94 * Transform code." This reverts commit 95d1335d --- minos/networks/snapshots/dispatchers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index d0781458..7326068e 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -76,9 +76,7 @@ async def select(self, *args, **kwargs) -> AsyncIterator[MinosSnapshotEntry]: :param kwargs: Additional named arguments. :return: A sequence of ``MinosSnapshotEntry`` objects. """ - # FIXME: Don't 'materialize' the full query, iterate over the generator. - entries = tuple(v async for v in self.submit_query_and_iter(_SELECT_ALL_ENTRIES_QUERY)) - for row in entries: + async for row in self.submit_query_and_iter(_SELECT_ALL_ENTRIES_QUERY): yield MinosSnapshotEntry(*row) async def dispatch(self) -> NoReturn: From 1abd36cb7fbee56da342310b8d336aa522f0e725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 15:23:25 +0200 Subject: [PATCH 186/239] ISSUE #94 * Remove duplicated dependency. --- requirements_dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 05df0b5b..7317fcf6 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -41,7 +41,6 @@ pathspec==0.8.1 peewee==3.14.4 pkginfo==1.7.0 pluggy==0.13.1 -psycopg2==2.8.6 psycopg2-binary==2.8.6 py==1.10.0 pycodestyle==2.7.0 From 66f678d1fea80217af652d95383027edb60fdbce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 15:24:28 +0200 Subject: [PATCH 187/239] ISSUE #94 * Update aiomisc to latest version. --- requirements_dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 7317fcf6..b877f4a8 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,5 @@ aiokafka==0.7.0 -aiomisc==12.1.0 +aiomisc==14.0.3 aiopg==1.2.1 alabaster==0.7.12 appdirs==1.4.4 From 729ff4c840d515e96e9406dd28f9780412649e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 15:30:14 +0200 Subject: [PATCH 188/239] ISSUE #94 * Minor change. --- minos/networks/broker/services.py | 2 +- minos/networks/snapshots/services.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index 4d8f14cc..b6b767bd 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -49,5 +49,5 @@ async def stop(self, err: Exception = None) -> None: :param err: Optional exception that stopped the execution. :return: This method does not return anything. """ - await self.dispatcher.destroy() await super().stop(err) + await self.dispatcher.destroy() diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 2443a1bf..511193c6 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -46,5 +46,5 @@ async def stop(self, err: Exception = None) -> None: :param err: Optional exception that stopped the execution. :return: This method does not return anything. """ - await self.dispatcher.destroy() await super().stop(err) + await self.dispatcher.destroy() From 433f158207bb8ce4e5c697a492eaf82d0ee2e7ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 15:44:33 +0200 Subject: [PATCH 189/239] ISSUE #94 * Minor change (2). --- minos/networks/broker/dispatchers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index 91477ad6..2b358b87 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -98,12 +98,12 @@ async def publish(self, topic: str, message: bytes) -> bool: await producer.start() # Produce message await producer.send_and_wait(topic, message) - # Wait for all pending messages to be delivered or expire. - await producer.stop() - published = True except Exception: published = False + finally: + # Wait for all pending messages to be delivered or expire. + await producer.stop() return published From c1a88979edec347c3aac1b5f8b4c59a93cbd83eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 15:54:39 +0200 Subject: [PATCH 190/239] ISSUE #94 * Refactor tests for `minos.networks.events`. --- tests/aggregate_classes.py | 5 ++ tests/test_event_manager.py | 130 +++++++++--------------------------- 2 files changed, 36 insertions(+), 99 deletions(-) diff --git a/tests/aggregate_classes.py b/tests/aggregate_classes.py index 501537a6..39b8b2e5 100644 --- a/tests/aggregate_classes.py +++ b/tests/aggregate_classes.py @@ -29,3 +29,8 @@ class Car(Aggregate): doors: int color: str owner: Optional[list[ModelRef[Owner]]] + + +class AggregateTest(Aggregate): + """Aggregate for testing purposes""" + test: int diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 6892cb5f..9a4ac565 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -5,56 +5,36 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -import pytest -from aiokafka import ( - AIOKafkaProducer, -) +import unittest + +import aiopg + from minos.common import ( - Aggregate, -) -from minos.common.broker import ( Event, ) -from minos.common.configuration.config import ( - MinosConfig, +from minos.common.testing import ( + PostgresAsyncTestCase, ) from minos.networks.event import ( - EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService, MinosEventServer, + event_handler_table_creation, ) -from tests.database_testcase import ( - EventHandlerPostgresAsyncTestCase, +from tests.aggregate_classes import ( + AggregateTest, +) +from tests.utils import ( + BASE_PATH, ) -@pytest.fixture(scope="module") -def config(): - return MinosConfig(path="./tests/test_config.yml") - - -@pytest.fixture(scope="module") -def services(config): - return [ - EventHandlerDatabaseInitializer(config=config), - MinosEventServer(conf=config), - MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config), - ] - - -class AggregateTest(Aggregate): - test: int - - -class TestPostgreSqlMinosEventHandler(EventHandlerPostgresAsyncTestCase): - async def test_database_connection(self): - database = await self._database() - async with database as connect: - assert database.closed == 0 +class TestPostgreSqlMinosEventHandler(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" async def test_if_queue_table_exists(self): - database = await self._database() - async with database as connect: + await event_handler_table_creation(self.config) + + async with aiopg.connect(**self.events_queue_db) as connect: async with connect.cursor() as cur: await cur.execute( "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'event_queue';" @@ -66,21 +46,25 @@ async def test_if_queue_table_exists(self): assert ret == [(1,)] async def test_event_queue_add(self): + await event_handler_table_creation(self.config) + model = AggregateTest(test_id=1, test=2, id=1, version=1) event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) bin_data = event_instance.avro_bytes Event.from_avro_bytes(bin_data) - m = MinosEventServer(conf=self._broker_config()) + m = MinosEventServer(conf=self.config) affected_rows, id = await m.event_queue_add(topic=event_instance.topic, partition=0, binary=bin_data) assert affected_rows == 1 assert id > 0 async def test_get_event_handler(self): + await event_handler_table_creation(self.config) + model = AggregateTest(test_id=1, test=2, id=1, version=1) event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) - m = MinosEventHandlerPeriodicService(conf=self._broker_config(), interval=0.5) + m = MinosEventHandlerPeriodicService(conf=self.config, interval=0.5) cls = m.get_event_handler(topic="TicketAdded") result = await cls(topic="TicketAdded", event=event_instance) @@ -88,30 +72,30 @@ async def test_get_event_handler(self): assert result == "request_added" async def test_event_queue_checker(self): + await event_handler_table_creation(self.config) + model = AggregateTest(test_id=1, test=2, id=1, version=1) event_instance = Event(topic="TicketAdded", model=model.classname, items=[]) bin_data = event_instance.avro_bytes Event.from_avro_bytes(bin_data) - m = MinosEventServer(conf=self._broker_config()) + m = MinosEventServer(conf=self.config) affected_rows, id = await m.event_queue_add(topic=event_instance.topic, partition=0, binary=bin_data) assert affected_rows == 1 assert id > 0 - database = await self._database() - async with database as connect: + async with aiopg.connect(**self.events_queue_db) as connect: async with connect.cursor() as cur: await cur.execute("SELECT COUNT(*) FROM event_queue WHERE id=%d" % (id)) records = await cur.fetchone() assert records[0] == 1 - m = MinosEventHandlerPeriodicService(conf=self._broker_config(), interval=0.5) + m = MinosEventHandlerPeriodicService(conf=self.config, interval=0.5) await m.event_queue_checker() - database = await self._database() - async with database as connect: + async with aiopg.connect(**self.events_queue_db) as connect: async with connect.cursor() as cur: await cur.execute("SELECT COUNT(*) FROM event_queue WHERE id=%d" % (id)) records = await cur.fetchone() @@ -119,57 +103,5 @@ async def test_event_queue_checker(self): assert records[0] == 0 -async def test_producer_kafka(config, loop): - - # basic_config( - # level=logging.INFO, - # buffered=True, - # log_format='color', - # flush_interval=2 - # ) - kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" - producer = AIOKafkaProducer(loop=loop, bootstrap_servers=kafka_conn_data) - # Get cluster layout and topic/partition allocation - await producer.start() - # Produce messages - - model = AggregateTest(test_id=1, test=2, id=1, version=1) - event_instance = Event(topic="TicketAdded", model=model.classname, items=[model]) - bin_data = event_instance.avro_bytes - - model2 = AggregateTest(test_id=2, test=2, id=1, version=1) - event_instance_2 = Event(topic="TicketDeleted", model=model2.classname, items=[model2]) - bin_data2 = event_instance_2.avro_bytes - - for i in range(0, 10): - await producer.send_and_wait(event_instance.topic, bin_data) - await producer.send_and_wait(event_instance_2.topic, bin_data2) - - await producer.stop() - - -""" -async def test_consumer_kafka(config,loop): - handler = {item.name: {'controller': item.controller, 'action': item.action} - for item in config.events.items} - topics = list(handler.keys()) - kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" - broker_group_name = f"event_{config.service.name}" - - m = MinosEventServer(conf=config) - consumer = AIOKafkaConsumer( - loop=loop, - group_id=broker_group_name, - auto_offset_reset="latest", - bootstrap_servers=kafka_conn_data, consumer_timeout_ms=500 - ) - - await consumer.start() - consumer.subscribe(topics) - - for i in range(0, 2): - msg = await consumer.getone() - await m.handle_single_message(msg) - - await consumer.stop() -""" +if __name__ == "__main__": + unittest.main() From ce6d53fa7011005ecd62544cc23d2f57262bafc7 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 3 May 2021 13:55:10 +0000 Subject: [PATCH 191/239] Restyled by black --- minos/networks/broker/abc.py | 12 +++-------- minos/networks/broker/commands.py | 8 ++------ minos/networks/broker/dispatchers.py | 12 +++-------- minos/networks/broker/events.py | 12 +++-------- minos/networks/broker/services.py | 20 ++++++------------- minos/networks/snapshots/dispatchers.py | 8 ++------ minos/networks/snapshots/services.py | 14 ++++--------- tests/aggregate_classes.py | 5 ++--- tests/test_event_manager.py | 16 ++++----------- .../test_networks/test_broker/test_command.py | 8 ++------ .../test_broker/test_dispatcher.py | 16 ++++----------- .../test_networks/test_broker/test_events.py | 8 ++------ .../test_networks/test_broker/test_service.py | 20 +++++-------------- .../test_snapshots/test_services.py | 20 +++++-------------- 14 files changed, 47 insertions(+), 132 deletions(-) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index b2c01d9e..ff832667 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,15 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ( - ABC, -) -from datetime import ( - datetime, -) -from typing import ( - NoReturn, -) +from abc import ABC +from datetime import datetime +from typing import NoReturn from minos.common import ( MinosBaseBroker, diff --git a/minos/networks/broker/commands.py b/minos/networks/broker/commands.py index 29fe5e86..77730e80 100644 --- a/minos/networks/broker/commands.py +++ b/minos/networks/broker/commands.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NoReturn, @@ -20,9 +18,7 @@ MinosConfig, ) -from .abc import ( - MinosBroker, -) +from .abc import MinosBroker class MinosCommandBroker(MinosBroker): diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index 2b358b87..fd7bde5e 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( AsyncIterator, @@ -16,17 +14,13 @@ Optional, ) -from aiokafka import ( - AIOKafkaProducer, -) +from aiokafka import AIOKafkaProducer from minos.common import ( MinosConfig, MinosConfigException, ) -from .abc import ( - MinosBrokerSetup, -) +from .abc import MinosBrokerSetup class MinosQueueDispatcher(MinosBrokerSetup): diff --git a/minos/networks/broker/events.py b/minos/networks/broker/events.py index b19b587a..f9034980 100644 --- a/minos/networks/broker/events.py +++ b/minos/networks/broker/events.py @@ -5,13 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations -from typing import ( - Optional, -) +from typing import Optional from minos.common import ( Aggregate, @@ -19,9 +15,7 @@ MinosConfig, ) -from .abc import ( - MinosBroker, -) +from .abc import MinosBroker class MinosEventBroker(MinosBroker): diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index b6b767bd..edfae788 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -5,20 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) - -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) - -from .dispatchers import ( - MinosQueueDispatcher, -) +from __future__ import annotations + +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig + +from .dispatchers import MinosQueueDispatcher class MinosQueueService(PeriodicService): diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 7326068e..6a41e452 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( Any, @@ -28,9 +26,7 @@ import_module, ) -from .entries import ( - MinosSnapshotEntry, -) +from .entries import MinosSnapshotEntry class MinosSnapshotDispatcher(PostgreSqlMinosDatabase): diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 511193c6..b40cba06 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -6,16 +6,10 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfig, -) - -from .dispatchers import ( - MinosSnapshotDispatcher, -) +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfig + +from .dispatchers import MinosSnapshotDispatcher class MinosSnapshotService(PeriodicService): diff --git a/tests/aggregate_classes.py b/tests/aggregate_classes.py index 39b8b2e5..a01b46e3 100644 --- a/tests/aggregate_classes.py +++ b/tests/aggregate_classes.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - Optional, -) +from typing import Optional from minos.common import ( Aggregate, @@ -33,4 +31,5 @@ class Car(Aggregate): class AggregateTest(Aggregate): """Aggregate for testing purposes""" + test: int diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 9a4ac565..6dde52d1 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -9,23 +9,15 @@ import aiopg -from minos.common import ( - Event, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import Event +from minos.common.testing import PostgresAsyncTestCase from minos.networks.event import ( MinosEventHandlerPeriodicService, MinosEventServer, event_handler_table_creation, ) -from tests.aggregate_classes import ( - AggregateTest, -) -from tests.utils import ( - BASE_PATH, -) +from tests.aggregate_classes import AggregateTest +from tests.utils import BASE_PATH class TestPostgreSqlMinosEventHandler(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 15780838..8bde9b72 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -8,12 +8,8 @@ import unittest import aiopg -from minos.common import ( - MinosConfig, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import MinosConfig +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py index be51f22d..57ef8749 100644 --- a/tests/test_networks/test_broker/test_dispatcher.py +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -1,17 +1,9 @@ import unittest -from minos.common import ( - MinosConfigException, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) -from minos.networks import ( - MinosQueueDispatcher, -) -from tests.utils import ( - BASE_PATH, -) +from minos.common import MinosConfigException +from minos.common.testing import PostgresAsyncTestCase +from minos.networks import MinosQueueDispatcher +from tests.utils import BASE_PATH class TestQueueDispatcher(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index 5cbcd721..14f6443d 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -1,12 +1,8 @@ import unittest import aiopg -from minos.common import ( - MinosConfig, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import MinosConfig +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosEventBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index 3416b989..cd8d99c1 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -6,26 +6,16 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ import unittest -from unittest.mock import ( - MagicMock, -) +from unittest.mock import MagicMock -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfigException, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfigException +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosQueueDispatcher, MinosQueueService, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class TestMinosQueueService(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index c7f9fd63..cea71549 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -7,26 +7,16 @@ """ import unittest -from unittest.mock import ( - MagicMock, -) +from unittest.mock import MagicMock -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common import ( - MinosConfigException, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from aiomisc.service.periodic import PeriodicService +from minos.common import MinosConfigException +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosSnapshotDispatcher, MinosSnapshotService, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class TestMinosSnapshotService(PostgresAsyncTestCase): From a33cf428d57310848e86e5c0ced074d569e88792 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 3 May 2021 13:55:11 +0000 Subject: [PATCH 192/239] Restyled by isort --- minos/networks/broker/abc.py | 12 ++++++++--- minos/networks/broker/commands.py | 8 ++++++-- minos/networks/broker/dispatchers.py | 12 ++++++++--- minos/networks/broker/events.py | 12 ++++++++--- minos/networks/broker/services.py | 20 +++++++++++++------ minos/networks/snapshots/dispatchers.py | 8 ++++++-- minos/networks/snapshots/services.py | 14 +++++++++---- tests/aggregate_classes.py | 4 +++- tests/test_event_manager.py | 17 +++++++++++----- .../test_networks/test_broker/test_command.py | 8 ++++++-- .../test_broker/test_dispatcher.py | 16 +++++++++++---- .../test_networks/test_broker/test_events.py | 8 ++++++-- .../test_networks/test_broker/test_service.py | 20 ++++++++++++++----- .../test_snapshots/test_services.py | 20 ++++++++++++++----- 14 files changed, 132 insertions(+), 47 deletions(-) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py index ff832667..b2c01d9e 100644 --- a/minos/networks/broker/abc.py +++ b/minos/networks/broker/abc.py @@ -5,9 +5,15 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ABC -from datetime import datetime -from typing import NoReturn +from abc import ( + ABC, +) +from datetime import ( + datetime, +) +from typing import ( + NoReturn, +) from minos.common import ( MinosBaseBroker, diff --git a/minos/networks/broker/commands.py b/minos/networks/broker/commands.py index 77730e80..29fe5e86 100644 --- a/minos/networks/broker/commands.py +++ b/minos/networks/broker/commands.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( NoReturn, @@ -18,7 +20,9 @@ MinosConfig, ) -from .abc import MinosBroker +from .abc import ( + MinosBroker, +) class MinosCommandBroker(MinosBroker): diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py index fd7bde5e..2b358b87 100644 --- a/minos/networks/broker/dispatchers.py +++ b/minos/networks/broker/dispatchers.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( AsyncIterator, @@ -14,13 +16,17 @@ Optional, ) -from aiokafka import AIOKafkaProducer +from aiokafka import ( + AIOKafkaProducer, +) from minos.common import ( MinosConfig, MinosConfigException, ) -from .abc import MinosBrokerSetup +from .abc import ( + MinosBrokerSetup, +) class MinosQueueDispatcher(MinosBrokerSetup): diff --git a/minos/networks/broker/events.py b/minos/networks/broker/events.py index f9034980..b19b587a 100644 --- a/minos/networks/broker/events.py +++ b/minos/networks/broker/events.py @@ -5,9 +5,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) -from typing import Optional +from typing import ( + Optional, +) from minos.common import ( Aggregate, @@ -15,7 +19,9 @@ MinosConfig, ) -from .abc import MinosBroker +from .abc import ( + MinosBroker, +) class MinosEventBroker(MinosBroker): diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index edfae788..b6b767bd 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -5,12 +5,20 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations - -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig - -from .dispatchers import MinosQueueDispatcher +from __future__ import ( + annotations, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosQueueDispatcher, +) class MinosQueueService(PeriodicService): diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 6a41e452..7326068e 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( Any, @@ -26,7 +28,9 @@ import_module, ) -from .entries import MinosSnapshotEntry +from .entries import ( + MinosSnapshotEntry, +) class MinosSnapshotDispatcher(PostgreSqlMinosDatabase): diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index b40cba06..511193c6 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -6,10 +6,16 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfig - -from .dispatchers import MinosSnapshotDispatcher +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosSnapshotDispatcher, +) class MinosSnapshotService(PeriodicService): diff --git a/tests/aggregate_classes.py b/tests/aggregate_classes.py index a01b46e3..22b73185 100644 --- a/tests/aggregate_classes.py +++ b/tests/aggregate_classes.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import Optional +from typing import ( + Optional, +) from minos.common import ( Aggregate, diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 6dde52d1..9d3c97c9 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -8,16 +8,23 @@ import unittest import aiopg - -from minos.common import Event -from minos.common.testing import PostgresAsyncTestCase +from minos.common import ( + Event, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks.event import ( MinosEventHandlerPeriodicService, MinosEventServer, event_handler_table_creation, ) -from tests.aggregate_classes import AggregateTest -from tests.utils import BASE_PATH +from tests.aggregate_classes import ( + AggregateTest, +) +from tests.utils import ( + BASE_PATH, +) class TestPostgreSqlMinosEventHandler(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 8bde9b72..15780838 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -8,8 +8,12 @@ import unittest import aiopg -from minos.common import MinosConfig -from minos.common.testing import PostgresAsyncTestCase +from minos.common import ( + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py index 57ef8749..be51f22d 100644 --- a/tests/test_networks/test_broker/test_dispatcher.py +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -1,9 +1,17 @@ import unittest -from minos.common import MinosConfigException -from minos.common.testing import PostgresAsyncTestCase -from minos.networks import MinosQueueDispatcher -from tests.utils import BASE_PATH +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosQueueDispatcher, +) +from tests.utils import ( + BASE_PATH, +) class TestQueueDispatcher(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py index 14f6443d..5cbcd721 100644 --- a/tests/test_networks/test_broker/test_events.py +++ b/tests/test_networks/test_broker/test_events.py @@ -1,8 +1,12 @@ import unittest import aiopg -from minos.common import MinosConfig -from minos.common.testing import PostgresAsyncTestCase +from minos.common import ( + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosEventBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index cd8d99c1..3416b989 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -6,16 +6,26 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ import unittest -from unittest.mock import MagicMock +from unittest.mock import ( + MagicMock, +) -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfigException -from minos.common.testing import PostgresAsyncTestCase +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosQueueDispatcher, MinosQueueService, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class TestMinosQueueService(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index cea71549..c7f9fd63 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -7,16 +7,26 @@ """ import unittest -from unittest.mock import MagicMock +from unittest.mock import ( + MagicMock, +) -from aiomisc.service.periodic import PeriodicService -from minos.common import MinosConfigException -from minos.common.testing import PostgresAsyncTestCase +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosSnapshotDispatcher, MinosSnapshotService, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class TestMinosSnapshotService(PostgresAsyncTestCase): From d1a2e6846742a7893768f70ee54c5cd6826dc10b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 15:56:15 +0200 Subject: [PATCH 193/239] ISSUE #94 * Remove deprecated code. --- tests/database_testcase.py | 73 -------------------------------------- 1 file changed, 73 deletions(-) delete mode 100644 tests/database_testcase.py diff --git a/tests/database_testcase.py b/tests/database_testcase.py deleted file mode 100644 index a6d2148e..00000000 --- a/tests/database_testcase.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -Copyright (C) 2021 Clariteia SL -This file is part of minos framework. -Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. -""" -import os -import unittest - -import aiopg -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.networks.event import ( - event_handler_table_creation, -) - - -class EventHandlerPostgresAsyncTestCase(unittest.IsolatedAsyncioTestCase): - def setUp(self) -> None: - self._meta_kwargs = { - "host": os.getenv("POSTGRES_HOST", "localhost"), - "port": os.getenv("POSTGRES_PORT", 5432), - "database": os.getenv("POSTGRES_DATABASE", "postgres"), - "user": os.getenv("POSTGRES_USER", "postgres"), - "password": os.getenv("POSTGRES_PASSWORD", ""), - } - - self.kwargs = self._meta_kwargs | { - "database": "broker_db", - "user": "broker", - "password": "br0k3r", - } - - async def asyncSetUp(self): - await event_handler_table_creation(self._broker_config()) - """ - async with aiopg.connect(**self._meta_kwargs) as connection: - async with connection.cursor() as cursor: - - template = "DROP DATABASE IF EXISTS {database};" - await cursor.execute(template.format(**self.kwargs)) - - template = "DROP ROLE IF EXISTS {user};" - await cursor.execute(template.format(**self.kwargs)) - - template = "CREATE ROLE {user} WITH CREATEDB LOGIN ENCRYPTED PASSWORD {password!r};" - await cursor.execute(template.format(**self.kwargs)) - - template = "CREATE DATABASE {database} WITH OWNER = {user};" - await cursor.execute(template.format(**self.kwargs)) - """ - - @staticmethod - def _broker_config(): - return MinosConfig(path="./tests/test_config.yml") - - async def _database(self): - conf = self._broker_config() - db_dsn = ( - f"dbname={conf.events.queue.database} user={conf.events.queue.user} " - f"password={conf.events.queue.password} host={conf.events.queue.host}" - ) - return await aiopg.connect(db_dsn) - - async def asyncTearDown(self): - pass - """ - database = await self._database() - async with database as connection: - async with connection.cursor() as cursor: - template = "DROP TABLE IF EXISTS event_queue" - await cursor.execute(template) - """ From 94e3affece4feee2c1c8481fb7a1521ee1a51a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 16:01:07 +0200 Subject: [PATCH 194/239] ISSUE #94 * Change test interval. --- tests/test_networks/test_broker/test_service.py | 12 ++++++------ tests/test_networks/test_snapshots/test_services.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index 3416b989..b3014762 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -33,33 +33,33 @@ class TestMinosQueueService(PostgresAsyncTestCase): def test_is_instance(self): with self.config: - service = MinosQueueService(interval=10) + service = MinosQueueService(interval=0.1) self.assertIsInstance(service, PeriodicService) def test_dispatcher_empty(self): with self.assertRaises(MinosConfigException): - MinosQueueService(interval=10) + MinosQueueService(interval=0.1) def test_dispatcher_config(self): - service = MinosQueueService(interval=10, config=self.config) + service = MinosQueueService(interval=0.1, config=self.config) dispatcher = service.dispatcher self.assertIsInstance(dispatcher, MinosQueueDispatcher) self.assertFalse(dispatcher.already_setup) def test_dispatcher_config_context(self): with self.config: - service = MinosQueueService(interval=10) + service = MinosQueueService(interval=0.1) self.assertIsInstance(service.dispatcher, MinosQueueDispatcher) async def test_start(self): - service = MinosQueueService(interval=10, loop=None, config=self.config) + service = MinosQueueService(interval=0.1, loop=None, config=self.config) service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) await service.start() self.assertTrue(1, service.dispatcher.setup.call_count) await service.stop() async def test_callback(self): - service = MinosQueueService(interval=10, loop=None, config=self.config) + service = MinosQueueService(interval=0.1, loop=None, config=self.config) service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) await service.start() self.assertEqual(1, service.dispatcher.dispatch.call_count) diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index c7f9fd63..31f969e3 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -34,33 +34,33 @@ class TestMinosSnapshotService(PostgresAsyncTestCase): def test_is_instance(self): with self.config: - service = MinosSnapshotService(interval=10) + service = MinosSnapshotService(interval=0.1) self.assertIsInstance(service, PeriodicService) def test_dispatcher_config_raises(self): with self.assertRaises(MinosConfigException): - MinosSnapshotService(interval=10) + MinosSnapshotService(interval=0.1) def test_dispatcher_config(self): - service = MinosSnapshotService(interval=10, config=self.config) + service = MinosSnapshotService(interval=0.1, config=self.config) dispatcher = service.dispatcher self.assertIsInstance(dispatcher, MinosSnapshotDispatcher) self.assertFalse(dispatcher.already_setup) def test_dispatcher_config_context(self): with self.config: - service = MinosSnapshotService(interval=10) + service = MinosSnapshotService(interval=0.1) self.assertIsInstance(service.dispatcher, MinosSnapshotDispatcher) async def test_start(self): - service = MinosSnapshotService(interval=1, loop=None, config=self.config) + service = MinosSnapshotService(interval=0.1, loop=None, config=self.config) service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) await service.start() self.assertTrue(1, service.dispatcher.setup.call_count) await service.stop() async def test_callback(self): - service = MinosSnapshotService(interval=1, loop=None, config=self.config) + service = MinosSnapshotService(interval=0.1, loop=None, config=self.config) service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) await service.start() self.assertEqual(1, service.dispatcher.dispatch.call_count) From f14dd2024c44834d3b4130822f97533ed131b837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 16:06:44 +0200 Subject: [PATCH 195/239] ISSUE #94 * Don't start service. --- minos/networks/broker/services.py | 2 +- minos/networks/snapshots/services.py | 2 +- tests/test_networks/test_broker/test_service.py | 5 +++-- tests/test_networks/test_snapshots/test_services.py | 5 +++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py index b6b767bd..39e808c9 100644 --- a/minos/networks/broker/services.py +++ b/minos/networks/broker/services.py @@ -33,8 +33,8 @@ async def start(self) -> None: :return: This method does not return anything. """ - await super().start() await self.dispatcher.setup() + await super().start() async def callback(self) -> None: """Method to be called periodically by the internal ``aiomisc`` logic. diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py index 511193c6..4113f9e2 100644 --- a/minos/networks/snapshots/services.py +++ b/minos/networks/snapshots/services.py @@ -30,8 +30,8 @@ async def start(self) -> None: :return: This method does not return anything. """ - await super().start() await self.dispatcher.setup() + await super().start() async def callback(self) -> None: """Callback implementation to be executed periodically. diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py index b3014762..86d47af2 100644 --- a/tests/test_networks/test_broker/test_service.py +++ b/tests/test_networks/test_broker/test_service.py @@ -60,10 +60,11 @@ async def test_start(self): async def test_callback(self): service = MinosQueueService(interval=0.1, loop=None, config=self.config) + await service.dispatcher.setup() service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) - await service.start() + await service.callback() self.assertEqual(1, service.dispatcher.dispatch.call_count) - await service.stop() + await service.dispatcher.destroy() if __name__ == "__main__": diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py index 31f969e3..2e599778 100644 --- a/tests/test_networks/test_snapshots/test_services.py +++ b/tests/test_networks/test_snapshots/test_services.py @@ -61,10 +61,11 @@ async def test_start(self): async def test_callback(self): service = MinosSnapshotService(interval=0.1, loop=None, config=self.config) + await service.dispatcher.setup() service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) - await service.start() + await service.callback() self.assertEqual(1, service.dispatcher.dispatch.call_count) - await service.stop() + await service.dispatcher.destroy() if __name__ == "__main__": From f3b83bc4ce267f0236b3e6d0ef23dabab795b331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 16:13:40 +0200 Subject: [PATCH 196/239] ISSUE #94 * Revert unneeded changes. --- tests/aggregate_classes.py | 6 -- tests/database_testcase.py | 73 ++++++++++++++++++ tests/test_event_manager.py | 149 +++++++++++++++++++++++++++--------- 3 files changed, 185 insertions(+), 43 deletions(-) create mode 100644 tests/database_testcase.py diff --git a/tests/aggregate_classes.py b/tests/aggregate_classes.py index 22b73185..501537a6 100644 --- a/tests/aggregate_classes.py +++ b/tests/aggregate_classes.py @@ -29,9 +29,3 @@ class Car(Aggregate): doors: int color: str owner: Optional[list[ModelRef[Owner]]] - - -class AggregateTest(Aggregate): - """Aggregate for testing purposes""" - - test: int diff --git a/tests/database_testcase.py b/tests/database_testcase.py new file mode 100644 index 00000000..a6d2148e --- /dev/null +++ b/tests/database_testcase.py @@ -0,0 +1,73 @@ +""" +Copyright (C) 2021 Clariteia SL +This file is part of minos framework. +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +import os +import unittest + +import aiopg +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.networks.event import ( + event_handler_table_creation, +) + + +class EventHandlerPostgresAsyncTestCase(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self._meta_kwargs = { + "host": os.getenv("POSTGRES_HOST", "localhost"), + "port": os.getenv("POSTGRES_PORT", 5432), + "database": os.getenv("POSTGRES_DATABASE", "postgres"), + "user": os.getenv("POSTGRES_USER", "postgres"), + "password": os.getenv("POSTGRES_PASSWORD", ""), + } + + self.kwargs = self._meta_kwargs | { + "database": "broker_db", + "user": "broker", + "password": "br0k3r", + } + + async def asyncSetUp(self): + await event_handler_table_creation(self._broker_config()) + """ + async with aiopg.connect(**self._meta_kwargs) as connection: + async with connection.cursor() as cursor: + + template = "DROP DATABASE IF EXISTS {database};" + await cursor.execute(template.format(**self.kwargs)) + + template = "DROP ROLE IF EXISTS {user};" + await cursor.execute(template.format(**self.kwargs)) + + template = "CREATE ROLE {user} WITH CREATEDB LOGIN ENCRYPTED PASSWORD {password!r};" + await cursor.execute(template.format(**self.kwargs)) + + template = "CREATE DATABASE {database} WITH OWNER = {user};" + await cursor.execute(template.format(**self.kwargs)) + """ + + @staticmethod + def _broker_config(): + return MinosConfig(path="./tests/test_config.yml") + + async def _database(self): + conf = self._broker_config() + db_dsn = ( + f"dbname={conf.events.queue.database} user={conf.events.queue.user} " + f"password={conf.events.queue.password} host={conf.events.queue.host}" + ) + return await aiopg.connect(db_dsn) + + async def asyncTearDown(self): + pass + """ + database = await self._database() + async with database as connection: + async with connection.cursor() as cursor: + template = "DROP TABLE IF EXISTS event_queue" + await cursor.execute(template) + """ diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py index 9d3c97c9..f0bc87c4 100644 --- a/tests/test_event_manager.py +++ b/tests/test_event_manager.py @@ -1,40 +1,67 @@ -""" -Copyright (C) 2021 Clariteia SL - -This file is part of minos framework. - -Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. -""" -import unittest - -import aiopg +import asyncio +import logging +import random +import string + +import pytest +from aiokafka import ( + AIOKafkaConsumer, + AIOKafkaProducer, +) +from aiomisc.log import ( + basic_config, +) from minos.common import ( + Aggregate, +) +from minos.common.broker import ( Event, ) -from minos.common.testing import ( - PostgresAsyncTestCase, +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.logs import ( + log, ) from minos.networks.event import ( + EventHandlerDatabaseInitializer, MinosEventHandlerPeriodicService, MinosEventServer, - event_handler_table_creation, -) -from tests.aggregate_classes import ( - AggregateTest, ) -from tests.utils import ( - BASE_PATH, +from tests.database_testcase import ( + EventHandlerPostgresAsyncTestCase, ) -class TestPostgreSqlMinosEventHandler(PostgresAsyncTestCase): - CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" +@pytest.fixture() +def config(): + return MinosConfig(path="./tests/test_config.yml") + + +@pytest.fixture() +def services(config): + return [ + EventHandlerDatabaseInitializer(config=config), + MinosEventServer(conf=config), + MinosEventHandlerPeriodicService(interval=0.5, delay=0, conf=config), + ] + + +class AggregateTest(Aggregate): + test: int - async def test_if_queue_table_exists(self): - await event_handler_table_creation(self.config) - async with aiopg.connect(**self.events_queue_db) as connect: +class TestPostgreSqlMinosEventHandler(EventHandlerPostgresAsyncTestCase): + async def test_database_connection(self): + database = await self._database() + async with database as connect: + assert database.closed == 0 + + async def test_if_queue_table_exists(self): + database = await self._database() + async with database as connect: async with connect.cursor() as cur: + await cur.execute( "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'event_queue';" ) @@ -45,25 +72,21 @@ async def test_if_queue_table_exists(self): assert ret == [(1,)] async def test_event_queue_add(self): - await event_handler_table_creation(self.config) - model = AggregateTest(test_id=1, test=2, id=1, version=1) event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) bin_data = event_instance.avro_bytes Event.from_avro_bytes(bin_data) - m = MinosEventServer(conf=self.config) + m = MinosEventServer(conf=self._broker_config()) affected_rows, id = await m.event_queue_add(topic=event_instance.topic, partition=0, binary=bin_data) assert affected_rows == 1 assert id > 0 async def test_get_event_handler(self): - await event_handler_table_creation(self.config) - model = AggregateTest(test_id=1, test=2, id=1, version=1) event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) - m = MinosEventHandlerPeriodicService(conf=self.config, interval=0.5) + m = MinosEventHandlerPeriodicService(conf=self._broker_config(), interval=0.5) cls = m.get_event_handler(topic="TicketAdded") result = await cls(topic="TicketAdded", event=event_instance) @@ -71,30 +94,30 @@ async def test_get_event_handler(self): assert result == "request_added" async def test_event_queue_checker(self): - await event_handler_table_creation(self.config) - model = AggregateTest(test_id=1, test=2, id=1, version=1) event_instance = Event(topic="TicketAdded", model=model.classname, items=[]) bin_data = event_instance.avro_bytes Event.from_avro_bytes(bin_data) - m = MinosEventServer(conf=self.config) + m = MinosEventServer(conf=self._broker_config()) affected_rows, id = await m.event_queue_add(topic=event_instance.topic, partition=0, binary=bin_data) assert affected_rows == 1 assert id > 0 - async with aiopg.connect(**self.events_queue_db) as connect: + database = await self._database() + async with database as connect: async with connect.cursor() as cur: await cur.execute("SELECT COUNT(*) FROM event_queue WHERE id=%d" % (id)) records = await cur.fetchone() assert records[0] == 1 - m = MinosEventHandlerPeriodicService(conf=self.config, interval=0.5) + m = MinosEventHandlerPeriodicService(conf=self._broker_config(), interval=0.5) await m.event_queue_checker() - async with aiopg.connect(**self.events_queue_db) as connect: + database = await self._database() + async with database as connect: async with connect.cursor() as cur: await cur.execute("SELECT COUNT(*) FROM event_queue WHERE id=%d" % (id)) records = await cur.fetchone() @@ -102,5 +125,57 @@ async def test_event_queue_checker(self): assert records[0] == 0 -if __name__ == "__main__": - unittest.main() +async def test_producer_kafka(config, loop): + + # basic_config( + # level=logging.INFO, + # buffered=True, + # log_format='color', + # flush_interval=2 + # ) + kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" + producer = AIOKafkaProducer(loop=loop, bootstrap_servers=kafka_conn_data) + # Get cluster layout and topic/partition allocation + await producer.start() + # Produce messages + + model = AggregateTest(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TicketAdded", model=model.classname, items=[model]) + bin_data = event_instance.avro_bytes + + model2 = AggregateTest(test_id=2, test=2, id=1, version=1) + event_instance_2 = Event(topic="TicketDeleted", model=model2.classname, items=[model2]) + bin_data2 = event_instance_2.avro_bytes + + for i in range(0, 10): + await producer.send_and_wait(event_instance.topic, bin_data) + await producer.send_and_wait(event_instance_2.topic, bin_data2) + + await producer.stop() + + +""" +async def test_consumer_kafka(config,loop): + handler = {item.name: {'controller': item.controller, 'action': item.action} + for item in config.events.items} + topics = list(handler.keys()) + kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" + broker_group_name = f"event_{config.service.name}" + + m = MinosEventServer(conf=config) + consumer = AIOKafkaConsumer( + loop=loop, + group_id=broker_group_name, + auto_offset_reset="latest", + bootstrap_servers=kafka_conn_data, consumer_timeout_ms=500 + ) + + await consumer.start() + consumer.subscribe(topics) + + for i in range(0, 2): + msg = await consumer.getone() + await m.handle_single_message(msg) + + await consumer.stop() +""" From ec26dbeff64d4d741a0627d76147f8a97041bacc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 16:35:42 +0200 Subject: [PATCH 197/239] ISSUE #94 * Add tests for logic to ignore previous versions. --- .../test_snapshots/test_dispatchers.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 43d32df5..f73c9964 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -10,6 +10,9 @@ from datetime import ( datetime, ) +from unittest.mock import ( + patch, +) from minos.common import ( MinosConfigException, @@ -62,6 +65,30 @@ async def test_dispatch_select(self): MinosSnapshotEntry.from_aggregate(Car(2, 2, 3, "blue")), MinosSnapshotEntry.from_aggregate(Car(3, 1, 3, "blue")), ] + self._assert_equal_snapshot_entries(expected, observed) + + async def test_dispatch_ignore_previous_version(self): + with self.config: + dispatcher = MinosSnapshotDispatcher.from_config() + await dispatcher.setup() + + car = Car(1, 1, 3, "blue") + # noinspection PyTypeChecker + aggregate_name: str = car.classname + + async def _fn(*args, **kwargs): + yield MinosRepositoryEntry(1, aggregate_name, 1, car.avro_bytes) + yield MinosRepositoryEntry(1, aggregate_name, 3, car.avro_bytes) + yield MinosRepositoryEntry(1, aggregate_name, 2, car.avro_bytes) + + with patch("minos.common.PostgreSqlMinosRepository.select", _fn): + await dispatcher.dispatch() + observed = [v async for v in dispatcher.select()] + + expected = [MinosSnapshotEntry(1, aggregate_name, 3, car.avro_bytes)] + self._assert_equal_snapshot_entries(expected, observed) + + def _assert_equal_snapshot_entries(self, expected: list[MinosSnapshotEntry], observed: list[MinosSnapshotEntry]): self.assertEqual(len(expected), len(observed)) for exp, obs in zip(expected, observed): self.assertEqual(exp.aggregate, obs.aggregate) From 442ca107a0551b94d3233d69e104f4f2b9dd29be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 16:45:59 +0200 Subject: [PATCH 198/239] ISSUE #94 * Add missing environment variable. --- .github/workflows/python-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 83b1d563..9030e1fe 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -35,6 +35,7 @@ jobs: MINOS_COMMANDS_QUEUE_HOST: postgres MINOS_COMMANDS_BROKER: kafka MINOS_REPOSITORY_HOST: postgres + MINOS_SNAPSHOT_HOST: postgres steps: - name: Check out repository code From 37a37a6e0d068df3dd70e0da1382f0ee502a48b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 16:55:02 +0200 Subject: [PATCH 199/239] Revert "ISSUE #94 * Add missing environment variable." This reverts commit 442ca107 --- .github/workflows/python-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 9030e1fe..83b1d563 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -35,7 +35,6 @@ jobs: MINOS_COMMANDS_QUEUE_HOST: postgres MINOS_COMMANDS_BROKER: kafka MINOS_REPOSITORY_HOST: postgres - MINOS_SNAPSHOT_HOST: postgres steps: - name: Check out repository code From a62e805b26b4abc12f358f734b9817ed1f440940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 17:00:12 +0200 Subject: [PATCH 200/239] ISSUE #90 * Start using `snapshot` config. --- .github/workflows/python-tests.yml | 1 + minos/networks/snapshots/dispatchers.py | 2 +- tests/test_config.yml | 6 ++++++ .../test_snapshots/test_dispatchers.py | 17 ++++++++--------- tests/wrong_test_config.yml | 6 ++++++ 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 83b1d563..9030e1fe 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -35,6 +35,7 @@ jobs: MINOS_COMMANDS_QUEUE_HOST: postgres MINOS_COMMANDS_BROKER: kafka MINOS_REPOSITORY_HOST: postgres + MINOS_SNAPSHOT_HOST: postgres steps: - name: Check out repository code diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 7326068e..aecae79f 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -60,7 +60,7 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> MinosSnapsh if config is None: raise MinosConfigException("The config object must be setup.") # noinspection PyProtectedMember - return cls(*args, **config.repository._asdict(), repository=config.repository._asdict(), **kwargs) + return cls(*args, **config.snapshot._asdict(), repository=config.repository._asdict(), **kwargs) async def _setup(self) -> NoReturn: await self._create_broker_table() diff --git a/tests/test_config.yml b/tests/test_config.yml index 951db9dd..1e621a1e 100644 --- a/tests/test_config.yml +++ b/tests/test_config.yml @@ -15,6 +15,12 @@ repository: password: min0s host: localhost port: 5432 +snapshot: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 events: broker: localhost port: 9092 diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 43d32df5..53dcdfeb 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -39,11 +39,11 @@ def test_type(self): def test_from_config(self): dispatcher = MinosSnapshotDispatcher.from_config(config=self.config) - self.assertEqual(self.config.repository.host, dispatcher.host) - self.assertEqual(self.config.repository.port, dispatcher.port) - self.assertEqual(self.config.repository.database, dispatcher.database) - self.assertEqual(self.config.repository.user, dispatcher.user) - self.assertEqual(self.config.repository.password, dispatcher.password) + self.assertEqual(self.config.snapshot.host, dispatcher.host) + self.assertEqual(self.config.snapshot.port, dispatcher.port) + self.assertEqual(self.config.snapshot.database, dispatcher.database) + self.assertEqual(self.config.snapshot.user, dispatcher.user) + self.assertEqual(self.config.snapshot.password, dispatcher.password) self.assertEqual(0, dispatcher.offset) def test_from_config_raises(self): @@ -53,10 +53,9 @@ def test_from_config_raises(self): async def test_dispatch_select(self): await self._populate() with self.config: - dispatcher = MinosSnapshotDispatcher.from_config() - await dispatcher.setup() - await dispatcher.dispatch() - observed = [v async for v in dispatcher.select()] + async with MinosSnapshotDispatcher.from_config() as dispatcher: + await dispatcher.dispatch() + observed = [v async for v in dispatcher.select()] expected = [ MinosSnapshotEntry.from_aggregate(Car(2, 2, 3, "blue")), diff --git a/tests/wrong_test_config.yml b/tests/wrong_test_config.yml index 57eeb572..9792f28a 100644 --- a/tests/wrong_test_config.yml +++ b/tests/wrong_test_config.yml @@ -15,6 +15,12 @@ repository: password: min0s host: localhost port: 5432 +snapshot: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 events: broker: localhost port: 4092 From f6e5073a23f517f1436b9a65429334477a64daf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 18:02:20 +0200 Subject: [PATCH 201/239] ISSUE #89 * Create skeleton for offset storage. --- minos/networks/snapshots/dispatchers.py | 59 ++++++++++++++++++++----- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 7326068e..c95d0567 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -36,7 +36,7 @@ class MinosSnapshotDispatcher(PostgreSqlMinosDatabase): """Minos Snapshot Dispatcher class.""" - def __init__(self, *args, repository: dict[str, Any] = None, offset: int = 0, **kwargs): + def __init__(self, *args, repository: dict[str, Any] = None, offset: int = None, **kwargs): super().__init__(*args, **kwargs) self.offset = offset @@ -63,10 +63,33 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> MinosSnapsh return cls(*args, **config.repository._asdict(), repository=config.repository._asdict(), **kwargs) async def _setup(self) -> NoReturn: - await self._create_broker_table() - - async def _create_broker_table(self) -> NoReturn: await self.submit_query(_CREATE_TABLE_QUERY) + await self.submit_query(_CREATE_OFFSET_TABLE_QUERY) + + async def dispatch(self) -> NoReturn: + """Perform a dispatching step, based on the sequence of non already processed ``MinosRepositoryEntry`` objects. + + :return: This method does not return anything. + """ + offset = self.offset + if offset is None: + offset = await self._load_offset() + + async for entry in self.repository.select(id_ge=offset): + await self._dispatch_one(entry) + + await self._store_offset(offset) + + async def _load_offset(self) -> int: + # noinspection PyBroadException + try: + raw = await self.submit_query_and_fetchone(_SELECT_OFFSET_QUERY) + return raw[0] + except Exception: + return 0 + + async def _store_offset(self, offset: int) -> NoReturn: + await self.submit_query(_INSERT_OFFSET_QUERY, {"value": offset}) # noinspection PyUnusedLocal async def select(self, *args, **kwargs) -> AsyncIterator[MinosSnapshotEntry]: @@ -79,14 +102,6 @@ async def select(self, *args, **kwargs) -> AsyncIterator[MinosSnapshotEntry]: async for row in self.submit_query_and_iter(_SELECT_ALL_ENTRIES_QUERY): yield MinosSnapshotEntry(*row) - async def dispatch(self) -> NoReturn: - """Perform a dispatching step, based on the sequence of non already processed ``MinosRepositoryEntry`` objects. - - :return: This method does not return anything. - """ - async for entry in self.repository.select(id_ge=self.offset): - await self._dispatch_one(entry) - async def _dispatch_one(self, event_entry: MinosRepositoryEntry) -> Optional[MinosSnapshotEntry]: if event_entry.action is MinosRepositoryAction.DELETE: await self._submit_delete(event_entry) @@ -183,3 +198,23 @@ async def _submit_update_or_create(self, entry: MinosSnapshotEntry) -> MinosSnap UPDATE SET version = %(version)s, data = %(data)s, updated_at = NOW() RETURNING created_at, updated_at; """.strip() + +_CREATE_OFFSET_TABLE_QUERY = """ +CREATE TABLE IF NOT EXISTS snapshot_aux_offset ( + id bool PRIMARY KEY DEFAULT TRUE, + value BIGINT NOT NULL, + CONSTRAINT id_uni CHECK (id) +); +""".strip() + +_SELECT_OFFSET_QUERY = """ +SELECT value +FROM snapshot_aux_offset +WHERE id = TRUE; +""" +_INSERT_OFFSET_QUERY = """ +INSERT INTO snapshot_aux_offset (id, value) +VALUES (TRUE, %(value)s) +ON CONFLICT (id) +DO UPDATE SET value = %(value)s; +""".strip() From 0ffcb93683a4d249493f54c857ed8d22d85fdf8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 18:03:06 +0200 Subject: [PATCH 202/239] ISSUE #89 * Fix test. --- tests/test_networks/test_snapshots/test_dispatchers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 43d32df5..b11bb36c 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -44,7 +44,7 @@ def test_from_config(self): self.assertEqual(self.config.repository.database, dispatcher.database) self.assertEqual(self.config.repository.user, dispatcher.user) self.assertEqual(self.config.repository.password, dispatcher.password) - self.assertEqual(0, dispatcher.offset) + self.assertEqual(None, dispatcher.offset) def test_from_config_raises(self): with self.assertRaises(MinosConfigException): From ca3d6098273c8ff24ea796337dd6663ea300058d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Mon, 3 May 2021 18:07:48 +0200 Subject: [PATCH 203/239] ISSUE #89 * Remove `snapshot` attribute. --- minos/networks/snapshots/dispatchers.py | 9 ++------- tests/test_networks/test_snapshots/test_dispatchers.py | 1 - 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index c95d0567..7faf12b4 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -36,11 +36,8 @@ class MinosSnapshotDispatcher(PostgreSqlMinosDatabase): """Minos Snapshot Dispatcher class.""" - def __init__(self, *args, repository: dict[str, Any] = None, offset: int = None, **kwargs): + def __init__(self, *args, repository: dict[str, Any] = None, **kwargs): super().__init__(*args, **kwargs) - - self.offset = offset - self.repository = PostgreSqlMinosRepository(**repository) async def _destroy(self) -> NoReturn: @@ -71,9 +68,7 @@ async def dispatch(self) -> NoReturn: :return: This method does not return anything. """ - offset = self.offset - if offset is None: - offset = await self._load_offset() + offset = await self._load_offset() async for entry in self.repository.select(id_ge=offset): await self._dispatch_one(entry) diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index b11bb36c..35f22766 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -44,7 +44,6 @@ def test_from_config(self): self.assertEqual(self.config.repository.database, dispatcher.database) self.assertEqual(self.config.repository.user, dispatcher.user) self.assertEqual(self.config.repository.password, dispatcher.password) - self.assertEqual(None, dispatcher.offset) def test_from_config_raises(self): with self.assertRaises(MinosConfigException): From 268e32bcdddddf5646d76fe40c33f663d2e1730c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Tue, 4 May 2021 08:23:49 +0200 Subject: [PATCH 204/239] ISSUE #89 * Add tests for table creation. --- .../test_snapshots/test_dispatchers.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 35f22766..738122f3 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -11,6 +11,8 @@ datetime, ) +import aiopg + from minos.common import ( MinosConfigException, MinosRepositoryEntry, @@ -49,6 +51,28 @@ def test_from_config_raises(self): with self.assertRaises(MinosConfigException): MinosSnapshotDispatcher.from_config() + async def test_setup_snapshot_table(self): + async with MinosSnapshotDispatcher.from_config(config=self.config): + async with aiopg.connect(**self.repository_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute( + "SELECT EXISTS (SELECT FROM pg_tables " + "WHERE schemaname = 'public' AND tablename = 'snapshot');" + ) + observed = (await cursor.fetchone())[0] + self.assertEqual(True, observed) + + async def test_setup_snapshot_aux_offset_table(self): + async with MinosSnapshotDispatcher.from_config(config=self.config): + async with aiopg.connect(**self.repository_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute( + "SELECT EXISTS (SELECT FROM pg_tables WHERE " + "schemaname = 'public' AND tablename = 'snapshot_aux_offset');" + ) + observed = (await cursor.fetchone())[0] + self.assertEqual(True, observed) + async def test_dispatch_select(self): await self._populate() with self.config: From 8a2818153dc75d1304d1e171dd3a1e15017b1e39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Tue, 4 May 2021 08:24:44 +0200 Subject: [PATCH 205/239] ISSUE #89 * Minor change. --- tests/test_networks/test_snapshots/test_dispatchers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 738122f3..f4b3f711 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -12,7 +12,6 @@ ) import aiopg - from minos.common import ( MinosConfigException, MinosRepositoryEntry, From aa028f4f7373a224b346a34c745735db10ba1f3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Tue, 4 May 2021 08:51:52 +0200 Subject: [PATCH 206/239] ISSUE #89 * Add tests related with offset. --- minos/networks/snapshots/dispatchers.py | 5 +++ .../test_snapshots/test_dispatchers.py | 41 ++++++++++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 7faf12b4..9f4cf02e 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -70,8 +70,13 @@ async def dispatch(self) -> NoReturn: """ offset = await self._load_offset() + ids = set() async for entry in self.repository.select(id_ge=offset): await self._dispatch_one(entry) + ids.add(entry.id) + while offset + 1 in ids: + offset += 1 + ids.remove(offset) await self._store_offset(offset) diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index f4b3f711..9987b1d5 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -10,6 +10,7 @@ from datetime import ( datetime, ) +from unittest.mock import MagicMock, call import aiopg from minos.common import ( @@ -90,13 +91,40 @@ async def test_dispatch_select(self): self.assertIsInstance(obs.created_at, datetime) self.assertIsInstance(obs.updated_at, datetime) + async def test_dispatch_with_offset(self): + async with await self._populate() as repository: + async with MinosSnapshotDispatcher.from_config(config=self.config) as dispatcher: + mock = MagicMock(side_effect=dispatcher.repository.select) + dispatcher.repository.select = mock + + await dispatcher.dispatch() + self.assertEqual(1, mock.call_count) + self.assertEqual(call(id_ge=0), mock.call_args) + mock.reset_mock() + + # noinspection PyTypeChecker + await repository.insert(MinosRepositoryEntry(3, Car.classname, 1, Car(1, 1, 3, "blue").avro_bytes)) + + await dispatcher.dispatch() + self.assertEqual(1, mock.call_count) + self.assertEqual(call(id_ge=7), mock.call_args) + mock.reset_mock() + + await dispatcher.dispatch() + self.assertEqual(1, mock.call_count) + self.assertEqual(call(id_ge=8), mock.call_args) + mock.reset_mock() + + await dispatcher.dispatch() + self.assertEqual(1, mock.call_count) + self.assertEqual(call(id_ge=8), mock.call_args) + mock.reset_mock() + async def _populate(self): - with self.config: - car = Car(1, 1, 3, "blue") - # noinspection PyTypeChecker - aggregate_name: str = car.classname - repository = PostgreSqlMinosRepository.from_config() - await repository.setup() + car = Car(1, 1, 3, "blue") + # noinspection PyTypeChecker + aggregate_name: str = car.classname + async with PostgreSqlMinosRepository.from_config(config=self.config) as repository: await repository.insert(MinosRepositoryEntry(1, aggregate_name, 1, car.avro_bytes)) await repository.update(MinosRepositoryEntry(1, aggregate_name, 2, car.avro_bytes)) await repository.insert(MinosRepositoryEntry(2, aggregate_name, 1, car.avro_bytes)) @@ -104,6 +132,7 @@ async def _populate(self): await repository.delete(MinosRepositoryEntry(1, aggregate_name, 4)) await repository.update(MinosRepositoryEntry(2, aggregate_name, 2, car.avro_bytes)) await repository.insert(MinosRepositoryEntry(3, aggregate_name, 1, car.avro_bytes)) + return repository if __name__ == "__main__": From 86c7d5105eb13f04bacba6f0988d71ec6658248b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Tue, 4 May 2021 08:59:06 +0200 Subject: [PATCH 207/239] ISSUE #89 * Minor changes. --- minos/networks/snapshots/dispatchers.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py index 9f4cf02e..ac3547b4 100644 --- a/minos/networks/snapshots/dispatchers.py +++ b/minos/networks/snapshots/dispatchers.py @@ -71,12 +71,17 @@ async def dispatch(self) -> NoReturn: offset = await self._load_offset() ids = set() + + def _update_offset(e: MinosRepositoryEntry, o: int): + ids.add(e.id) + while o + 1 in ids: + o += 1 + ids.remove(o) + return o + async for entry in self.repository.select(id_ge=offset): await self._dispatch_one(entry) - ids.add(entry.id) - while offset + 1 in ids: - offset += 1 - ids.remove(offset) + offset = _update_offset(entry, offset) await self._store_offset(offset) From 8b5855c5af8f2adeb761d46d48254b0358cbe881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Tue, 4 May 2021 08:59:21 +0200 Subject: [PATCH 208/239] ISSUE #89 * Minor changes (2). --- tests/test_networks/test_snapshots/test_dispatchers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index 9987b1d5..48017e5a 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -10,7 +10,10 @@ from datetime import ( datetime, ) -from unittest.mock import MagicMock, call +from unittest.mock import ( + MagicMock, + call, +) import aiopg from minos.common import ( From 227ba7d4adcaec5bcc4c503f1947e7d9af1fe2fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Tue, 4 May 2021 10:43:49 +0200 Subject: [PATCH 209/239] ISSUE #100 * Remove `.idea` directory. --- .idea/.gitignore | 8 -------- .idea/inspectionProfiles/Project_Default.xml | 13 ------------- .idea/inspectionProfiles/profiles_settings.xml | 6 ------ .idea/misc.xml | 4 ---- .idea/modules.xml | 8 -------- .idea/vcs.xml | 6 ------ 6 files changed, 45 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/inspectionProfiles/Project_Default.xml delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 73f69e09..00000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 06f7730f..00000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2da..00000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 1cc5b395..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 70bc1b36..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 18d0b3726997dc3ce0bb2c1e1dfeb3d468d7e798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Tue, 4 May 2021 10:44:31 +0200 Subject: [PATCH 210/239] ISSUE #100 * Add `.idea` to `.gitignore`. --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 43091aa9..23ae5f8a 100644 --- a/.gitignore +++ b/.gitignore @@ -102,4 +102,7 @@ ENV/ .mypy_cache/ # IDE settings -.vscode/ \ No newline at end of file +.vscode/ + +# Intellij IDEa / PyCharm / etc. +.idea From 63536a3bd2b4fc6dd177e3cbc7004007e07092d3 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 4 May 2021 11:26:52 +0200 Subject: [PATCH 211/239] EventHandler Issue 87 eventhandler refactor and tests #106 --- minos/networks/event/event_server.py | 20 ++++-- .../test_event/test_event_server.py | 67 +++++-------------- 2 files changed, 28 insertions(+), 59 deletions(-) diff --git a/minos/networks/event/event_server.py b/minos/networks/event/event_server.py index 948a00a4..b69a2193 100644 --- a/minos/networks/event/event_server.py +++ b/minos/networks/event/event_server.py @@ -15,7 +15,7 @@ NoReturn, Optional, Awaitable, - Any, + Any, AsyncIterator, ) import aiopg @@ -122,14 +122,20 @@ async def handle_single_message(self, msg): """ # the handler receive a message and store in the queue database # check if the event binary string is well formatted + if not self._is_valid_event(msg.value): + return + affected_rows, id = await self.event_queue_add(msg.topic, msg.partition, msg.value) + return affected_rows, id + + + def _is_valid_event(self, value: bytes): try: - Event.from_avro_bytes(msg.value) - affected_rows, id = await self.event_queue_add(msg.topic, msg.partition, msg.value) - return affected_rows, id - finally: - pass + Event.from_avro_bytes(value) + return True + except: + return False - async def handle_message(self, consumer: Any): + async def handle_message(self, consumer: AsyncIterator): """Message consumer. It consumes the messages and sends them for processing. diff --git a/tests/test_networks/test_event/test_event_server.py b/tests/test_networks/test_event/test_event_server.py index 148e64af..5f4b03e0 100644 --- a/tests/test_networks/test_event/test_event_server.py +++ b/tests/test_networks/test_event/test_event_server.py @@ -1,3 +1,6 @@ +import time +from collections import namedtuple + from minos.common.testing import ( PostgresAsyncTestCase, ) @@ -39,10 +42,10 @@ async def kafka_producer(config): event_instance_2 = Event(topic="TicketDeleted", model=model2.classname, items=[model2]) bin_data2 = event_instance_2.avro_bytes - for i in range(0, 10): - await producer.send_and_wait(event_instance.topic, bin_data) - await producer.send_and_wait(event_instance_2.topic, bin_data2) - + await producer.send_and_wait(event_instance.topic, bin_data) + time.sleep(1) + await producer.send_and_wait(event_instance_2.topic, bin_data2) + time.sleep(1) await producer.stop() @@ -72,58 +75,18 @@ async def test_event_queue_add(self): assert affected_rows == 1 assert id > 0 - """ - async def test_consumer_kafka(self): - await kafka_producer(self.config) - handler = {item.name: {'controller': item.controller, 'action': item.action} - for item in self.config.events.items} - topics = list(handler.keys()) - kafka_conn_data = f"{self.config.events.broker.host}:{self.config.events.broker.port}" - broker_group_name = f"event_{self.config.service.name}" - - event_server = MinosEventServer.from_config(config=self.config) - await event_server.setup() - - consumer = AIOKafkaConsumer( - loop=None, - group_id=broker_group_name, - auto_offset_reset="latest", - bootstrap_servers=kafka_conn_data, consumer_timeout_ms=500 - ) - - await consumer.start() - consumer.subscribe(topics) - - msg = await consumer.getone() - affected_rows, id = await event_server.handle_single_message(msg) - assert affected_rows > 0 - assert id > 0 - - await consumer.stop() - - async def test_handle_message(self): - await kafka_producer(self.config) - handler = {item.name: {'controller': item.controller, 'action': item.action} - for item in self.config.events.items} - topics = list(handler.keys()) - kafka_conn_data = f"{self.config.events.broker.host}:{self.config.events.broker.port}" - broker_group_name = f"event_{self.config.service.name}" - event_server = MinosEventServer.from_config(config=self.config) await event_server.setup() - consumer = AIOKafkaConsumer( - loop=None, - group_id=broker_group_name, - auto_offset_reset="latest", - bootstrap_servers=kafka_conn_data, consumer_timeout_ms=500 - ) + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TicketAdded", model=model.classname, items=[model]) + bin_data = event_instance.avro_bytes + + Mensaje = namedtuple("Mensaje",["topic", "partition", "value"]) - await consumer.start() - consumer.subscribe(topics) + async def consumer(): + yield Mensaje(topic="TicketAdded",partition=0, value=bin_data) - await event_server.handle_message(consumer) + await event_server.handle_message(consumer()) - await consumer.stop() - """ From 8f07752d954e70e5f3ed01e050ac8c76dac523d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Tue, 4 May 2021 12:27:21 +0200 Subject: [PATCH 212/239] ISSUE #90 * Increase `minos.common` version up to `0.0.6`. * Use proper config. --- requirements_dev.txt | 2 +- tests/test_networks/test_snapshots/test_dispatchers.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index b877f4a8..34855353 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -32,7 +32,7 @@ keyring==23.0.1 lmdb==1.1.1 MarkupSafe==1.1.1 mccabe==0.6.1 -minos-microservice-common==0.0.5 +minos-microservice-common==0.0.6 more-itertools==8.7.0 mypy-extensions==0.4.3 orjson==3.5.1 diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py index e005c72e..5d254ce1 100644 --- a/tests/test_networks/test_snapshots/test_dispatchers.py +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -57,7 +57,7 @@ def test_from_config_raises(self): async def test_setup_snapshot_table(self): async with MinosSnapshotDispatcher.from_config(config=self.config): - async with aiopg.connect(**self.repository_db) as connection: + async with aiopg.connect(**self.snapshot_db) as connection: async with connection.cursor() as cursor: await cursor.execute( "SELECT EXISTS (SELECT FROM pg_tables " @@ -68,7 +68,7 @@ async def test_setup_snapshot_table(self): async def test_setup_snapshot_aux_offset_table(self): async with MinosSnapshotDispatcher.from_config(config=self.config): - async with aiopg.connect(**self.repository_db) as connection: + async with aiopg.connect(**self.snapshot_db) as connection: async with connection.cursor() as cursor: await cursor.execute( "SELECT EXISTS (SELECT FROM pg_tables WHERE " From e5e2b0cd02323f2801636155b4eaded816edf23e Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 4 May 2021 12:59:48 +0200 Subject: [PATCH 213/239] EventHandler Kafka tests EventHandler Refactor code and add more tests #87 --- minos/networks/event/services.py | 20 ++++++-- .../test_event/test_event_server.py | 12 +++++ .../test_event/test_event_services.py | 49 +++++++++++-------- 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/minos/networks/event/services.py b/minos/networks/event/services.py index d120d95e..02e6e72a 100644 --- a/minos/networks/event/services.py +++ b/minos/networks/event/services.py @@ -26,11 +26,14 @@ class MinosEventServerService(Service): def __init__(self, config: MinosConfig = None, **kwargs): super().__init__(**kwargs) self.dispatcher = MinosEventServer.from_config(config=config) + self.consumer = None + """ def create_task(self, coro: Awaitable[Any]): task = self.loop.create_task(coro) self.dispatcher._tasks.add(task) task.add_done_callback(self.dispatcher._tasks.remove) + """ async def start(self) -> None: """Method to be called at the startup by the internal ``aiomisc`` loigc. @@ -39,18 +42,25 @@ async def start(self) -> None: """ await self.dispatcher.setup() - self.start_event.set() # start the Service Event Consumer for Kafka - consumer = AIOKafkaConsumer( + self.consumer = AIOKafkaConsumer( group_id=self.dispatcher._broker_group_name, auto_offset_reset="latest", bootstrap_servers=self.dispatcher._kafka_conn_data, ) - await consumer.start() - consumer.subscribe(self.dispatcher._topics) + await self.consumer.start() + self.consumer.subscribe(self.dispatcher._topics) + + #self.create_task(self.dispatcher.handle_message(self.consumer)) + + await self.dispatcher.handle_message(self.consumer) + + async def stop(self, exception: Exception = None) -> Any: + if self.consumer is not None: + await self.consumer.stop() + - self.create_task(self.dispatcher.handle_message(consumer)) class MinosEventPeriodicService(PeriodicService): diff --git a/tests/test_networks/test_event/test_event_server.py b/tests/test_networks/test_event/test_event_server.py index 5f4b03e0..032348bb 100644 --- a/tests/test_networks/test_event/test_event_server.py +++ b/tests/test_networks/test_event/test_event_server.py @@ -90,3 +90,15 @@ async def consumer(): await event_server.handle_message(consumer()) + async def test_handle_message_ko(self): + event_server = MinosEventServer.from_config(config=self.config) + await event_server.setup() + + bin_data = bytes(b'test') + + Mensaje = namedtuple("Mensaje",["topic", "partition", "value"]) + + async def consumer(): + yield Mensaje(topic="TicketAdded",partition=0, value=bin_data) + + await event_server.handle_message(consumer()) diff --git a/tests/test_networks/test_event/test_event_services.py b/tests/test_networks/test_event/test_event_services.py index b4ed0537..800f784f 100644 --- a/tests/test_networks/test_event/test_event_services.py +++ b/tests/test_networks/test_event/test_event_services.py @@ -2,9 +2,6 @@ from unittest.mock import ( MagicMock, ) -from minos.common.configuration.config import ( - MinosConfig, -) from minos.networks import ( MinosEventServerService, MinosEventPeriodicService @@ -17,28 +14,38 @@ ) -""" -@pytest.fixture() -def services(config): - return [ - MinosEventServerService(config=config), - MinosEventPeriodicService(interval=0.5, delay=0, config=config), - ] -""" +class TestMinosEventServer(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_start(self): + service = MinosEventServerService(loop=None, config=self.config) + + async def _fn(consumer): + self.assertEqual(service.consumer, consumer) + + mock = MagicMock(side_effect=_fn) + service.dispatcher.handle_message = mock + await service.start() + self.assertTrue(1, mock.call_count) + await service.stop() + class TestMinosQueueService(PostgresAsyncTestCase): CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" async def test_start(self): - with self.config: - service = MinosEventPeriodicService(interval=1, loop=None) - service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) - await service.start() - self.assertTrue(1, service.dispatcher.setup.call_count) + service = MinosEventPeriodicService(interval=1, loop=None, config=self.config) + mock = MagicMock(side_effect=service.dispatcher.setup) + service.dispatcher.setup = mock + await service.start() + self.assertTrue(1, mock.call_count) + await service.stop() async def test_callback(self): - with self.config: - service = MinosEventPeriodicService(interval=1, loop=None) - service.dispatcher.event_queue_checker = MagicMock(side_effect=service.dispatcher.event_queue_checker) - await service.start() - self.assertEqual(1, service.dispatcher.event_queue_checker.call_count) + service = MinosEventPeriodicService(interval=1, loop=None, config=self.config) + await service.dispatcher.setup() + mock = MagicMock(side_effect=service.dispatcher.event_queue_checker) + service.dispatcher.event_queue_checker = mock + await service.callback() + self.assertEqual(1, mock.call_count) + await service.dispatcher.destroy() From 5ce10bacdd22fb642edccb6ad9f29ac107a9ceee Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 4 May 2021 11:00:11 +0000 Subject: [PATCH 214/239] Restyled by black --- minos/networks/__init__.py | 7 +--- minos/networks/event/__init__.py | 13 ++------ minos/networks/event/abc.py | 16 +++------- minos/networks/event/dispatcher.py | 32 +++++-------------- minos/networks/event/event_server.py | 29 +++++------------ minos/networks/event/services.py | 20 +++--------- .../test_event/test_dispatcher.py | 23 +++++-------- .../test_event/test_event_server.py | 22 +++++-------- .../test_event/test_event_services.py | 17 +++------- 9 files changed, 49 insertions(+), 130 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 61e47a5c..1f347010 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -23,9 +23,4 @@ MinosSnapshotEntry, MinosSnapshotService, ) -from .event import ( - MinosEventServer, - MinosEventHandler, - MinosEventServerService, - MinosEventPeriodicService -) +from .event import MinosEventServer, MinosEventHandler, MinosEventServerService, MinosEventPeriodicService diff --git a/minos/networks/event/__init__.py b/minos/networks/event/__init__.py index ffadcde9..e3936652 100644 --- a/minos/networks/event/__init__.py +++ b/minos/networks/event/__init__.py @@ -6,15 +6,8 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatcher import ( - MinosEventHandler, -) +from .dispatcher import MinosEventHandler -from .event_server import ( - MinosEventServer, -) +from .event_server import MinosEventServer -from .services import ( - MinosEventServerService, - MinosEventPeriodicService -) +from .services import MinosEventServerService, MinosEventPeriodicService diff --git a/minos/networks/event/abc.py b/minos/networks/event/abc.py index e130e7b7..2ce2c51e 100644 --- a/minos/networks/event/abc.py +++ b/minos/networks/event/abc.py @@ -5,20 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ( - ABC, -) -from datetime import ( - datetime, -) -from typing import ( - NoReturn, -) +from abc import ABC +from datetime import datetime +from typing import NoReturn import aiopg -from minos.common import ( - MinosSetup, -) +from minos.common import MinosSetup class MinosEventSetup(MinosSetup): diff --git a/minos/networks/event/dispatcher.py b/minos/networks/event/dispatcher.py index a07f9128..71f69d7a 100644 --- a/minos/networks/event/dispatcher.py +++ b/minos/networks/event/dispatcher.py @@ -5,9 +5,7 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NamedTuple, @@ -19,27 +17,13 @@ import aiopg -from aiomisc.service.periodic import ( - PeriodicService, -) -from minos.common.broker import ( - Event, -) -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.common.importlib import ( - import_module, -) -from minos.common.logs import ( - log, -) -from minos.networks.exceptions import ( - MinosNetworkException, -) -from .abc import ( - MinosEventSetup, -) +from aiomisc.service.periodic import PeriodicService +from minos.common.broker import Event +from minos.common.configuration.config import MinosConfig +from minos.common.importlib import import_module +from minos.common.logs import log +from minos.networks.exceptions import MinosNetworkException +from .abc import MinosEventSetup class MinosEventHandler(MinosEventSetup): diff --git a/minos/networks/event/event_server.py b/minos/networks/event/event_server.py index b69a2193..ba008f0f 100644 --- a/minos/networks/event/event_server.py +++ b/minos/networks/event/event_server.py @@ -5,9 +5,7 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import ( - annotations, -) +from __future__ import annotations import asyncio import datetime @@ -15,26 +13,17 @@ NoReturn, Optional, Awaitable, - Any, AsyncIterator, + Any, + AsyncIterator, ) import aiopg -from minos.common.broker import ( - Event, -) -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.common.logs import ( - log, -) -from minos.networks.exceptions import ( - MinosNetworkException, -) -from .abc import ( - MinosEventSetup, -) +from minos.common.broker import Event +from minos.common.configuration.config import MinosConfig +from minos.common.logs import log +from minos.networks.exceptions import MinosNetworkException +from .abc import MinosEventSetup class MinosEventServer(MinosEventSetup): @@ -127,7 +116,6 @@ async def handle_single_message(self, msg): affected_rows, id = await self.event_queue_add(msg.topic, msg.partition, msg.value) return affected_rows, id - def _is_valid_event(self, value: bytes): try: Event.from_avro_bytes(value) @@ -146,4 +134,3 @@ async def handle_message(self, consumer: AsyncIterator): async for msg in consumer: await self.handle_single_message(msg) - diff --git a/minos/networks/event/services.py b/minos/networks/event/services.py index 02e6e72a..96287b63 100644 --- a/minos/networks/event/services.py +++ b/minos/networks/event/services.py @@ -2,18 +2,10 @@ Service, PeriodicService, ) -from minos.common import ( - MinosConfig, -) -from .dispatcher import ( - MinosEventHandler, -) -from .event_server import ( - MinosEventServer -) -from aiokafka import ( - AIOKafkaConsumer, -) +from minos.common import MinosConfig +from .dispatcher import MinosEventHandler +from .event_server import MinosEventServer +from aiokafka import AIOKafkaConsumer from typing import ( Awaitable, Any, @@ -52,7 +44,7 @@ async def start(self) -> None: await self.consumer.start() self.consumer.subscribe(self.dispatcher._topics) - #self.create_task(self.dispatcher.handle_message(self.consumer)) + # self.create_task(self.dispatcher.handle_message(self.consumer)) await self.dispatcher.handle_message(self.consumer) @@ -61,8 +53,6 @@ async def stop(self, exception: Exception = None) -> Any: await self.consumer.stop() - - class MinosEventPeriodicService(PeriodicService): """Minos QueueDispatcherService class.""" diff --git a/tests/test_networks/test_event/test_dispatcher.py b/tests/test_networks/test_event/test_dispatcher.py index f3397429..b6917d44 100644 --- a/tests/test_networks/test_event/test_dispatcher.py +++ b/tests/test_networks/test_event/test_dispatcher.py @@ -1,23 +1,14 @@ - import datetime import aiopg -from minos.common.testing import ( - PostgresAsyncTestCase, -) -from minos.networks import ( - MinosEventHandler, -) +from minos.common.testing import PostgresAsyncTestCase +from minos.networks import MinosEventHandler from tests.utils import ( BASE_PATH, NaiveAggregate, ) -from minos.common import ( - Event -) -from minos.networks.exceptions import ( - MinosNetworkException, -) +from minos.common import Event +from minos.networks.exceptions import MinosNetworkException class TestEventDispatcher(PostgresAsyncTestCase): @@ -61,7 +52,10 @@ async def test_non_implemented_action(self): cls = m.get_event_handler(topic=event_instance.topic) await cls(topic=event_instance.topic, event=event_instance) - self.assertTrue("topic NotExisting have no controller/action configured, please review th configuration file" in str(context.exception)) + self.assertTrue( + "topic NotExisting have no controller/action configured, please review th configuration file" + in str(context.exception) + ) async def test_none_config(self): event_handler = MinosEventHandler.from_config(config=None) @@ -90,7 +84,6 @@ async def test_event_queue_checker(self): assert affected_rows == 1 assert queue_id[0] > 0 - # Must get the record, call on_reply function and delete the record from DB await event_handler.event_queue_checker() diff --git a/tests/test_networks/test_event/test_event_server.py b/tests/test_networks/test_event/test_event_server.py index 032348bb..7dc28fb7 100644 --- a/tests/test_networks/test_event/test_event_server.py +++ b/tests/test_networks/test_event/test_event_server.py @@ -1,19 +1,13 @@ import time from collections import namedtuple -from minos.common.testing import ( - PostgresAsyncTestCase, -) -from minos.networks import ( - MinosEventServer, -) +from minos.common.testing import PostgresAsyncTestCase +from minos.networks import MinosEventServer from tests.utils import ( BASE_PATH, NaiveAggregate, ) -from minos.common import ( - Event -) +from minos.common import Event from aiokafka import ( AIOKafkaConsumer, AIOKafkaProducer, @@ -83,10 +77,10 @@ async def test_handle_message(self): event_instance = Event(topic="TicketAdded", model=model.classname, items=[model]) bin_data = event_instance.avro_bytes - Mensaje = namedtuple("Mensaje",["topic", "partition", "value"]) + Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) async def consumer(): - yield Mensaje(topic="TicketAdded",partition=0, value=bin_data) + yield Mensaje(topic="TicketAdded", partition=0, value=bin_data) await event_server.handle_message(consumer()) @@ -94,11 +88,11 @@ async def test_handle_message_ko(self): event_server = MinosEventServer.from_config(config=self.config) await event_server.setup() - bin_data = bytes(b'test') + bin_data = bytes(b"test") - Mensaje = namedtuple("Mensaje",["topic", "partition", "value"]) + Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) async def consumer(): - yield Mensaje(topic="TicketAdded",partition=0, value=bin_data) + yield Mensaje(topic="TicketAdded", partition=0, value=bin_data) await event_server.handle_message(consumer()) diff --git a/tests/test_networks/test_event/test_event_services.py b/tests/test_networks/test_event/test_event_services.py index 800f784f..bedc96f3 100644 --- a/tests/test_networks/test_event/test_event_services.py +++ b/tests/test_networks/test_event/test_event_services.py @@ -1,17 +1,8 @@ import unittest -from unittest.mock import ( - MagicMock, -) -from minos.networks import ( - MinosEventServerService, - MinosEventPeriodicService -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) -from tests.utils import ( - BASE_PATH, -) +from unittest.mock import MagicMock +from minos.networks import MinosEventServerService, MinosEventPeriodicService +from minos.common.testing import PostgresAsyncTestCase +from tests.utils import BASE_PATH class TestMinosEventServer(PostgresAsyncTestCase): From 6638add8b1bd30d96c67acb4cc2fab53a4a903db Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 4 May 2021 11:00:12 +0000 Subject: [PATCH 215/239] Restyled by isort --- minos/networks/__init__.py | 7 +++- minos/networks/event/__init__.py | 15 +++++--- minos/networks/event/abc.py | 16 ++++++--- minos/networks/event/dispatcher.py | 36 +++++++++++++------ minos/networks/event/event_server.py | 30 +++++++++++----- minos/networks/event/services.py | 26 +++++++++----- .../test_event/test_dispatcher.py | 18 +++++++--- .../test_event/test_event_server.py | 24 ++++++++----- .../test_event/test_event_services.py | 18 +++++++--- 9 files changed, 136 insertions(+), 54 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 1f347010..4345fdf0 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,6 +13,12 @@ MinosQueueDispatcher, MinosQueueService, ) +from .event import ( + MinosEventHandler, + MinosEventPeriodicService, + MinosEventServer, + MinosEventServerService, +) from .exceptions import ( MinosNetworkException, MinosPreviousVersionSnapshotException, @@ -23,4 +29,3 @@ MinosSnapshotEntry, MinosSnapshotService, ) -from .event import MinosEventServer, MinosEventHandler, MinosEventServerService, MinosEventPeriodicService diff --git a/minos/networks/event/__init__.py b/minos/networks/event/__init__.py index e3936652..3901a526 100644 --- a/minos/networks/event/__init__.py +++ b/minos/networks/event/__init__.py @@ -6,8 +6,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatcher import MinosEventHandler - -from .event_server import MinosEventServer - -from .services import MinosEventServerService, MinosEventPeriodicService +from .dispatcher import ( + MinosEventHandler, +) +from .event_server import ( + MinosEventServer, +) +from .services import ( + MinosEventPeriodicService, + MinosEventServerService, +) diff --git a/minos/networks/event/abc.py b/minos/networks/event/abc.py index 2ce2c51e..e130e7b7 100644 --- a/minos/networks/event/abc.py +++ b/minos/networks/event/abc.py @@ -5,12 +5,20 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ABC -from datetime import datetime -from typing import NoReturn +from abc import ( + ABC, +) +from datetime import ( + datetime, +) +from typing import ( + NoReturn, +) import aiopg -from minos.common import MinosSetup +from minos.common import ( + MinosSetup, +) class MinosEventSetup(MinosSetup): diff --git a/minos/networks/event/dispatcher.py b/minos/networks/event/dispatcher.py index 71f69d7a..9695d4f3 100644 --- a/minos/networks/event/dispatcher.py +++ b/minos/networks/event/dispatcher.py @@ -5,25 +5,41 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( + Any, + Callable, NamedTuple, NoReturn, Optional, - Callable, - Any, ) import aiopg +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common.broker import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.importlib import ( + import_module, +) +from minos.common.logs import ( + log, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) -from aiomisc.service.periodic import PeriodicService -from minos.common.broker import Event -from minos.common.configuration.config import MinosConfig -from minos.common.importlib import import_module -from minos.common.logs import log -from minos.networks.exceptions import MinosNetworkException -from .abc import MinosEventSetup +from .abc import ( + MinosEventSetup, +) class MinosEventHandler(MinosEventSetup): diff --git a/minos/networks/event/event_server.py b/minos/networks/event/event_server.py index ba008f0f..a078de6d 100644 --- a/minos/networks/event/event_server.py +++ b/minos/networks/event/event_server.py @@ -5,25 +5,37 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import annotations +from __future__ import ( + annotations, +) import asyncio import datetime from typing import ( - NoReturn, - Optional, - Awaitable, Any, AsyncIterator, + Awaitable, + NoReturn, + Optional, ) import aiopg +from minos.common.broker import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.logs import ( + log, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) -from minos.common.broker import Event -from minos.common.configuration.config import MinosConfig -from minos.common.logs import log -from minos.networks.exceptions import MinosNetworkException -from .abc import MinosEventSetup +from .abc import ( + MinosEventSetup, +) class MinosEventServer(MinosEventSetup): diff --git a/minos/networks/event/services.py b/minos/networks/event/services.py index 96287b63..2cb3dd24 100644 --- a/minos/networks/event/services.py +++ b/minos/networks/event/services.py @@ -1,14 +1,24 @@ +from typing import ( + Any, + Awaitable, +) + +from aiokafka import ( + AIOKafkaConsumer, +) from aiomisc.service.periodic import ( - Service, PeriodicService, + Service, ) -from minos.common import MinosConfig -from .dispatcher import MinosEventHandler -from .event_server import MinosEventServer -from aiokafka import AIOKafkaConsumer -from typing import ( - Awaitable, - Any, +from minos.common import ( + MinosConfig, +) + +from .dispatcher import ( + MinosEventHandler, +) +from .event_server import ( + MinosEventServer, ) diff --git a/tests/test_networks/test_event/test_dispatcher.py b/tests/test_networks/test_event/test_dispatcher.py index b6917d44..866f2070 100644 --- a/tests/test_networks/test_event/test_dispatcher.py +++ b/tests/test_networks/test_event/test_dispatcher.py @@ -1,14 +1,22 @@ import datetime -import aiopg -from minos.common.testing import PostgresAsyncTestCase -from minos.networks import MinosEventHandler +import aiopg +from minos.common import ( + Event, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosEventHandler, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) from tests.utils import ( BASE_PATH, NaiveAggregate, ) -from minos.common import Event -from minos.networks.exceptions import MinosNetworkException class TestEventDispatcher(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_event/test_event_server.py b/tests/test_networks/test_event/test_event_server.py index 7dc28fb7..ff59fd11 100644 --- a/tests/test_networks/test_event/test_event_server.py +++ b/tests/test_networks/test_event/test_event_server.py @@ -1,17 +1,25 @@ import time -from collections import namedtuple - -from minos.common.testing import PostgresAsyncTestCase -from minos.networks import MinosEventServer -from tests.utils import ( - BASE_PATH, - NaiveAggregate, +from collections import ( + namedtuple, ) -from minos.common import Event + from aiokafka import ( AIOKafkaConsumer, AIOKafkaProducer, ) +from minos.common import ( + Event, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosEventServer, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) async def kafka_producer(config): diff --git a/tests/test_networks/test_event/test_event_services.py b/tests/test_networks/test_event/test_event_services.py index bedc96f3..cd86703a 100644 --- a/tests/test_networks/test_event/test_event_services.py +++ b/tests/test_networks/test_event/test_event_services.py @@ -1,8 +1,18 @@ import unittest -from unittest.mock import MagicMock -from minos.networks import MinosEventServerService, MinosEventPeriodicService -from minos.common.testing import PostgresAsyncTestCase -from tests.utils import BASE_PATH +from unittest.mock import ( + MagicMock, +) + +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosEventPeriodicService, + MinosEventServerService, +) +from tests.utils import ( + BASE_PATH, +) class TestMinosEventServer(PostgresAsyncTestCase): From 3ed9b327d003b1bd3d268f7b564f69aaa709d8b9 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 6 May 2021 11:57:44 +0200 Subject: [PATCH 216/239] Update minos_microservice_common library to 0.0.7 Update minos_microservice_common library to 0.0.7 #111 --- minos/networks/broker/commands.py | 6 ++++-- minos/networks/event/dispatcher.py | 2 +- minos/networks/event/event_server.py | 2 +- tests/test_networks/test_broker/test_command.py | 6 +++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/minos/networks/broker/commands.py b/minos/networks/broker/commands.py index 29fe5e86..0357893c 100644 --- a/minos/networks/broker/commands.py +++ b/minos/networks/broker/commands.py @@ -30,9 +30,11 @@ class MinosCommandBroker(MinosBroker): ACTION = "command" - def __init__(self, *args, reply_on: str, **kwargs): + def __init__(self, *args, saga_id: str, task_id: str, reply_on: str, **kwargs): super().__init__(*args, **kwargs) self.reply_on = reply_on + self.saga_id = saga_id + self.task_id = task_id @classmethod def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosCommandBroker]: @@ -55,5 +57,5 @@ async def send(self, items: list[Aggregate]) -> int: :param items: A list of aggregates. :return: This method does not return anything. """ - command = Command(self.topic, items, self.reply_on) + command = Command(self.topic, items, self.saga_id, self.task_id, self.reply_on) return await self._send_bytes(command.topic, command.avro_bytes) diff --git a/minos/networks/event/dispatcher.py b/minos/networks/event/dispatcher.py index 9695d4f3..6570f3b3 100644 --- a/minos/networks/event/dispatcher.py +++ b/minos/networks/event/dispatcher.py @@ -21,7 +21,7 @@ from aiomisc.service.periodic import ( PeriodicService, ) -from minos.common.broker import ( +from minos.common import ( Event, ) from minos.common.configuration.config import ( diff --git a/minos/networks/event/event_server.py b/minos/networks/event/event_server.py index a078de6d..e7326139 100644 --- a/minos/networks/event/event_server.py +++ b/minos/networks/event/event_server.py @@ -20,7 +20,7 @@ ) import aiopg -from minos.common.broker import ( +from minos.common import ( Event, ) from minos.common.configuration.config import ( diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 15780838..30d56a19 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -28,7 +28,7 @@ class TestMinosCommandBroker(PostgresAsyncTestCase): CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" async def test_commands_broker_insertion(self): - broker = MinosCommandBroker.from_config("CommandBroker", config=self.config, reply_on="test_reply_on") + broker = MinosCommandBroker.from_config("CommandBroker", config=self.config, saga_id= "9347839473kfslf", task_id= "92839283hjijh232", reply_on="test_reply_on") await broker.setup() item = NaiveAggregate(test_id=1, test=2, id=1, version=1) @@ -37,7 +37,7 @@ async def test_commands_broker_insertion(self): assert queue_id > 0 async def test_if_commands_was_deleted(self): - broker = MinosCommandBroker.from_config("CommandBroker-Delete", config=self.config, reply_on="test_reply_on") + broker = MinosCommandBroker.from_config("CommandBroker-Delete", config=self.config, saga_id= "9347839473kfslf", task_id= "92839283hjijh232", reply_on="test_reply_on") await broker.setup() item = NaiveAggregate(test_id=1, test=2, id=1, version=1) @@ -57,7 +57,7 @@ async def test_if_commands_was_deleted(self): assert records[0] == 0 async def test_if_commands_retry_was_incremented(self): - broker = MinosCommandBroker.from_config("CommandBroker-Delete", config=self.config, reply_on="test_reply_on") + broker = MinosCommandBroker.from_config("CommandBroker-Delete", config=self.config, saga_id= "9347839473kfslf", task_id= "92839283hjijh232", reply_on="test_reply_on") await broker.setup() item = NaiveAggregate(test_id=1, test=2, id=1, version=1) From 55037e7abd798847d738bf8d083b50840527d238 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 6 May 2021 14:30:39 +0200 Subject: [PATCH 217/239] Refactor EventHandler and create new package for handler Refactor EventHandler and create new package for handler #114 --- minos/networks/__init__.py | 2 +- minos/networks/handler/__init__.py | 14 ++++++++++++++ minos/networks/{event => handler}/abc.py | 8 +++++--- minos/networks/handler/command/__init__.py | 0 .../handler/command_reply/__init__.py | 0 .../networks/{ => handler}/event/__init__.py | 0 .../{ => handler}/event/dispatcher.py | 15 ++++++++------- .../{ => handler}/event/event_server.py | 19 ++++++------------- .../networks/{ => handler}/event/services.py | 0 tests/test_networks/test_handler/__init__.py | 0 .../{ => test_handler}/test_event/__init__.py | 0 .../test_event/test_dispatcher.py | 0 .../test_event/test_event_server.py | 0 .../test_event/test_event_services.py | 0 14 files changed, 34 insertions(+), 24 deletions(-) create mode 100644 minos/networks/handler/__init__.py rename minos/networks/{event => handler}/abc.py (86%) create mode 100644 minos/networks/handler/command/__init__.py create mode 100644 minos/networks/handler/command_reply/__init__.py rename minos/networks/{ => handler}/event/__init__.py (100%) rename minos/networks/{ => handler}/event/dispatcher.py (92%) rename minos/networks/{ => handler}/event/event_server.py (93%) rename minos/networks/{ => handler}/event/services.py (100%) create mode 100644 tests/test_networks/test_handler/__init__.py rename tests/test_networks/{ => test_handler}/test_event/__init__.py (100%) rename tests/test_networks/{ => test_handler}/test_event/test_dispatcher.py (100%) rename tests/test_networks/{ => test_handler}/test_event/test_event_server.py (100%) rename tests/test_networks/{ => test_handler}/test_event/test_event_services.py (100%) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 4345fdf0..04aa5dd0 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,7 +13,7 @@ MinosQueueDispatcher, MinosQueueService, ) -from .event import ( +from .handler import ( MinosEventHandler, MinosEventPeriodicService, MinosEventServer, diff --git a/minos/networks/handler/__init__.py b/minos/networks/handler/__init__.py new file mode 100644 index 00000000..ae23b7e7 --- /dev/null +++ b/minos/networks/handler/__init__.py @@ -0,0 +1,14 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from .event import ( + MinosEventHandler, + MinosEventServer, + MinosEventPeriodicService, + MinosEventServerService, +) diff --git a/minos/networks/event/abc.py b/minos/networks/handler/abc.py similarity index 86% rename from minos/networks/event/abc.py rename to minos/networks/handler/abc.py index e130e7b7..4b6d37a7 100644 --- a/minos/networks/event/abc.py +++ b/minos/networks/handler/abc.py @@ -21,11 +21,12 @@ ) -class MinosEventSetup(MinosSetup): +class MinosHandlerSetup(MinosSetup): """Minos Broker Setup Class""" def __init__( self, + table_name: str, *args, host: str = None, port: int = None, @@ -41,6 +42,7 @@ def __init__( self.database = database self.user = user self.password = password + self.table_name = table_name async def _setup(self) -> NoReturn: await self._create_event_queue_table() @@ -49,12 +51,12 @@ async def _create_event_queue_table(self) -> NoReturn: async with self._connection() as connect: async with connect.cursor() as cur: await cur.execute( - 'CREATE TABLE IF NOT EXISTS "event_queue" (' + 'CREATE TABLE IF NOT EXISTS "%s" (' '"id" BIGSERIAL NOT NULL PRIMARY KEY, ' '"topic" VARCHAR(255) NOT NULL, ' '"partition_id" INTEGER,' '"binary_data" BYTEA NOT NULL, ' - '"creation_date" TIMESTAMP NOT NULL);' + '"creation_date" TIMESTAMP NOT NULL);' % (self.table_name) ) def _connection(self): diff --git a/minos/networks/handler/command/__init__.py b/minos/networks/handler/command/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/minos/networks/handler/command_reply/__init__.py b/minos/networks/handler/command_reply/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/minos/networks/event/__init__.py b/minos/networks/handler/event/__init__.py similarity index 100% rename from minos/networks/event/__init__.py rename to minos/networks/handler/event/__init__.py diff --git a/minos/networks/event/dispatcher.py b/minos/networks/handler/event/dispatcher.py similarity index 92% rename from minos/networks/event/dispatcher.py rename to minos/networks/handler/event/dispatcher.py index 6570f3b3..7a02478b 100644 --- a/minos/networks/event/dispatcher.py +++ b/minos/networks/handler/event/dispatcher.py @@ -37,21 +37,22 @@ MinosNetworkException, ) -from .abc import ( - MinosEventSetup, +from ..abc import ( + MinosHandlerSetup, ) -class MinosEventHandler(MinosEventSetup): +class MinosEventHandler(MinosHandlerSetup): """ Event Handler """ + TABLE = "event_queue" __slots__ = "_db_dsn", "_handlers", "_event_items", "_topics", "_conf" def __init__(self, *, config: MinosConfig, **kwargs: Any): - super().__init__(**kwargs, **config.events.queue._asdict()) + super().__init__(table_name=self.TABLE, **kwargs, **config.events.queue._asdict()) self._db_dsn = ( f"dbname={config.events.queue.database} user={config.events.queue.user} " f"password={config.events.queue.password} host={config.events.queue.host}" @@ -126,8 +127,8 @@ async def event_queue_checker(self) -> NoReturn: async with pool.acquire() as connect: async with connect.cursor() as cur: await cur.execute( - "SELECT * FROM event_queue ORDER BY creation_date ASC LIMIT %d;" - % (self._conf.events.queue.records), + "SELECT * FROM %s ORDER BY creation_date ASC LIMIT %d;" + % (self.TABLE, self._conf.events.queue.records), ) async for row in cur: call_ok = False @@ -140,4 +141,4 @@ async def event_queue_checker(self) -> NoReturn: if call_ok: # Delete from database If the event was sent successfully to Kafka. async with connect.cursor() as cur2: - await cur2.execute("DELETE FROM event_queue WHERE id=%d;" % row[0]) + await cur2.execute("DELETE FROM %s WHERE id=%d;" % (self.TABLE, row[0])) diff --git a/minos/networks/event/event_server.py b/minos/networks/handler/event/event_server.py similarity index 93% rename from minos/networks/event/event_server.py rename to minos/networks/handler/event/event_server.py index e7326139..9a46c941 100644 --- a/minos/networks/event/event_server.py +++ b/minos/networks/handler/event/event_server.py @@ -14,8 +14,6 @@ from typing import ( Any, AsyncIterator, - Awaitable, - NoReturn, Optional, ) @@ -26,19 +24,12 @@ from minos.common.configuration.config import ( MinosConfig, ) -from minos.common.logs import ( - log, -) -from minos.networks.exceptions import ( - MinosNetworkException, -) - -from .abc import ( - MinosEventSetup, +from ..abc import ( + MinosHandlerSetup, ) -class MinosEventServer(MinosEventSetup): +class MinosEventServer(MinosHandlerSetup): """ Event Manager @@ -46,10 +37,12 @@ class MinosEventServer(MinosEventSetup): """ + TABLE = "event_queue" + __slots__ = "_tasks", "_db_dsn", "_handlers", "_topics", "_kafka_conn_data", "_broker_group_name" def __init__(self, *, config: MinosConfig, **kwargs: Any): - super().__init__(**kwargs, **config.events.queue._asdict()) + super().__init__(table_name=self.TABLE, **kwargs, **config.events.queue._asdict()) self._tasks = set() # type: t.Set[asyncio.Task] self._db_dsn = ( f"dbname={config.events.queue.database} user={config.events.queue.user} " diff --git a/minos/networks/event/services.py b/minos/networks/handler/event/services.py similarity index 100% rename from minos/networks/event/services.py rename to minos/networks/handler/event/services.py diff --git a/tests/test_networks/test_handler/__init__.py b/tests/test_networks/test_handler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_networks/test_event/__init__.py b/tests/test_networks/test_handler/test_event/__init__.py similarity index 100% rename from tests/test_networks/test_event/__init__.py rename to tests/test_networks/test_handler/test_event/__init__.py diff --git a/tests/test_networks/test_event/test_dispatcher.py b/tests/test_networks/test_handler/test_event/test_dispatcher.py similarity index 100% rename from tests/test_networks/test_event/test_dispatcher.py rename to tests/test_networks/test_handler/test_event/test_dispatcher.py diff --git a/tests/test_networks/test_event/test_event_server.py b/tests/test_networks/test_handler/test_event/test_event_server.py similarity index 100% rename from tests/test_networks/test_event/test_event_server.py rename to tests/test_networks/test_handler/test_event/test_event_server.py diff --git a/tests/test_networks/test_event/test_event_services.py b/tests/test_networks/test_handler/test_event/test_event_services.py similarity index 100% rename from tests/test_networks/test_event/test_event_services.py rename to tests/test_networks/test_handler/test_event/test_event_services.py From 7aa454ad98f22c1187e109502c0f2507abbfd10f Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Thu, 6 May 2021 18:54:22 +0200 Subject: [PATCH 218/239] Extract logic to generic MinosHandlerServer CommandHandler #109 --- minos/networks/__init__.py | 2 +- minos/networks/handler/__init__.py | 2 +- minos/networks/handler/event/__init__.py | 4 +-- minos/networks/handler/event/server.py | 28 ++++++++++++++++ minos/networks/handler/event/services.py | 7 ++-- .../{event/event_server.py => server.py} | 32 +++++++++---------- .../test_event/test_event_server.py | 18 +++++------ 7 files changed, 60 insertions(+), 33 deletions(-) create mode 100644 minos/networks/handler/event/server.py rename minos/networks/handler/{event/event_server.py => server.py} (76%) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 04aa5dd0..72fccf4d 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -16,7 +16,7 @@ from .handler import ( MinosEventHandler, MinosEventPeriodicService, - MinosEventServer, + MinosEventHandlerServer, MinosEventServerService, ) from .exceptions import ( diff --git a/minos/networks/handler/__init__.py b/minos/networks/handler/__init__.py index ae23b7e7..b2d62279 100644 --- a/minos/networks/handler/__init__.py +++ b/minos/networks/handler/__init__.py @@ -8,7 +8,7 @@ from .event import ( MinosEventHandler, - MinosEventServer, + MinosEventHandlerServer, MinosEventPeriodicService, MinosEventServerService, ) diff --git a/minos/networks/handler/event/__init__.py b/minos/networks/handler/event/__init__.py index 3901a526..97ef115c 100644 --- a/minos/networks/handler/event/__init__.py +++ b/minos/networks/handler/event/__init__.py @@ -9,8 +9,8 @@ from .dispatcher import ( MinosEventHandler, ) -from .event_server import ( - MinosEventServer, +from .server import ( + MinosEventHandlerServer, ) from .services import ( MinosEventPeriodicService, diff --git a/minos/networks/handler/event/server.py b/minos/networks/handler/event/server.py new file mode 100644 index 00000000..dba09bf9 --- /dev/null +++ b/minos/networks/handler/event/server.py @@ -0,0 +1,28 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from minos.common.configuration.config import ( + MinosConfig, +) +from typing import ( + Any, + NamedTuple, + AsyncIterator, + Optional, +) +from ..server import ( + MinosHandlerServer, +) + + +class MinosEventHandlerServer(MinosHandlerServer): + + TABLE = "event_queue" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(table_name=self.TABLE, config=config.events, **kwargs) + self._broker_group_name = f"event_{config.service.name}" diff --git a/minos/networks/handler/event/services.py b/minos/networks/handler/event/services.py index 2cb3dd24..7e473e2c 100644 --- a/minos/networks/handler/event/services.py +++ b/minos/networks/handler/event/services.py @@ -1,6 +1,5 @@ from typing import ( Any, - Awaitable, ) from aiokafka import ( @@ -17,8 +16,8 @@ from .dispatcher import ( MinosEventHandler, ) -from .event_server import ( - MinosEventServer, +from .server import ( + MinosEventHandlerServer, ) @@ -27,7 +26,7 @@ class MinosEventServerService(Service): def __init__(self, config: MinosConfig = None, **kwargs): super().__init__(**kwargs) - self.dispatcher = MinosEventServer.from_config(config=config) + self.dispatcher = MinosEventHandlerServer.from_config(config=config) self.consumer = None """ diff --git a/minos/networks/handler/event/event_server.py b/minos/networks/handler/server.py similarity index 76% rename from minos/networks/handler/event/event_server.py rename to minos/networks/handler/server.py index 9a46c941..c176dd57 100644 --- a/minos/networks/handler/event/event_server.py +++ b/minos/networks/handler/server.py @@ -8,13 +8,14 @@ from __future__ import ( annotations, ) - +from psycopg2.extensions import AsIs import asyncio import datetime from typing import ( Any, AsyncIterator, Optional, + NamedTuple, ) import aiopg @@ -24,12 +25,12 @@ from minos.common.configuration.config import ( MinosConfig, ) -from ..abc import ( +from minos.networks.handler.abc import ( MinosHandlerSetup, ) -class MinosEventServer(MinosHandlerSetup): +class MinosHandlerServer(MinosHandlerSetup): """ Event Manager @@ -37,26 +38,25 @@ class MinosEventServer(MinosHandlerSetup): """ - TABLE = "event_queue" __slots__ = "_tasks", "_db_dsn", "_handlers", "_topics", "_kafka_conn_data", "_broker_group_name" - def __init__(self, *, config: MinosConfig, **kwargs: Any): - super().__init__(table_name=self.TABLE, **kwargs, **config.events.queue._asdict()) + def __init__(self, *, table_name: str, config: NamedTuple, **kwargs: Any): + super().__init__(table_name=table_name, **kwargs, **config.queue._asdict()) self._tasks = set() # type: t.Set[asyncio.Task] self._db_dsn = ( - f"dbname={config.events.queue.database} user={config.events.queue.user} " - f"password={config.events.queue.password} host={config.events.queue.host}" + f"dbname={config.queue.database} user={config.queue.user} " + f"password={config.queue.password} host={config.queue.host}" ) self._handler = { - item.name: {"controller": item.controller, "action": item.action} for item in config.events.items + item.name: {"controller": item.controller, "action": item.action} for item in config.items } self._topics = list(self._handler.keys()) - self._kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" - self._broker_group_name = f"event_{config.service.name}" + self._kafka_conn_data = f"{config.broker.host}:{config.broker.port}" + self._table_name = table_name @classmethod - def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosEventServer]: + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosHandlerServer]: """Build a new repository from config. :param args: Additional positional arguments. :param config: Config instance. If `None` is provided, default config is chosen. @@ -70,7 +70,7 @@ def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[Mi # noinspection PyProtectedMember return cls(*args, config=config, **kwargs) - async def event_queue_add(self, topic: str, partition: int, binary: bytes): + async def queue_add(self, topic: str, partition: int, binary: bytes): """Insert row to event_queue table. Retrieves number of affected rows and row ID. @@ -93,8 +93,8 @@ async def event_queue_add(self, topic: str, partition: int, binary: bytes): async with pool.acquire() as connect: async with connect.cursor() as cur: await cur.execute( - "INSERT INTO event_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", - (topic, partition, binary, datetime.datetime.now(),), + "INSERT INTO %s (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + (AsIs(self._table_name), topic, partition, binary, datetime.datetime.now(),), ) queue_id = await cur.fetchone() @@ -118,7 +118,7 @@ async def handle_single_message(self, msg): # check if the event binary string is well formatted if not self._is_valid_event(msg.value): return - affected_rows, id = await self.event_queue_add(msg.topic, msg.partition, msg.value) + affected_rows, id = await self.queue_add(msg.topic, msg.partition, msg.value) return affected_rows, id def _is_valid_event(self, value: bytes): diff --git a/tests/test_networks/test_handler/test_event/test_event_server.py b/tests/test_networks/test_handler/test_event/test_event_server.py index ff59fd11..ac70425c 100644 --- a/tests/test_networks/test_handler/test_event/test_event_server.py +++ b/tests/test_networks/test_handler/test_event/test_event_server.py @@ -14,7 +14,7 @@ PostgresAsyncTestCase, ) from minos.networks import ( - MinosEventServer, + MinosEventHandlerServer, ) from tests.utils import ( BASE_PATH, @@ -55,30 +55,30 @@ class TestEventServer(PostgresAsyncTestCase): CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" def test_from_config(self): - dispatcher = MinosEventServer.from_config(config=self.config) - self.assertIsInstance(dispatcher, MinosEventServer) + dispatcher = MinosEventHandlerServer.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosEventHandlerServer) async def test_none_config(self): - event_server = MinosEventServer.from_config(config=None) + event_server = MinosEventHandlerServer.from_config(config=None) self.assertIsNone(event_server) - async def test_event_queue_add(self): + async def test_queue_add(self): model = NaiveAggregate(test_id=1, test=2, id=1, version=1) event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) bin_data = event_instance.avro_bytes Event.from_avro_bytes(bin_data) - event_server = MinosEventServer.from_config(config=self.config) + event_server = MinosEventHandlerServer.from_config(config=self.config) await event_server.setup() - affected_rows, id = await event_server.event_queue_add(topic=event_instance.topic, partition=0, binary=bin_data) + affected_rows, id = await event_server.queue_add(topic=event_instance.topic, partition=0, binary=bin_data) assert affected_rows == 1 assert id > 0 async def test_handle_message(self): - event_server = MinosEventServer.from_config(config=self.config) + event_server = MinosEventHandlerServer.from_config(config=self.config) await event_server.setup() model = NaiveAggregate(test_id=1, test=2, id=1, version=1) @@ -93,7 +93,7 @@ async def consumer(): await event_server.handle_message(consumer()) async def test_handle_message_ko(self): - event_server = MinosEventServer.from_config(config=self.config) + event_server = MinosEventHandlerServer.from_config(config=self.config) await event_server.setup() bin_data = bytes(b"test") From d939ca23466701d7f61aa093a5b83db2a53d5f2f Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 7 May 2021 09:46:23 +0200 Subject: [PATCH 219/239] Create common Dispatcher Handler Refactor EventHandler and create new package for handler #114 --- minos/networks/__init__.py | 2 +- minos/networks/handler/__init__.py | 2 +- minos/networks/handler/dispatcher.py | 145 ++++++++++++++++++ minos/networks/handler/event/__init__.py | 6 +- minos/networks/handler/event/dispatcher.py | 137 ++--------------- minos/networks/handler/event/server.py | 13 +- minos/networks/handler/event/services.py | 6 +- minos/networks/handler/server.py | 18 +-- .../test_event/test_dispatcher.py | 18 +-- .../test_event/test_event_services.py | 4 +- 10 files changed, 195 insertions(+), 156 deletions(-) create mode 100644 minos/networks/handler/dispatcher.py diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 72fccf4d..1e6a612d 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -14,7 +14,7 @@ MinosQueueService, ) from .handler import ( - MinosEventHandler, + MinosEventHandlerDispatcher, MinosEventPeriodicService, MinosEventHandlerServer, MinosEventServerService, diff --git a/minos/networks/handler/__init__.py b/minos/networks/handler/__init__.py index b2d62279..a4e38280 100644 --- a/minos/networks/handler/__init__.py +++ b/minos/networks/handler/__init__.py @@ -7,7 +7,7 @@ """ from .event import ( - MinosEventHandler, + MinosEventHandlerDispatcher, MinosEventHandlerServer, MinosEventPeriodicService, MinosEventServerService, diff --git a/minos/networks/handler/dispatcher.py b/minos/networks/handler/dispatcher.py new file mode 100644 index 00000000..2a03e110 --- /dev/null +++ b/minos/networks/handler/dispatcher.py @@ -0,0 +1,145 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from __future__ import ( + annotations, +) + +from abc import abstractmethod +from typing import ( + Any, + Callable, + NoReturn, + Optional, + NamedTuple, +) + +import aiopg +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.importlib import ( + import_module, +) +from minos.common.logs import ( + log, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) + +from minos.networks.handler.abc import ( + MinosHandlerSetup, +) + + +class MinosHandlerDispatcher(MinosHandlerSetup): + """ + Event Handler + + """ + + __slots__ = "_db_dsn", "_handlers", "_event_items", "_topics", "_conf" + + def __init__(self, *, table_name: str, config: NamedTuple, **kwargs: Any): + super().__init__(table_name=table_name, **kwargs, **config.queue._asdict()) + self._db_dsn = ( + f"dbname={config.queue.database} user={config.queue.user} " + f"password={config.queue.password} host={config.queue.host}" + ) + self._handlers = { + item.name: {"controller": item.controller, "action": item.action} for item in config.items + } + self._event_items = config.items + self._topics = list(self._handlers.keys()) + self._conf = config + self._table_name = table_name + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosHandlerDispatcher]: + """Build a new repository from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + return None + # noinspection PyProtectedMember + return cls(*args, config=config, **kwargs) + + def get_event_handler(self, topic: str) -> Callable: + + """Get Event instance to call. + + Gets the instance of the class and method to call. + + Args: + topic: Kafka topic. Example: "TicketAdded" + + Raises: + MinosNetworkException: topic TicketAdded have no controller/action configured, please review th configuration file. + """ + for event in self._event_items: + if event.name == topic: + # the topic exist, get the controller and the action + controller = event.controller + action = event.action + + object_class = import_module(controller) + log.debug(object_class()) + instance_class = object_class() + class_method = getattr(instance_class, action) + + return class_method + raise MinosNetworkException( + f"topic {topic} have no controller/action configured, " f"please review th configuration file" + ) + + async def queue_checker(self) -> NoReturn: + """Event Queue Checker and dispatcher. + + It is in charge of querying the database and calling the action according to the topic. + + 1. Get periodically 10 records (or as many as defined in config > queue > records). + 2. Instantiate the action (asynchronous) by passing it the model. + 3. If the invoked function terminates successfully, remove the event from the database. + + Raises: + Exception: An error occurred inserting record. + """ + db_dsn = ( + f"dbname={self._conf.queue.database} user={self._conf.queue.user} " + f"password={self._conf.queue.password} host={self._conf.queue.host}" + ) + async with aiopg.create_pool(db_dsn) as pool: + async with pool.acquire() as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT * FROM %s ORDER BY creation_date ASC LIMIT %d;" + % (self._table_name, self._conf.queue.records), + ) + async for row in cur: + call_ok = False + try: + reply_on = self.get_event_handler(topic=row[1]) + valid_instance, event_instance = self._is_valid_event(row[3]) + if not valid_instance: + return + await reply_on(topic=row[1], event=event_instance) + call_ok = True + finally: + if call_ok: + # Delete from database If the event was sent successfully to Kafka. + async with connect.cursor() as cur2: + await cur2.execute("DELETE FROM %s WHERE id=%d;" % (self._table_name, row[0])) + + @abstractmethod + def _is_valid_event(self, value: bytes): + raise Exception("Method not implemented") diff --git a/minos/networks/handler/event/__init__.py b/minos/networks/handler/event/__init__.py index 97ef115c..da35486e 100644 --- a/minos/networks/handler/event/__init__.py +++ b/minos/networks/handler/event/__init__.py @@ -6,9 +6,6 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .dispatcher import ( - MinosEventHandler, -) from .server import ( MinosEventHandlerServer, ) @@ -16,3 +13,6 @@ MinosEventPeriodicService, MinosEventServerService, ) +from .dispatcher import ( + MinosEventHandlerDispatcher, +) diff --git a/minos/networks/handler/event/dispatcher.py b/minos/networks/handler/event/dispatcher.py index 7a02478b..f635552a 100644 --- a/minos/networks/handler/event/dispatcher.py +++ b/minos/networks/handler/event/dispatcher.py @@ -5,140 +5,31 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import ( - annotations, +from minos.common.configuration.config import ( + MinosConfig, ) - from typing import ( Any, - Callable, - NamedTuple, - NoReturn, - Optional, ) - -import aiopg -from aiomisc.service.periodic import ( - PeriodicService, +from ..dispatcher import ( + MinosHandlerDispatcher, ) from minos.common import ( Event, ) -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.common.importlib import ( - import_module, -) -from minos.common.logs import ( - log, -) -from minos.networks.exceptions import ( - MinosNetworkException, -) - -from ..abc import ( - MinosHandlerSetup, -) -class MinosEventHandler(MinosHandlerSetup): - """ - Event Handler +class MinosEventHandlerDispatcher(MinosHandlerDispatcher): - """ TABLE = "event_queue" - __slots__ = "_db_dsn", "_handlers", "_event_items", "_topics", "_conf" - def __init__(self, *, config: MinosConfig, **kwargs: Any): - super().__init__(table_name=self.TABLE, **kwargs, **config.events.queue._asdict()) - self._db_dsn = ( - f"dbname={config.events.queue.database} user={config.events.queue.user} " - f"password={config.events.queue.password} host={config.events.queue.host}" - ) - self._handlers = { - item.name: {"controller": item.controller, "action": item.action} for item in config.events.items - } - self._event_items = config.events.items - self._topics = list(self._handlers.keys()) - self._conf = config - - @classmethod - def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosEventHandler]: - """Build a new repository from config. - :param args: Additional positional arguments. - :param config: Config instance. If `None` is provided, default config is chosen. - :param kwargs: Additional named arguments. - :return: A `MinosRepository` instance. - """ - if config is None: - config = MinosConfig.get_default() - if config is None: - return None - # noinspection PyProtectedMember - return cls(*args, config=config, **kwargs) - - def get_event_handler(self, topic: str) -> Callable: - - """Get Event instance to call. - - Gets the instance of the class and method to call. - - Args: - topic: Kafka topic. Example: "TicketAdded" - - Raises: - MinosNetworkException: topic TicketAdded have no controller/action configured, please review th configuration file. - """ - for event in self._event_items: - if event.name == topic: - # the topic exist, get the controller and the action - controller = event.controller - action = event.action - - object_class = import_module(controller) - log.debug(object_class()) - instance_class = object_class() - class_method = getattr(instance_class, action) - - return class_method - raise MinosNetworkException( - f"topic {topic} have no controller/action configured, " f"please review th configuration file" - ) - - async def event_queue_checker(self) -> NoReturn: - """Event Queue Checker and dispatcher. - - It is in charge of querying the database and calling the action according to the topic. - - 1. Get periodically 10 records (or as many as defined in config > queue > records). - 2. Instantiate the action (asynchronous) by passing it the model. - 3. If the invoked function terminates successfully, remove the event from the database. - - Raises: - Exception: An error occurred inserting record. - """ - db_dsn = ( - f"dbname={self._conf.events.queue.database} user={self._conf.events.queue.user} " - f"password={self._conf.events.queue.password} host={self._conf.events.queue.host}" - ) - async with aiopg.create_pool(db_dsn) as pool: - async with pool.acquire() as connect: - async with connect.cursor() as cur: - await cur.execute( - "SELECT * FROM %s ORDER BY creation_date ASC LIMIT %d;" - % (self.TABLE, self._conf.events.queue.records), - ) - async for row in cur: - call_ok = False - try: - reply_on = self.get_event_handler(topic=row[1]) - event_instance = Event.from_avro_bytes(row[3]) - await reply_on(topic=row[1], event=event_instance) - call_ok = True - finally: - if call_ok: - # Delete from database If the event was sent successfully to Kafka. - async with connect.cursor() as cur2: - await cur2.execute("DELETE FROM %s WHERE id=%d;" % (self.TABLE, row[0])) + super().__init__(table_name=self.TABLE, config=config.events, **kwargs) + self._broker_group_name = f"event_{config.service.name}" + + def _is_valid_event(self, value: bytes): + try: + event_instance = Event.from_avro_bytes(value) + return True, event_instance + except: + return False, None diff --git a/minos/networks/handler/event/server.py b/minos/networks/handler/event/server.py index dba09bf9..fce63571 100644 --- a/minos/networks/handler/event/server.py +++ b/minos/networks/handler/event/server.py @@ -10,13 +10,13 @@ ) from typing import ( Any, - NamedTuple, - AsyncIterator, - Optional, ) from ..server import ( MinosHandlerServer, ) +from minos.common import ( + Event, +) class MinosEventHandlerServer(MinosHandlerServer): @@ -26,3 +26,10 @@ class MinosEventHandlerServer(MinosHandlerServer): def __init__(self, *, config: MinosConfig, **kwargs: Any): super().__init__(table_name=self.TABLE, config=config.events, **kwargs) self._broker_group_name = f"event_{config.service.name}" + + def _is_valid_event(self, value: bytes): + try: + Event.from_avro_bytes(value) + return True + except: + return False diff --git a/minos/networks/handler/event/services.py b/minos/networks/handler/event/services.py index 7e473e2c..e75490f6 100644 --- a/minos/networks/handler/event/services.py +++ b/minos/networks/handler/event/services.py @@ -14,7 +14,7 @@ ) from .dispatcher import ( - MinosEventHandler, + MinosEventHandlerDispatcher, ) from .server import ( MinosEventHandlerServer, @@ -67,7 +67,7 @@ class MinosEventPeriodicService(PeriodicService): def __init__(self, config: MinosConfig = None, **kwargs): super().__init__(**kwargs) - self.dispatcher = MinosEventHandler.from_config(config=config) + self.dispatcher = MinosEventHandlerDispatcher.from_config(config=config) async def start(self) -> None: """Method to be called at the startup by the internal ``aiomisc`` loigc. @@ -82,4 +82,4 @@ async def callback(self) -> None: :return:This method does not return anything. """ - await self.dispatcher.event_queue_checker() + await self.dispatcher.queue_checker() diff --git a/minos/networks/handler/server.py b/minos/networks/handler/server.py index c176dd57..8d6e275c 100644 --- a/minos/networks/handler/server.py +++ b/minos/networks/handler/server.py @@ -8,6 +8,9 @@ from __future__ import ( annotations, ) + +from abc import abstractmethod + from psycopg2.extensions import AsIs import asyncio import datetime @@ -17,11 +20,7 @@ Optional, NamedTuple, ) - import aiopg -from minos.common import ( - Event, -) from minos.common.configuration.config import ( MinosConfig, ) @@ -32,9 +31,9 @@ class MinosHandlerServer(MinosHandlerSetup): """ - Event Manager + Handler Server - Consumer for the Broker ( at the moment only Kafka is supported ) + Generic insert for queue_* table. (Support Command, CommandReply and Event) """ @@ -121,12 +120,9 @@ async def handle_single_message(self, msg): affected_rows, id = await self.queue_add(msg.topic, msg.partition, msg.value) return affected_rows, id + @abstractmethod def _is_valid_event(self, value: bytes): - try: - Event.from_avro_bytes(value) - return True - except: - return False + raise Exception("Method not implemented") async def handle_message(self, consumer: AsyncIterator): """Message consumer. diff --git a/tests/test_networks/test_handler/test_event/test_dispatcher.py b/tests/test_networks/test_handler/test_event/test_dispatcher.py index 866f2070..b0c9a828 100644 --- a/tests/test_networks/test_handler/test_event/test_dispatcher.py +++ b/tests/test_networks/test_handler/test_event/test_dispatcher.py @@ -8,7 +8,7 @@ PostgresAsyncTestCase, ) from minos.networks import ( - MinosEventHandler, + MinosEventHandlerDispatcher, ) from minos.networks.exceptions import ( MinosNetworkException, @@ -23,11 +23,11 @@ class TestEventDispatcher(PostgresAsyncTestCase): CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" def test_from_config(self): - dispatcher = MinosEventHandler.from_config(config=self.config) - self.assertIsInstance(dispatcher, MinosEventHandler) + dispatcher = MinosEventHandlerDispatcher.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosEventHandlerDispatcher) async def test_if_queue_table_exists(self): - event_handler = MinosEventHandler.from_config(config=self.config) + event_handler = MinosEventHandlerDispatcher.from_config(config=self.config) await event_handler.setup() async with aiopg.connect(**self.events_queue_db) as connect: @@ -44,7 +44,7 @@ async def test_if_queue_table_exists(self): async def test_get_event_handler(self): model = NaiveAggregate(test_id=1, test=2, id=1, version=1) event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) - m = MinosEventHandler.from_config(config=self.config) + m = MinosEventHandlerDispatcher.from_config(config=self.config) cls = m.get_event_handler(topic="TicketAdded") result = await cls(topic="TicketAdded", event=event_instance) @@ -54,7 +54,7 @@ async def test_get_event_handler(self): async def test_non_implemented_action(self): model = NaiveAggregate(test_id=1, test=2, id=1, version=1) event_instance = Event(topic="NotExisting", model=model.classname, items=[]) - m = MinosEventHandler.from_config(config=self.config) + m = MinosEventHandlerDispatcher.from_config(config=self.config) with self.assertRaises(MinosNetworkException) as context: cls = m.get_event_handler(topic=event_instance.topic) @@ -66,12 +66,12 @@ async def test_non_implemented_action(self): ) async def test_none_config(self): - event_handler = MinosEventHandler.from_config(config=None) + event_handler = MinosEventHandlerDispatcher.from_config(config=None) self.assertIsNone(event_handler) async def test_event_queue_checker(self): - event_handler = MinosEventHandler.from_config(config=self.config) + event_handler = MinosEventHandlerDispatcher.from_config(config=self.config) await event_handler.setup() model = NaiveAggregate(test_id=1, test=2, id=1, version=1) @@ -93,7 +93,7 @@ async def test_event_queue_checker(self): assert queue_id[0] > 0 # Must get the record, call on_reply function and delete the record from DB - await event_handler.event_queue_checker() + await event_handler.queue_checker() async with aiopg.connect(**self.events_queue_db) as connect: async with connect.cursor() as cur: diff --git a/tests/test_networks/test_handler/test_event/test_event_services.py b/tests/test_networks/test_handler/test_event/test_event_services.py index cd86703a..64a7c198 100644 --- a/tests/test_networks/test_handler/test_event/test_event_services.py +++ b/tests/test_networks/test_handler/test_event/test_event_services.py @@ -45,8 +45,8 @@ async def test_start(self): async def test_callback(self): service = MinosEventPeriodicService(interval=1, loop=None, config=self.config) await service.dispatcher.setup() - mock = MagicMock(side_effect=service.dispatcher.event_queue_checker) - service.dispatcher.event_queue_checker = mock + mock = MagicMock(side_effect=service.dispatcher.queue_checker) + service.dispatcher.queue_checker = mock await service.callback() self.assertEqual(1, mock.call_count) await service.dispatcher.destroy() From eb2f824e5a7350867f981110f3057a652f8baee2 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 7 May 2021 09:49:32 +0200 Subject: [PATCH 220/239] Update libraries --- requirements_dev.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 34855353..2843d3a0 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,5 @@ aiokafka==0.7.0 -aiomisc==14.0.3 +aiomisc==12.1.0 aiopg==1.2.1 alabaster==0.7.12 appdirs==1.4.4 @@ -32,7 +32,7 @@ keyring==23.0.1 lmdb==1.1.1 MarkupSafe==1.1.1 mccabe==0.6.1 -minos-microservice-common==0.0.6 +minos-microservice-common==0.0.7 more-itertools==8.7.0 mypy-extensions==0.4.3 orjson==3.5.1 @@ -41,6 +41,7 @@ pathspec==0.8.1 peewee==3.14.4 pkginfo==1.7.0 pluggy==0.13.1 +psycopg2==2.8.6 psycopg2-binary==2.8.6 py==1.10.0 pycodestyle==2.7.0 From 3358fb5edb215693721ccd5f1b14f9b77106a9a1 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Fri, 7 May 2021 16:15:51 +0200 Subject: [PATCH 221/239] Update test_config.yml Add SAGA Config #117 --- tests/test_config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_config.yml b/tests/test_config.yml index 1e621a1e..54a4cbca 100644 --- a/tests/test_config.yml +++ b/tests/test_config.yml @@ -63,3 +63,10 @@ commands: port: 5432 records: 10 retry: 2 +saga: + - name: AddOrder + controller: minos.services.OrderService + action: add_order + - name: DeleteOrder + controller: minos.services.OrderService + action: delete_order From 0c9a083450020fdc5090dfe5ddd48a9c704be229 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Mon, 10 May 2021 19:41:20 +0200 Subject: [PATCH 222/239] Command, CommandReply and Event Add SAGA Config #117 Refactor EventHandler and create new package for handler #114 CommandHandler review #113 --- minos/networks/__init__.py | 9 ++ minos/networks/handler/__init__.py | 14 +++ minos/networks/handler/command/__init__.py | 18 +++ minos/networks/handler/command/dispatcher.py | 35 ++++++ minos/networks/handler/command/server.py | 36 ++++++ minos/networks/handler/command/services.py | 71 +++++++++++ .../handler/command_reply/__init__.py | 18 +++ .../handler/command_reply/dispatcher.py | 36 ++++++ .../networks/handler/command_reply/server.py | 36 ++++++ .../handler/command_reply/services.py | 71 +++++++++++ minos/networks/handler/dispatcher.py | 6 +- minos/networks/handler/event/dispatcher.py | 6 +- minos/networks/handler/event/server.py | 3 +- minos/networks/handler/event/services.py | 33 ++---- minos/networks/handler/server.py | 22 +++- tests/services/CommandTestService.py | 17 +++ tests/services/SagaTestService.py | 12 ++ tests/test_config.yml | 29 +++-- .../test_handler/test_command/__init__.py | 7 ++ .../test_command/test_command_server.py | 71 +++++++++++ .../test_command/test_command_services.py | 52 ++++++++ .../test_command/test_dispatcher.py | 103 ++++++++++++++++ .../test_command_reply/__init__.py | 7 ++ .../test_command__reply_server.py | 71 +++++++++++ .../test_command_reply_services.py | 51 ++++++++ .../test_command_reply/test_dispatcher.py | 111 ++++++++++++++++++ .../test_event/test_event_server.py | 35 ------ .../test_event/test_event_services.py | 2 +- 28 files changed, 902 insertions(+), 80 deletions(-) create mode 100644 minos/networks/handler/command/dispatcher.py create mode 100644 minos/networks/handler/command/server.py create mode 100644 minos/networks/handler/command/services.py create mode 100644 minos/networks/handler/command_reply/dispatcher.py create mode 100644 minos/networks/handler/command_reply/server.py create mode 100644 minos/networks/handler/command_reply/services.py create mode 100644 tests/services/CommandTestService.py create mode 100644 tests/services/SagaTestService.py create mode 100644 tests/test_networks/test_handler/test_command/__init__.py create mode 100644 tests/test_networks/test_handler/test_command/test_command_server.py create mode 100644 tests/test_networks/test_handler/test_command/test_command_services.py create mode 100644 tests/test_networks/test_handler/test_command/test_dispatcher.py create mode 100644 tests/test_networks/test_handler/test_command_reply/__init__.py create mode 100644 tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py create mode 100644 tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py create mode 100644 tests/test_networks/test_handler/test_command_reply/test_dispatcher.py diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 1e6a612d..eefc8dcb 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -18,6 +18,15 @@ MinosEventPeriodicService, MinosEventHandlerServer, MinosEventServerService, + MinosCommandHandlerDispatcher, + MinosCommandHandlerServer, + MinosCommandPeriodicService, + MinosCommandServerService, + MinosCommandReplyHandlerDispatcher, + MinosCommandReplyHandlerServer, + MinosCommandReplyPeriodicService, + MinosCommandReplyServerService, + ) from .exceptions import ( MinosNetworkException, diff --git a/minos/networks/handler/__init__.py b/minos/networks/handler/__init__.py index a4e38280..9b2b1f09 100644 --- a/minos/networks/handler/__init__.py +++ b/minos/networks/handler/__init__.py @@ -12,3 +12,17 @@ MinosEventPeriodicService, MinosEventServerService, ) + +from .command import ( + MinosCommandHandlerDispatcher, + MinosCommandHandlerServer, + MinosCommandPeriodicService, + MinosCommandServerService, +) + +from .command_reply import ( + MinosCommandReplyHandlerDispatcher, + MinosCommandReplyHandlerServer, + MinosCommandReplyPeriodicService, + MinosCommandReplyServerService, +) diff --git a/minos/networks/handler/command/__init__.py b/minos/networks/handler/command/__init__.py index e69de29b..3453e2ce 100644 --- a/minos/networks/handler/command/__init__.py +++ b/minos/networks/handler/command/__init__.py @@ -0,0 +1,18 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from .server import ( + MinosCommandHandlerServer, +) +from .services import ( + MinosCommandPeriodicService, + MinosCommandServerService, +) +from .dispatcher import ( + MinosCommandHandlerDispatcher, +) diff --git a/minos/networks/handler/command/dispatcher.py b/minos/networks/handler/command/dispatcher.py new file mode 100644 index 00000000..5792a791 --- /dev/null +++ b/minos/networks/handler/command/dispatcher.py @@ -0,0 +1,35 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from minos.common.configuration.config import ( + MinosConfig, +) +from typing import ( + Any, +) +from ..dispatcher import ( + MinosHandlerDispatcher, +) +from minos.common import ( + Command, +) + + +class MinosCommandHandlerDispatcher(MinosHandlerDispatcher): + + TABLE = "command_queue" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(table_name=self.TABLE, config=config.commands, **kwargs) + self._broker_group_name = f"event_{config.service.name}" + + def _is_valid_instance(self, value: bytes): + try: + instance = Command.from_avro_bytes(value) + return True, instance + except: + return False, None diff --git a/minos/networks/handler/command/server.py b/minos/networks/handler/command/server.py new file mode 100644 index 00000000..8265540b --- /dev/null +++ b/minos/networks/handler/command/server.py @@ -0,0 +1,36 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from minos.common.configuration.config import ( + MinosConfig, +) +from typing import ( + Any, +) +from ..server import ( + MinosHandlerServer, +) +from minos.common import ( + Command, +) + + +class MinosCommandHandlerServer(MinosHandlerServer): + + TABLE = "command_queue" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(table_name=self.TABLE, config=config.commands, **kwargs) + self._kafka_conn_data = f"{config.commands.broker.host}:{config.commands.broker.port}" + self._broker_group_name = f"event_{config.service.name}" + + def _is_valid_instance(self, value: bytes): + try: + Command.from_avro_bytes(value) + return True + except: + return False diff --git a/minos/networks/handler/command/services.py b/minos/networks/handler/command/services.py new file mode 100644 index 00000000..b101c3bb --- /dev/null +++ b/minos/networks/handler/command/services.py @@ -0,0 +1,71 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from typing import ( + Any, +) +from aiomisc.service.periodic import ( + PeriodicService, + Service, +) +from minos.common import ( + MinosConfig, +) +from .dispatcher import ( + MinosCommandHandlerDispatcher, +) +from .server import ( + MinosCommandHandlerServer, +) + + +class MinosCommandServerService(Service): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosCommandHandlerServer.from_config(config=config) + self.consumer = None + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await self.dispatcher.setup() + + self.consumer = await self.dispatcher.kafka_consumer(self.dispatcher._topics, self.dispatcher._broker_group_name, + self.dispatcher._kafka_conn_data) + await self.dispatcher.handle_message(self.consumer) + + async def stop(self, exception: Exception = None) -> Any: + if self.consumer is not None: + await self.consumer.stop() + + +class MinosCommandPeriodicService(PeriodicService): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosCommandHandlerDispatcher.from_config(config=config) + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await super().start() + await self.dispatcher.setup() + + async def callback(self) -> None: + """Method to be called periodically by the internal ``aiomisc`` logic. + + :return:This method does not return anything. + """ + await self.dispatcher.queue_checker() diff --git a/minos/networks/handler/command_reply/__init__.py b/minos/networks/handler/command_reply/__init__.py index e69de29b..05b9595a 100644 --- a/minos/networks/handler/command_reply/__init__.py +++ b/minos/networks/handler/command_reply/__init__.py @@ -0,0 +1,18 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from .server import ( + MinosCommandReplyHandlerServer, +) +from .services import ( + MinosCommandReplyPeriodicService, + MinosCommandReplyServerService, +) +from .dispatcher import ( + MinosCommandReplyHandlerDispatcher, +) diff --git a/minos/networks/handler/command_reply/dispatcher.py b/minos/networks/handler/command_reply/dispatcher.py new file mode 100644 index 00000000..a62ecf88 --- /dev/null +++ b/minos/networks/handler/command_reply/dispatcher.py @@ -0,0 +1,36 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. +import collections + +from minos.common.configuration.config import ( + MinosConfig, +) +from typing import ( + Any, +) +from ..dispatcher import ( + MinosHandlerDispatcher, +) +from minos.common import ( + CommandReply, +) + + +class MinosCommandReplyHandlerDispatcher(MinosHandlerDispatcher): + + TABLE = "command_reply_queue" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(table_name=self.TABLE, config=config.saga, **kwargs) + self._broker_group_name = f"event_{config.service.name}" + + def _is_valid_instance(self, value: bytes): + try: + instance = CommandReply.from_avro_bytes(value) + return True, instance + except: + return False, None diff --git a/minos/networks/handler/command_reply/server.py b/minos/networks/handler/command_reply/server.py new file mode 100644 index 00000000..25ee9e30 --- /dev/null +++ b/minos/networks/handler/command_reply/server.py @@ -0,0 +1,36 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from minos.common.configuration.config import ( + MinosConfig, +) +from typing import ( + Any, +) +from ..server import ( + MinosHandlerServer, +) +from minos.common import ( + CommandReply, +) + + +class MinosCommandReplyHandlerServer(MinosHandlerServer): + + TABLE = "command_reply_queue" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(table_name=self.TABLE, config=config.saga, **kwargs) + self._kafka_conn_data = f"{config.commands.broker.host}:{config.commands.broker.port}" + self._broker_group_name = f"event_{config.service.name}" + + def _is_valid_instance(self, value: bytes): + try: + CommandReply.from_avro_bytes(value) + return True + except: + return False diff --git a/minos/networks/handler/command_reply/services.py b/minos/networks/handler/command_reply/services.py new file mode 100644 index 00000000..ba763755 --- /dev/null +++ b/minos/networks/handler/command_reply/services.py @@ -0,0 +1,71 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from typing import ( + Any, +) +from aiomisc.service.periodic import ( + PeriodicService, + Service, +) +from minos.common import ( + MinosConfig, +) +from .dispatcher import ( + MinosCommandReplyHandlerDispatcher, +) +from .server import ( + MinosCommandReplyHandlerServer, +) + + +class MinosCommandReplyServerService(Service): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosCommandReplyHandlerServer.from_config(config=config) + self.consumer = None + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await self.dispatcher.setup() + + self.consumer = await self.dispatcher.kafka_consumer(self.dispatcher._topics, self.dispatcher._broker_group_name, + self.dispatcher._kafka_conn_data) + await self.dispatcher.handle_message(self.consumer) + + async def stop(self, exception: Exception = None) -> Any: + if self.consumer is not None: + await self.consumer.stop() + + +class MinosCommandReplyPeriodicService(PeriodicService): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosCommandReplyHandlerDispatcher.from_config(config=config) + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await super().start() + await self.dispatcher.setup() + + async def callback(self) -> None: + """Method to be called periodically by the internal ``aiomisc`` logic. + + :return:This method does not return anything. + """ + await self.dispatcher.queue_checker() diff --git a/minos/networks/handler/dispatcher.py b/minos/networks/handler/dispatcher.py index 2a03e110..95a42772 100644 --- a/minos/networks/handler/dispatcher.py +++ b/minos/networks/handler/dispatcher.py @@ -129,10 +129,10 @@ async def queue_checker(self) -> NoReturn: call_ok = False try: reply_on = self.get_event_handler(topic=row[1]) - valid_instance, event_instance = self._is_valid_event(row[3]) + valid_instance, instance = self._is_valid_instance(row[3]) if not valid_instance: return - await reply_on(topic=row[1], event=event_instance) + await reply_on(row[1], instance) call_ok = True finally: if call_ok: @@ -141,5 +141,5 @@ async def queue_checker(self) -> NoReturn: await cur2.execute("DELETE FROM %s WHERE id=%d;" % (self._table_name, row[0])) @abstractmethod - def _is_valid_event(self, value: bytes): + def _is_valid_instance(self, value: bytes): raise Exception("Method not implemented") diff --git a/minos/networks/handler/event/dispatcher.py b/minos/networks/handler/event/dispatcher.py index f635552a..fabe051c 100644 --- a/minos/networks/handler/event/dispatcher.py +++ b/minos/networks/handler/event/dispatcher.py @@ -27,9 +27,9 @@ def __init__(self, *, config: MinosConfig, **kwargs: Any): super().__init__(table_name=self.TABLE, config=config.events, **kwargs) self._broker_group_name = f"event_{config.service.name}" - def _is_valid_event(self, value: bytes): + def _is_valid_instance(self, value: bytes): try: - event_instance = Event.from_avro_bytes(value) - return True, event_instance + instance = Event.from_avro_bytes(value) + return True, instance except: return False, None diff --git a/minos/networks/handler/event/server.py b/minos/networks/handler/event/server.py index fce63571..8030bc8c 100644 --- a/minos/networks/handler/event/server.py +++ b/minos/networks/handler/event/server.py @@ -25,9 +25,10 @@ class MinosEventHandlerServer(MinosHandlerServer): def __init__(self, *, config: MinosConfig, **kwargs: Any): super().__init__(table_name=self.TABLE, config=config.events, **kwargs) + self._kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" self._broker_group_name = f"event_{config.service.name}" - def _is_valid_event(self, value: bytes): + def _is_valid_instance(self, value: bytes): try: Event.from_avro_bytes(value) return True diff --git a/minos/networks/handler/event/services.py b/minos/networks/handler/event/services.py index e75490f6..39b13710 100644 --- a/minos/networks/handler/event/services.py +++ b/minos/networks/handler/event/services.py @@ -1,10 +1,14 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + from typing import ( Any, ) - -from aiokafka import ( - AIOKafkaConsumer, -) from aiomisc.service.periodic import ( PeriodicService, Service, @@ -29,13 +33,6 @@ def __init__(self, config: MinosConfig = None, **kwargs): self.dispatcher = MinosEventHandlerServer.from_config(config=config) self.consumer = None - """ - def create_task(self, coro: Awaitable[Any]): - task = self.loop.create_task(coro) - self.dispatcher._tasks.add(task) - task.add_done_callback(self.dispatcher._tasks.remove) - """ - async def start(self) -> None: """Method to be called at the startup by the internal ``aiomisc`` loigc. @@ -43,18 +40,8 @@ async def start(self) -> None: """ await self.dispatcher.setup() - # start the Service Event Consumer for Kafka - self.consumer = AIOKafkaConsumer( - group_id=self.dispatcher._broker_group_name, - auto_offset_reset="latest", - bootstrap_servers=self.dispatcher._kafka_conn_data, - ) - - await self.consumer.start() - self.consumer.subscribe(self.dispatcher._topics) - - # self.create_task(self.dispatcher.handle_message(self.consumer)) - + self.consumer = await self.dispatcher.kafka_consumer(self.dispatcher._topics, self.dispatcher._broker_group_name, + self.dispatcher._kafka_conn_data) await self.dispatcher.handle_message(self.consumer) async def stop(self, exception: Exception = None) -> Any: diff --git a/minos/networks/handler/server.py b/minos/networks/handler/server.py index 8d6e275c..727b5f30 100644 --- a/minos/networks/handler/server.py +++ b/minos/networks/handler/server.py @@ -11,6 +11,7 @@ from abc import abstractmethod +from aiokafka import AIOKafkaConsumer from psycopg2.extensions import AsIs import asyncio import datetime @@ -38,7 +39,7 @@ class MinosHandlerServer(MinosHandlerSetup): """ - __slots__ = "_tasks", "_db_dsn", "_handlers", "_topics", "_kafka_conn_data", "_broker_group_name" + __slots__ = "_tasks", "_db_dsn", "_handlers", "_topics", "_broker_group_name" def __init__(self, *, table_name: str, config: NamedTuple, **kwargs: Any): super().__init__(table_name=table_name, **kwargs, **config.queue._asdict()) @@ -51,7 +52,6 @@ def __init__(self, *, table_name: str, config: NamedTuple, **kwargs: Any): item.name: {"controller": item.controller, "action": item.action} for item in config.items } self._topics = list(self._handler.keys()) - self._kafka_conn_data = f"{config.broker.host}:{config.broker.port}" self._table_name = table_name @classmethod @@ -115,13 +115,13 @@ async def handle_single_message(self, msg): """ # the handler receive a message and store in the queue database # check if the event binary string is well formatted - if not self._is_valid_event(msg.value): + if not self._is_valid_instance(msg.value): return affected_rows, id = await self.queue_add(msg.topic, msg.partition, msg.value) return affected_rows, id @abstractmethod - def _is_valid_event(self, value: bytes): + def _is_valid_instance(self, value: bytes): raise Exception("Method not implemented") async def handle_message(self, consumer: AsyncIterator): @@ -135,3 +135,17 @@ async def handle_message(self, consumer: AsyncIterator): async for msg in consumer: await self.handle_single_message(msg) + + @staticmethod + async def kafka_consumer(topics: list, group_name: str, conn: str): + # start the Service Event Consumer for Kafka + consumer = AIOKafkaConsumer( + group_id=group_name, + auto_offset_reset="latest", + bootstrap_servers=conn, + ) + + await consumer.start() + consumer.subscribe(topics) + + return consumer diff --git a/tests/services/CommandTestService.py b/tests/services/CommandTestService.py new file mode 100644 index 00000000..be12d04c --- /dev/null +++ b/tests/services/CommandTestService.py @@ -0,0 +1,17 @@ +from minos.common import ( + Command, +) + + +class CommandService(object): + async def get_order(self, topic: str, command: Command): + return "get_order" + + async def add_order(self, topic: str, command: Command): + return "add_order" + + async def delete_order(self, topic: str, command: Command): + return "delete_order" + + async def update_order(self, topic: str, command: Command): + return "update_order" diff --git a/tests/services/SagaTestService.py b/tests/services/SagaTestService.py new file mode 100644 index 00000000..12bec134 --- /dev/null +++ b/tests/services/SagaTestService.py @@ -0,0 +1,12 @@ +from minos.common import ( + CommandReply, +) + + +class SagaService(object): + async def add_order(self, topic: str, command: CommandReply): + return "add_order_saga" + + async def delete_order(self, topic: str, command: CommandReply): + return "delete_order_saga" + diff --git a/tests/test_config.yml b/tests/test_config.yml index 54a4cbca..345c73ca 100644 --- a/tests/test_config.yml +++ b/tests/test_config.yml @@ -44,16 +44,16 @@ commands: port: 9092 items: - name: AddOrder - controller: minos.services.OrderService + controller: tests.services.CommandTestService.CommandService action: add_order - name: DeleteOrder - controller: minos.services.OrderService + controller: tests.services.CommandTestService.CommandService action: delete_order - name: UpdateOrder - controller: minos.services.OrderService + controller: tests.services.CommandTestService.CommandService action: update_order - name: GetOrder - controller: minos.service.OrderService + controller: tests.service.CommandTestService.CommandService action: get_order queue: database: order_db @@ -64,9 +64,18 @@ commands: records: 10 retry: 2 saga: - - name: AddOrder - controller: minos.services.OrderService - action: add_order - - name: DeleteOrder - controller: minos.services.OrderService - action: delete_order + items: + - name: AddOrder + controller: tests.services.SagaTestService.SagaService + action: add_order + - name: DeleteOrder + controller: tests.services.SagaTestService.SagaService + action: delete_order + queue: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 + records: 10 + retry: 2 diff --git a/tests/test_networks/test_handler/test_command/__init__.py b/tests/test_networks/test_handler/test_command/__init__.py new file mode 100644 index 00000000..fed3716d --- /dev/null +++ b/tests/test_networks/test_handler/test_command/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" diff --git a/tests/test_networks/test_handler/test_command/test_command_server.py b/tests/test_networks/test_handler/test_command/test_command_server.py new file mode 100644 index 00000000..510cad1d --- /dev/null +++ b/tests/test_networks/test_handler/test_command/test_command_server.py @@ -0,0 +1,71 @@ +from collections import ( + namedtuple, +) +from minos.common import ( + Command, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandHandlerServer, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) + + +class TestCommandServer(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosCommandHandlerServer.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosCommandHandlerServer) + + async def test_none_config(self): + event_server = MinosCommandHandlerServer.from_config(config=None) + + self.assertIsNone(event_server) + + async def test_queue_add(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Command(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + bin_data = event_instance.avro_bytes + Command.from_avro_bytes(bin_data) + + event_server = MinosCommandHandlerServer.from_config(config=self.config) + await event_server.setup() + + affected_rows, id = await event_server.queue_add(topic=event_instance.topic, partition=0, binary=bin_data) + + assert affected_rows == 1 + assert id > 0 + + async def test_handle_message(self): + event_server = MinosCommandHandlerServer.from_config(config=self.config) + await event_server.setup() + + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Command(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + bin_data = event_instance.avro_bytes + + Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) + + async def consumer(): + yield Mensaje(topic="TicketAdded", partition=0, value=bin_data) + + await event_server.handle_message(consumer()) + + async def test_handle_message_ko(self): + event_server = MinosCommandHandlerServer.from_config(config=self.config) + await event_server.setup() + + bin_data = bytes(b"test") + + Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) + + async def consumer(): + yield Mensaje(topic="TicketAdded", partition=0, value=bin_data) + + await event_server.handle_message(consumer()) diff --git a/tests/test_networks/test_handler/test_command/test_command_services.py b/tests/test_networks/test_handler/test_command/test_command_services.py new file mode 100644 index 00000000..c8eaea79 --- /dev/null +++ b/tests/test_networks/test_handler/test_command/test_command_services.py @@ -0,0 +1,52 @@ +import unittest +from unittest.mock import ( + MagicMock, +) + +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandPeriodicService, + MinosCommandServerService, +) +from tests.utils import ( + BASE_PATH, +) + + +class TestMinosCommandServices(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_start(self): + service = MinosCommandServerService(loop=None, config=self.config) + + async def _fn(consumer): + self.assertEqual(service.consumer, consumer) + + mock = MagicMock(side_effect=_fn) + service.dispatcher.handle_message = mock + await service.start() + self.assertTrue(1, mock.call_count) + await service.stop() + + +class TestMinosQueueService(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_start(self): + service = MinosCommandPeriodicService(interval=1, loop=None, config=self.config) + mock = MagicMock(side_effect=service.dispatcher.setup) + service.dispatcher.setup = mock + await service.start() + self.assertTrue(1, mock.call_count) + await service.stop() + + async def test_callback(self): + service = MinosCommandPeriodicService(interval=1, loop=None, config=self.config) + await service.dispatcher.setup() + mock = MagicMock(side_effect=service.dispatcher.queue_checker) + service.dispatcher.queue_checker = mock + await service.callback() + self.assertEqual(1, mock.call_count) + await service.dispatcher.destroy() diff --git a/tests/test_networks/test_handler/test_command/test_dispatcher.py b/tests/test_networks/test_handler/test_command/test_dispatcher.py new file mode 100644 index 00000000..529b6271 --- /dev/null +++ b/tests/test_networks/test_handler/test_command/test_dispatcher.py @@ -0,0 +1,103 @@ +import datetime + +import aiopg +from minos.common import ( + Command, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandHandlerDispatcher, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) + + +class TestCommandDispatcher(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosCommandHandlerDispatcher.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosCommandHandlerDispatcher) + + async def test_if_queue_table_exists(self): + handler = MinosCommandHandlerDispatcher.from_config(config=self.config) + await handler.setup() + + async with aiopg.connect(**self.commands_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'command_queue';" + ) + ret = [] + async for row in cur: + ret.append(row) + + assert ret == [(1,)] + + async def test_get_event_handler(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Command(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + m = MinosCommandHandlerDispatcher.from_config(config=self.config) + + cls = m.get_event_handler(topic=event_instance.topic) + result = await cls(topic=event_instance.topic, command=event_instance) + + assert result == "add_order" + + async def test_non_implemented_action(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + instance = Command(topic="NotExisting", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + m = MinosCommandHandlerDispatcher.from_config(config=self.config) + + with self.assertRaises(MinosNetworkException) as context: + cls = m.get_event_handler(topic=instance.topic) + await cls(topic=instance.topic, command=instance) + + self.assertTrue( + "topic NotExisting have no controller/action configured, please review th configuration file" + in str(context.exception) + ) + + async def test_none_config(self): + handler = MinosCommandHandlerDispatcher.from_config(config=None) + + self.assertIsNone(handler) + + async def test_event_queue_checker(self): + handler = MinosCommandHandlerDispatcher.from_config(config=self.config) + await handler.setup() + + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + instance = Command(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + bin_data = instance.avro_bytes + Command.from_avro_bytes(bin_data) + + async with aiopg.connect(**self.commands_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO command_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + (instance.topic, 0, bin_data, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + assert affected_rows == 1 + assert queue_id[0] > 0 + + # Must get the record, call on_reply function and delete the record from DB + await handler.queue_checker() + + async with aiopg.connect(**self.commands_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM command_queue WHERE id=%d" % (queue_id)) + records = await cur.fetchone() + + assert records[0] == 0 diff --git a/tests/test_networks/test_handler/test_command_reply/__init__.py b/tests/test_networks/test_handler/test_command_reply/__init__.py new file mode 100644 index 00000000..fed3716d --- /dev/null +++ b/tests/test_networks/test_handler/test_command_reply/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" diff --git a/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py b/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py new file mode 100644 index 00000000..14c06547 --- /dev/null +++ b/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py @@ -0,0 +1,71 @@ +from collections import ( + namedtuple, +) +from minos.common import ( + CommandReply, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandReplyHandlerServer, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) + + +class TestCommandReplyServer(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosCommandReplyHandlerServer.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosCommandReplyHandlerServer) + + async def test_none_config(self): + event_server = MinosCommandReplyHandlerServer.from_config(config=None) + + self.assertIsNone(event_server) + + async def test_queue_add(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = CommandReply(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + bin_data = event_instance.avro_bytes + CommandReply.from_avro_bytes(bin_data) + + event_server = MinosCommandReplyHandlerServer.from_config(config=self.config) + await event_server.setup() + + affected_rows, id = await event_server.queue_add(topic=event_instance.topic, partition=0, binary=bin_data) + + assert affected_rows == 1 + assert id > 0 + + async def test_handle_message(self): + event_server = MinosCommandReplyHandlerServer.from_config(config=self.config) + await event_server.setup() + + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = CommandReply(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + bin_data = event_instance.avro_bytes + + Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) + + async def consumer(): + yield Mensaje(topic="AddOrder", partition=0, value=bin_data) + + await event_server.handle_message(consumer()) + + async def test_handle_message_ko(self): + event_server = MinosCommandReplyHandlerServer.from_config(config=self.config) + await event_server.setup() + + bin_data = bytes(b"test") + + Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) + + async def consumer(): + yield Mensaje(topic="AddOrder", partition=0, value=bin_data) + + await event_server.handle_message(consumer()) diff --git a/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py b/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py new file mode 100644 index 00000000..a193743d --- /dev/null +++ b/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py @@ -0,0 +1,51 @@ +from unittest.mock import ( + MagicMock, +) + +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandReplyPeriodicService, + MinosCommandReplyServerService, +) +from tests.utils import ( + BASE_PATH, +) + + +class TestMinosCommandReplyServices(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_start(self): + service = MinosCommandReplyServerService(loop=None, config=self.config) + + async def _fn(consumer): + self.assertEqual(service.consumer, consumer) + + mock = MagicMock(side_effect=_fn) + service.dispatcher.handle_message = mock + await service.start() + self.assertTrue(1, mock.call_count) + await service.stop() + + +class TestMinosCommandReplyQueueService(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_start(self): + service = MinosCommandReplyPeriodicService(interval=1, loop=None, config=self.config) + mock = MagicMock(side_effect=service.dispatcher.setup) + service.dispatcher.setup = mock + await service.start() + self.assertTrue(1, mock.call_count) + await service.stop() + + async def test_callback(self): + service = MinosCommandReplyPeriodicService(interval=1, loop=None, config=self.config) + await service.dispatcher.setup() + mock = MagicMock(side_effect=service.dispatcher.queue_checker) + service.dispatcher.queue_checker = mock + await service.callback() + self.assertEqual(1, mock.call_count) + await service.dispatcher.destroy() diff --git a/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py b/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py new file mode 100644 index 00000000..b268602e --- /dev/null +++ b/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py @@ -0,0 +1,111 @@ +import datetime + +import aiopg +from minos.common import ( + CommandReply, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandReplyHandlerDispatcher, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) + + +class TestCommandReplyDispatcher(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosCommandReplyHandlerDispatcher) + + async def test_if_queue_table_exists(self): + handler = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) + await handler.setup() + self._meta_saga_queue_db = self._config.saga.queue._asdict() + self._meta_saga_queue_db.pop("records") + self._meta_saga_queue_db.pop("retry") + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'command_reply_queue';" + ) + ret = [] + async for row in cur: + ret.append(row) + + assert ret == [(1,)] + + async def test_get_event_handler(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = CommandReply(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + m = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) + + cls = m.get_event_handler(topic=event_instance.topic) + result = await cls(topic=event_instance.topic, command=event_instance) + + assert result == "add_order_saga" + + async def test_non_implemented_action(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + instance = CommandReply(topic="NotExisting", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + m = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) + + with self.assertRaises(MinosNetworkException) as context: + cls = m.get_event_handler(topic=instance.topic) + await cls(topic=instance.topic, command=instance) + + self.assertTrue( + "topic NotExisting have no controller/action configured, please review th configuration file" + in str(context.exception) + ) + + async def test_none_config(self): + handler = MinosCommandReplyHandlerDispatcher.from_config(config=None) + + self.assertIsNone(handler) + + async def test_event_queue_checker(self): + handler = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) + await handler.setup() + + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + instance = CommandReply(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + bin_data = instance.avro_bytes + CommandReply.from_avro_bytes(bin_data) + + self._meta_saga_queue_db = self._config.saga.queue._asdict() + self._meta_saga_queue_db.pop("records") + self._meta_saga_queue_db.pop("retry") + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO command_reply_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + (instance.topic, 0, bin_data, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + assert affected_rows == 1 + assert queue_id[0] > 0 + + # Must get the record, call on_reply function and delete the record from DB + await handler.queue_checker() + + self._meta_saga_queue_db = self._config.saga.queue._asdict() + self._meta_saga_queue_db.pop("records") + self._meta_saga_queue_db.pop("retry") + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM command_reply_queue WHERE id=%d" % (queue_id)) + records = await cur.fetchone() + + assert records[0] == 0 diff --git a/tests/test_networks/test_handler/test_event/test_event_server.py b/tests/test_networks/test_handler/test_event/test_event_server.py index ac70425c..f152947e 100644 --- a/tests/test_networks/test_handler/test_event/test_event_server.py +++ b/tests/test_networks/test_handler/test_event/test_event_server.py @@ -1,12 +1,6 @@ -import time from collections import ( namedtuple, ) - -from aiokafka import ( - AIOKafkaConsumer, - AIOKafkaProducer, -) from minos.common import ( Event, ) @@ -22,35 +16,6 @@ ) -async def kafka_producer(config): - - # basic_config( - # level=logging.INFO, - # buffered=True, - # log_format='color', - # flush_interval=2 - # ) - kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" - producer = AIOKafkaProducer(bootstrap_servers=kafka_conn_data) - # Get cluster layout and topic/partition allocation - await producer.start() - # Produce messages - - model = NaiveAggregate(test_id=1, test=2, id=1, version=1) - event_instance = Event(topic="TicketAdded", model=model.classname, items=[model]) - bin_data = event_instance.avro_bytes - - model2 = NaiveAggregate(test_id=2, test=2, id=1, version=1) - event_instance_2 = Event(topic="TicketDeleted", model=model2.classname, items=[model2]) - bin_data2 = event_instance_2.avro_bytes - - await producer.send_and_wait(event_instance.topic, bin_data) - time.sleep(1) - await producer.send_and_wait(event_instance_2.topic, bin_data2) - time.sleep(1) - await producer.stop() - - class TestEventServer(PostgresAsyncTestCase): CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" diff --git a/tests/test_networks/test_handler/test_event/test_event_services.py b/tests/test_networks/test_handler/test_event/test_event_services.py index 64a7c198..54651f1e 100644 --- a/tests/test_networks/test_handler/test_event/test_event_services.py +++ b/tests/test_networks/test_handler/test_event/test_event_services.py @@ -15,7 +15,7 @@ ) -class TestMinosEventServer(PostgresAsyncTestCase): +class TestMinosEventServices(PostgresAsyncTestCase): CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" async def test_start(self): From dc73a3f8ded1c5833ee00073914ff3f248abde07 Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Mon, 10 May 2021 23:15:10 +0200 Subject: [PATCH 223/239] More tests EventHandler Refactor code and add more tests #87 --- minos/networks/handler/dispatcher.py | 2 +- minos/networks/handler/server.py | 2 +- .../test_command/test_dispatcher.py | 29 +++++++++++ .../test_command_reply/test_dispatcher.py | 49 +++++++++++++++++-- .../test_event/test_dispatcher.py | 30 ++++++++++++ 5 files changed, 106 insertions(+), 6 deletions(-) diff --git a/minos/networks/handler/dispatcher.py b/minos/networks/handler/dispatcher.py index 95a42772..d557cdc2 100644 --- a/minos/networks/handler/dispatcher.py +++ b/minos/networks/handler/dispatcher.py @@ -141,5 +141,5 @@ async def queue_checker(self) -> NoReturn: await cur2.execute("DELETE FROM %s WHERE id=%d;" % (self._table_name, row[0])) @abstractmethod - def _is_valid_instance(self, value: bytes): + def _is_valid_instance(self, value: bytes): # pragma: no cover raise Exception("Method not implemented") diff --git a/minos/networks/handler/server.py b/minos/networks/handler/server.py index 727b5f30..c0f29776 100644 --- a/minos/networks/handler/server.py +++ b/minos/networks/handler/server.py @@ -121,7 +121,7 @@ async def handle_single_message(self, msg): return affected_rows, id @abstractmethod - def _is_valid_instance(self, value: bytes): + def _is_valid_instance(self, value: bytes): # pragma: no cover raise Exception("Method not implemented") async def handle_message(self, consumer: AsyncIterator): diff --git a/tests/test_networks/test_handler/test_command/test_dispatcher.py b/tests/test_networks/test_handler/test_command/test_dispatcher.py index 529b6271..30622eb3 100644 --- a/tests/test_networks/test_handler/test_command/test_dispatcher.py +++ b/tests/test_networks/test_handler/test_command/test_dispatcher.py @@ -101,3 +101,32 @@ async def test_event_queue_checker(self): records = await cur.fetchone() assert records[0] == 0 + + async def test_event_queue_checker_wrong_event(self): + handler = MinosCommandHandlerDispatcher.from_config(config=self.config) + await handler.setup() + bin_data = bytes(b'Test') + + + async with aiopg.connect(**self.commands_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO command_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + ("AddOrder", 0, bin_data, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + assert affected_rows == 1 + assert queue_id[0] > 0 + + # Must get the record, call on_reply function and delete the record from DB + await handler.queue_checker() + + async with aiopg.connect(**self.commands_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM command_queue WHERE id=%d" % (queue_id)) + records = await cur.fetchone() + + assert records[0] == 1 diff --git a/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py b/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py index b268602e..8f657bf8 100644 --- a/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py +++ b/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py @@ -73,17 +73,22 @@ async def test_none_config(self): self.assertIsNone(handler) async def test_event_queue_checker(self): + self._meta_saga_queue_db = self._config.saga.queue._asdict() + self._meta_saga_queue_db.pop("records") + self._meta_saga_queue_db.pop("retry") + handler = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) await handler.setup() + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("DELETE FROM {table};".format(table=MinosCommandReplyHandlerDispatcher.TABLE)) + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) instance = CommandReply(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") bin_data = instance.avro_bytes CommandReply.from_avro_bytes(bin_data) - self._meta_saga_queue_db = self._config.saga.queue._asdict() - self._meta_saga_queue_db.pop("records") - self._meta_saga_queue_db.pop("retry") async with aiopg.connect(**self._meta_saga_queue_db) as connect: async with connect.cursor() as cur: await cur.execute( @@ -100,12 +105,48 @@ async def test_event_queue_checker(self): # Must get the record, call on_reply function and delete the record from DB await handler.queue_checker() + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM command_reply_queue WHERE id=%d" % (queue_id)) + records = await cur.fetchone() + + assert records[0] == 0 + + async def test_command_reply_queue_checker_wrong_event(self): self._meta_saga_queue_db = self._config.saga.queue._asdict() self._meta_saga_queue_db.pop("records") self._meta_saga_queue_db.pop("retry") + + handler = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) + await handler.setup() + + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("DELETE FROM {table};".format(table=MinosCommandReplyHandlerDispatcher.TABLE)) + + bin_data = bytes(b'Test') + + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO command_reply_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + ("AddOrder", 0, bin_data, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + assert affected_rows == 1 + assert queue_id[0] > 0 + + # Must get the record, call on_reply function and delete the record from DB + await handler.queue_checker() + async with aiopg.connect(**self._meta_saga_queue_db) as connect: async with connect.cursor() as cur: await cur.execute("SELECT COUNT(*) FROM command_reply_queue WHERE id=%d" % (queue_id)) records = await cur.fetchone() - assert records[0] == 0 + + + assert records[0] == 1 diff --git a/tests/test_networks/test_handler/test_event/test_dispatcher.py b/tests/test_networks/test_handler/test_event/test_dispatcher.py index b0c9a828..29df227b 100644 --- a/tests/test_networks/test_handler/test_event/test_dispatcher.py +++ b/tests/test_networks/test_handler/test_event/test_dispatcher.py @@ -101,3 +101,33 @@ async def test_event_queue_checker(self): records = await cur.fetchone() assert records[0] == 0 + + async def test_event_queue_checker_wrong_event(self): + handler = MinosEventHandlerDispatcher.from_config(config=self.config) + await handler.setup() + bin_data = bytes(b'Test') + + + async with aiopg.connect(**self.events_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO event_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + ("TicketAdded", 0, bin_data, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + assert affected_rows == 1 + assert queue_id[0] > 0 + + # Must get the record, call on_reply function and delete the record from DB + await handler.queue_checker() + + async with aiopg.connect(**self.events_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM event_queue WHERE id=%d" % (queue_id)) + records = await cur.fetchone() + + assert records[0] == 1 + From f464ff7aa40cb32c3a01a105ecb511f8b4f6cd5f Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Mon, 10 May 2021 23:18:11 +0200 Subject: [PATCH 224/239] Update requirements_dev.txt --- requirements_dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 2843d3a0..0323040a 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -32,7 +32,7 @@ keyring==23.0.1 lmdb==1.1.1 MarkupSafe==1.1.1 mccabe==0.6.1 -minos-microservice-common==0.0.7 +minos-microservice-common==0.0.9 more-itertools==8.7.0 mypy-extensions==0.4.3 orjson==3.5.1 From f7eef7ee081b1f25e769eb18a2ac9ca27cf0700d Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Mon, 10 May 2021 23:26:46 +0200 Subject: [PATCH 225/239] Update test_config.yml --- tests/test_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_config.yml b/tests/test_config.yml index 345c73ca..fb2f609f 100644 --- a/tests/test_config.yml +++ b/tests/test_config.yml @@ -75,7 +75,7 @@ saga: database: order_db user: minos password: min0s - host: localhost + host: postgres port: 5432 records: 10 retry: 2 From a83fac89b0e5b53187e74a930944604b6e5bfb7b Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 10 May 2021 21:27:02 +0000 Subject: [PATCH 226/239] Restyled by black --- minos/networks/__init__.py | 1 - minos/networks/broker/commands.py | 8 +--- minos/networks/handler/abc.py | 16 ++----- minos/networks/handler/command/__init__.py | 8 +--- minos/networks/handler/command/dispatcher.py | 16 ++----- minos/networks/handler/command/server.py | 16 ++----- minos/networks/handler/command/services.py | 21 +++------ .../handler/command_reply/__init__.py | 8 +--- .../handler/command_reply/dispatcher.py | 16 ++----- .../networks/handler/command_reply/server.py | 16 ++----- .../handler/command_reply/services.py | 21 +++------ minos/networks/handler/dispatcher.py | 28 +++-------- minos/networks/handler/event/__init__.py | 8 +--- minos/networks/handler/event/dispatcher.py | 16 ++----- minos/networks/handler/event/server.py | 16 ++----- minos/networks/handler/event/services.py | 21 +++------ minos/networks/handler/server.py | 23 ++------- tests/services/CommandTestService.py | 4 +- tests/services/SagaTestService.py | 5 +- .../test_networks/test_broker/test_command.py | 32 +++++++++---- .../test_command/test_command_server.py | 34 ++++++++------ .../test_command/test_command_services.py | 12 ++--- .../test_command/test_dispatcher.py | 46 +++++++++++------- .../test_command__reply_server.py | 34 ++++++++------ .../test_command_reply_services.py | 12 ++--- .../test_command_reply/test_dispatcher.py | 47 ++++++++++++------- .../test_event/test_dispatcher.py | 20 ++------ .../test_event/test_event_server.py | 16 ++----- .../test_event/test_event_services.py | 12 ++--- 29 files changed, 210 insertions(+), 323 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index eefc8dcb..2598bb3e 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -26,7 +26,6 @@ MinosCommandReplyHandlerServer, MinosCommandReplyPeriodicService, MinosCommandReplyServerService, - ) from .exceptions import ( MinosNetworkException, diff --git a/minos/networks/broker/commands.py b/minos/networks/broker/commands.py index 0357893c..d3a98485 100644 --- a/minos/networks/broker/commands.py +++ b/minos/networks/broker/commands.py @@ -5,9 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import ( - annotations, -) +from __future__ import annotations from typing import ( NoReturn, @@ -20,9 +18,7 @@ MinosConfig, ) -from .abc import ( - MinosBroker, -) +from .abc import MinosBroker class MinosCommandBroker(MinosBroker): diff --git a/minos/networks/handler/abc.py b/minos/networks/handler/abc.py index 4b6d37a7..d184e4e6 100644 --- a/minos/networks/handler/abc.py +++ b/minos/networks/handler/abc.py @@ -5,20 +5,12 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ( - ABC, -) -from datetime import ( - datetime, -) -from typing import ( - NoReturn, -) +from abc import ABC +from datetime import datetime +from typing import NoReturn import aiopg -from minos.common import ( - MinosSetup, -) +from minos.common import MinosSetup class MinosHandlerSetup(MinosSetup): diff --git a/minos/networks/handler/command/__init__.py b/minos/networks/handler/command/__init__.py index 3453e2ce..d4a6c6ca 100644 --- a/minos/networks/handler/command/__init__.py +++ b/minos/networks/handler/command/__init__.py @@ -6,13 +6,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .server import ( - MinosCommandHandlerServer, -) +from .server import MinosCommandHandlerServer from .services import ( MinosCommandPeriodicService, MinosCommandServerService, ) -from .dispatcher import ( - MinosCommandHandlerDispatcher, -) +from .dispatcher import MinosCommandHandlerDispatcher diff --git a/minos/networks/handler/command/dispatcher.py b/minos/networks/handler/command/dispatcher.py index 5792a791..4e76b2ed 100644 --- a/minos/networks/handler/command/dispatcher.py +++ b/minos/networks/handler/command/dispatcher.py @@ -5,18 +5,10 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from minos.common.configuration.config import ( - MinosConfig, -) -from typing import ( - Any, -) -from ..dispatcher import ( - MinosHandlerDispatcher, -) -from minos.common import ( - Command, -) +from minos.common.configuration.config import MinosConfig +from typing import Any +from ..dispatcher import MinosHandlerDispatcher +from minos.common import Command class MinosCommandHandlerDispatcher(MinosHandlerDispatcher): diff --git a/minos/networks/handler/command/server.py b/minos/networks/handler/command/server.py index 8265540b..a9b30bd8 100644 --- a/minos/networks/handler/command/server.py +++ b/minos/networks/handler/command/server.py @@ -5,18 +5,10 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from minos.common.configuration.config import ( - MinosConfig, -) -from typing import ( - Any, -) -from ..server import ( - MinosHandlerServer, -) -from minos.common import ( - Command, -) +from minos.common.configuration.config import MinosConfig +from typing import Any +from ..server import MinosHandlerServer +from minos.common import Command class MinosCommandHandlerServer(MinosHandlerServer): diff --git a/minos/networks/handler/command/services.py b/minos/networks/handler/command/services.py index b101c3bb..83f0636f 100644 --- a/minos/networks/handler/command/services.py +++ b/minos/networks/handler/command/services.py @@ -6,22 +6,14 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - Any, -) +from typing import Any from aiomisc.service.periodic import ( PeriodicService, Service, ) -from minos.common import ( - MinosConfig, -) -from .dispatcher import ( - MinosCommandHandlerDispatcher, -) -from .server import ( - MinosCommandHandlerServer, -) +from minos.common import MinosConfig +from .dispatcher import MinosCommandHandlerDispatcher +from .server import MinosCommandHandlerServer class MinosCommandServerService(Service): @@ -39,8 +31,9 @@ async def start(self) -> None: """ await self.dispatcher.setup() - self.consumer = await self.dispatcher.kafka_consumer(self.dispatcher._topics, self.dispatcher._broker_group_name, - self.dispatcher._kafka_conn_data) + self.consumer = await self.dispatcher.kafka_consumer( + self.dispatcher._topics, self.dispatcher._broker_group_name, self.dispatcher._kafka_conn_data + ) await self.dispatcher.handle_message(self.consumer) async def stop(self, exception: Exception = None) -> Any: diff --git a/minos/networks/handler/command_reply/__init__.py b/minos/networks/handler/command_reply/__init__.py index 05b9595a..e6dd5929 100644 --- a/minos/networks/handler/command_reply/__init__.py +++ b/minos/networks/handler/command_reply/__init__.py @@ -6,13 +6,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .server import ( - MinosCommandReplyHandlerServer, -) +from .server import MinosCommandReplyHandlerServer from .services import ( MinosCommandReplyPeriodicService, MinosCommandReplyServerService, ) -from .dispatcher import ( - MinosCommandReplyHandlerDispatcher, -) +from .dispatcher import MinosCommandReplyHandlerDispatcher diff --git a/minos/networks/handler/command_reply/dispatcher.py b/minos/networks/handler/command_reply/dispatcher.py index a62ecf88..d1234504 100644 --- a/minos/networks/handler/command_reply/dispatcher.py +++ b/minos/networks/handler/command_reply/dispatcher.py @@ -6,18 +6,10 @@ # permission of Clariteia SL. import collections -from minos.common.configuration.config import ( - MinosConfig, -) -from typing import ( - Any, -) -from ..dispatcher import ( - MinosHandlerDispatcher, -) -from minos.common import ( - CommandReply, -) +from minos.common.configuration.config import MinosConfig +from typing import Any +from ..dispatcher import MinosHandlerDispatcher +from minos.common import CommandReply class MinosCommandReplyHandlerDispatcher(MinosHandlerDispatcher): diff --git a/minos/networks/handler/command_reply/server.py b/minos/networks/handler/command_reply/server.py index 25ee9e30..86e845dc 100644 --- a/minos/networks/handler/command_reply/server.py +++ b/minos/networks/handler/command_reply/server.py @@ -5,18 +5,10 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from minos.common.configuration.config import ( - MinosConfig, -) -from typing import ( - Any, -) -from ..server import ( - MinosHandlerServer, -) -from minos.common import ( - CommandReply, -) +from minos.common.configuration.config import MinosConfig +from typing import Any +from ..server import MinosHandlerServer +from minos.common import CommandReply class MinosCommandReplyHandlerServer(MinosHandlerServer): diff --git a/minos/networks/handler/command_reply/services.py b/minos/networks/handler/command_reply/services.py index ba763755..d9db02c7 100644 --- a/minos/networks/handler/command_reply/services.py +++ b/minos/networks/handler/command_reply/services.py @@ -6,22 +6,14 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - Any, -) +from typing import Any from aiomisc.service.periodic import ( PeriodicService, Service, ) -from minos.common import ( - MinosConfig, -) -from .dispatcher import ( - MinosCommandReplyHandlerDispatcher, -) -from .server import ( - MinosCommandReplyHandlerServer, -) +from minos.common import MinosConfig +from .dispatcher import MinosCommandReplyHandlerDispatcher +from .server import MinosCommandReplyHandlerServer class MinosCommandReplyServerService(Service): @@ -39,8 +31,9 @@ async def start(self) -> None: """ await self.dispatcher.setup() - self.consumer = await self.dispatcher.kafka_consumer(self.dispatcher._topics, self.dispatcher._broker_group_name, - self.dispatcher._kafka_conn_data) + self.consumer = await self.dispatcher.kafka_consumer( + self.dispatcher._topics, self.dispatcher._broker_group_name, self.dispatcher._kafka_conn_data + ) await self.dispatcher.handle_message(self.consumer) async def stop(self, exception: Exception = None) -> Any: diff --git a/minos/networks/handler/dispatcher.py b/minos/networks/handler/dispatcher.py index d557cdc2..bac71da4 100644 --- a/minos/networks/handler/dispatcher.py +++ b/minos/networks/handler/dispatcher.py @@ -5,9 +5,7 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import ( - annotations, -) +from __future__ import annotations from abc import abstractmethod from typing import ( @@ -19,22 +17,12 @@ ) import aiopg -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.common.importlib import ( - import_module, -) -from minos.common.logs import ( - log, -) -from minos.networks.exceptions import ( - MinosNetworkException, -) +from minos.common.configuration.config import MinosConfig +from minos.common.importlib import import_module +from minos.common.logs import log +from minos.networks.exceptions import MinosNetworkException -from minos.networks.handler.abc import ( - MinosHandlerSetup, -) +from minos.networks.handler.abc import MinosHandlerSetup class MinosHandlerDispatcher(MinosHandlerSetup): @@ -51,9 +39,7 @@ def __init__(self, *, table_name: str, config: NamedTuple, **kwargs: Any): f"dbname={config.queue.database} user={config.queue.user} " f"password={config.queue.password} host={config.queue.host}" ) - self._handlers = { - item.name: {"controller": item.controller, "action": item.action} for item in config.items - } + self._handlers = {item.name: {"controller": item.controller, "action": item.action} for item in config.items} self._event_items = config.items self._topics = list(self._handlers.keys()) self._conf = config diff --git a/minos/networks/handler/event/__init__.py b/minos/networks/handler/event/__init__.py index da35486e..11da4d96 100644 --- a/minos/networks/handler/event/__init__.py +++ b/minos/networks/handler/event/__init__.py @@ -6,13 +6,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .server import ( - MinosEventHandlerServer, -) +from .server import MinosEventHandlerServer from .services import ( MinosEventPeriodicService, MinosEventServerService, ) -from .dispatcher import ( - MinosEventHandlerDispatcher, -) +from .dispatcher import MinosEventHandlerDispatcher diff --git a/minos/networks/handler/event/dispatcher.py b/minos/networks/handler/event/dispatcher.py index fabe051c..f9c672ed 100644 --- a/minos/networks/handler/event/dispatcher.py +++ b/minos/networks/handler/event/dispatcher.py @@ -5,18 +5,10 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from minos.common.configuration.config import ( - MinosConfig, -) -from typing import ( - Any, -) -from ..dispatcher import ( - MinosHandlerDispatcher, -) -from minos.common import ( - Event, -) +from minos.common.configuration.config import MinosConfig +from typing import Any +from ..dispatcher import MinosHandlerDispatcher +from minos.common import Event class MinosEventHandlerDispatcher(MinosHandlerDispatcher): diff --git a/minos/networks/handler/event/server.py b/minos/networks/handler/event/server.py index 8030bc8c..3f886364 100644 --- a/minos/networks/handler/event/server.py +++ b/minos/networks/handler/event/server.py @@ -5,18 +5,10 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from minos.common.configuration.config import ( - MinosConfig, -) -from typing import ( - Any, -) -from ..server import ( - MinosHandlerServer, -) -from minos.common import ( - Event, -) +from minos.common.configuration.config import MinosConfig +from typing import Any +from ..server import MinosHandlerServer +from minos.common import Event class MinosEventHandlerServer(MinosHandlerServer): diff --git a/minos/networks/handler/event/services.py b/minos/networks/handler/event/services.py index 39b13710..781ca6da 100644 --- a/minos/networks/handler/event/services.py +++ b/minos/networks/handler/event/services.py @@ -6,23 +6,15 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import ( - Any, -) +from typing import Any from aiomisc.service.periodic import ( PeriodicService, Service, ) -from minos.common import ( - MinosConfig, -) +from minos.common import MinosConfig -from .dispatcher import ( - MinosEventHandlerDispatcher, -) -from .server import ( - MinosEventHandlerServer, -) +from .dispatcher import MinosEventHandlerDispatcher +from .server import MinosEventHandlerServer class MinosEventServerService(Service): @@ -40,8 +32,9 @@ async def start(self) -> None: """ await self.dispatcher.setup() - self.consumer = await self.dispatcher.kafka_consumer(self.dispatcher._topics, self.dispatcher._broker_group_name, - self.dispatcher._kafka_conn_data) + self.consumer = await self.dispatcher.kafka_consumer( + self.dispatcher._topics, self.dispatcher._broker_group_name, self.dispatcher._kafka_conn_data + ) await self.dispatcher.handle_message(self.consumer) async def stop(self, exception: Exception = None) -> Any: diff --git a/minos/networks/handler/server.py b/minos/networks/handler/server.py index c0f29776..e2439be0 100644 --- a/minos/networks/handler/server.py +++ b/minos/networks/handler/server.py @@ -5,9 +5,7 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import ( - annotations, -) +from __future__ import annotations from abc import abstractmethod @@ -22,12 +20,8 @@ NamedTuple, ) import aiopg -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.networks.handler.abc import ( - MinosHandlerSetup, -) +from minos.common.configuration.config import MinosConfig +from minos.networks.handler.abc import MinosHandlerSetup class MinosHandlerServer(MinosHandlerSetup): @@ -38,7 +32,6 @@ class MinosHandlerServer(MinosHandlerSetup): """ - __slots__ = "_tasks", "_db_dsn", "_handlers", "_topics", "_broker_group_name" def __init__(self, *, table_name: str, config: NamedTuple, **kwargs: Any): @@ -48,9 +41,7 @@ def __init__(self, *, table_name: str, config: NamedTuple, **kwargs: Any): f"dbname={config.queue.database} user={config.queue.user} " f"password={config.queue.password} host={config.queue.host}" ) - self._handler = { - item.name: {"controller": item.controller, "action": item.action} for item in config.items - } + self._handler = {item.name: {"controller": item.controller, "action": item.action} for item in config.items} self._topics = list(self._handler.keys()) self._table_name = table_name @@ -139,11 +130,7 @@ async def handle_message(self, consumer: AsyncIterator): @staticmethod async def kafka_consumer(topics: list, group_name: str, conn: str): # start the Service Event Consumer for Kafka - consumer = AIOKafkaConsumer( - group_id=group_name, - auto_offset_reset="latest", - bootstrap_servers=conn, - ) + consumer = AIOKafkaConsumer(group_id=group_name, auto_offset_reset="latest", bootstrap_servers=conn,) await consumer.start() consumer.subscribe(topics) diff --git a/tests/services/CommandTestService.py b/tests/services/CommandTestService.py index be12d04c..71573a3c 100644 --- a/tests/services/CommandTestService.py +++ b/tests/services/CommandTestService.py @@ -1,6 +1,4 @@ -from minos.common import ( - Command, -) +from minos.common import Command class CommandService(object): diff --git a/tests/services/SagaTestService.py b/tests/services/SagaTestService.py index 12bec134..bd169d45 100644 --- a/tests/services/SagaTestService.py +++ b/tests/services/SagaTestService.py @@ -1,6 +1,4 @@ -from minos.common import ( - CommandReply, -) +from minos.common import CommandReply class SagaService(object): @@ -9,4 +7,3 @@ async def add_order(self, topic: str, command: CommandReply): async def delete_order(self, topic: str, command: CommandReply): return "delete_order_saga" - diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index 30d56a19..cb35244d 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -8,12 +8,8 @@ import unittest import aiopg -from minos.common import ( - MinosConfig, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common import MinosConfig +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, @@ -28,7 +24,13 @@ class TestMinosCommandBroker(PostgresAsyncTestCase): CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" async def test_commands_broker_insertion(self): - broker = MinosCommandBroker.from_config("CommandBroker", config=self.config, saga_id= "9347839473kfslf", task_id= "92839283hjijh232", reply_on="test_reply_on") + broker = MinosCommandBroker.from_config( + "CommandBroker", + config=self.config, + saga_id="9347839473kfslf", + task_id="92839283hjijh232", + reply_on="test_reply_on", + ) await broker.setup() item = NaiveAggregate(test_id=1, test=2, id=1, version=1) @@ -37,7 +39,13 @@ async def test_commands_broker_insertion(self): assert queue_id > 0 async def test_if_commands_was_deleted(self): - broker = MinosCommandBroker.from_config("CommandBroker-Delete", config=self.config, saga_id= "9347839473kfslf", task_id= "92839283hjijh232", reply_on="test_reply_on") + broker = MinosCommandBroker.from_config( + "CommandBroker-Delete", + config=self.config, + saga_id="9347839473kfslf", + task_id="92839283hjijh232", + reply_on="test_reply_on", + ) await broker.setup() item = NaiveAggregate(test_id=1, test=2, id=1, version=1) @@ -57,7 +65,13 @@ async def test_if_commands_was_deleted(self): assert records[0] == 0 async def test_if_commands_retry_was_incremented(self): - broker = MinosCommandBroker.from_config("CommandBroker-Delete", config=self.config, saga_id= "9347839473kfslf", task_id= "92839283hjijh232", reply_on="test_reply_on") + broker = MinosCommandBroker.from_config( + "CommandBroker-Delete", + config=self.config, + saga_id="9347839473kfslf", + task_id="92839283hjijh232", + reply_on="test_reply_on", + ) await broker.setup() item = NaiveAggregate(test_id=1, test=2, id=1, version=1) diff --git a/tests/test_networks/test_handler/test_command/test_command_server.py b/tests/test_networks/test_handler/test_command/test_command_server.py index 510cad1d..75508fc1 100644 --- a/tests/test_networks/test_handler/test_command/test_command_server.py +++ b/tests/test_networks/test_handler/test_command/test_command_server.py @@ -1,15 +1,7 @@ -from collections import ( - namedtuple, -) -from minos.common import ( - Command, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) -from minos.networks import ( - MinosCommandHandlerServer, -) +from collections import namedtuple +from minos.common import Command +from minos.common.testing import PostgresAsyncTestCase +from minos.networks import MinosCommandHandlerServer from tests.utils import ( BASE_PATH, NaiveAggregate, @@ -30,7 +22,14 @@ async def test_none_config(self): async def test_queue_add(self): model = NaiveAggregate(test_id=1, test=2, id=1, version=1) - event_instance = Command(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + event_instance = Command( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) bin_data = event_instance.avro_bytes Command.from_avro_bytes(bin_data) @@ -47,7 +46,14 @@ async def test_handle_message(self): await event_server.setup() model = NaiveAggregate(test_id=1, test=2, id=1, version=1) - event_instance = Command(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + event_instance = Command( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) bin_data = event_instance.avro_bytes Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) diff --git a/tests/test_networks/test_handler/test_command/test_command_services.py b/tests/test_networks/test_handler/test_command/test_command_services.py index c8eaea79..6c0ca7ee 100644 --- a/tests/test_networks/test_handler/test_command/test_command_services.py +++ b/tests/test_networks/test_handler/test_command/test_command_services.py @@ -1,18 +1,12 @@ import unittest -from unittest.mock import ( - MagicMock, -) +from unittest.mock import MagicMock -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosCommandPeriodicService, MinosCommandServerService, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class TestMinosCommandServices(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_handler/test_command/test_dispatcher.py b/tests/test_networks/test_handler/test_command/test_dispatcher.py index 30622eb3..2543b1a7 100644 --- a/tests/test_networks/test_handler/test_command/test_dispatcher.py +++ b/tests/test_networks/test_handler/test_command/test_dispatcher.py @@ -1,18 +1,10 @@ import datetime import aiopg -from minos.common import ( - Command, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) -from minos.networks import ( - MinosCommandHandlerDispatcher, -) -from minos.networks.exceptions import ( - MinosNetworkException, -) +from minos.common import Command +from minos.common.testing import PostgresAsyncTestCase +from minos.networks import MinosCommandHandlerDispatcher +from minos.networks.exceptions import MinosNetworkException from tests.utils import ( BASE_PATH, NaiveAggregate, @@ -43,7 +35,14 @@ async def test_if_queue_table_exists(self): async def test_get_event_handler(self): model = NaiveAggregate(test_id=1, test=2, id=1, version=1) - event_instance = Command(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + event_instance = Command( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) m = MinosCommandHandlerDispatcher.from_config(config=self.config) cls = m.get_event_handler(topic=event_instance.topic) @@ -53,7 +52,14 @@ async def test_get_event_handler(self): async def test_non_implemented_action(self): model = NaiveAggregate(test_id=1, test=2, id=1, version=1) - instance = Command(topic="NotExisting", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + instance = Command( + topic="NotExisting", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) m = MinosCommandHandlerDispatcher.from_config(config=self.config) with self.assertRaises(MinosNetworkException) as context: @@ -75,7 +81,14 @@ async def test_event_queue_checker(self): await handler.setup() model = NaiveAggregate(test_id=1, test=2, id=1, version=1) - instance = Command(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + instance = Command( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) bin_data = instance.avro_bytes Command.from_avro_bytes(bin_data) @@ -105,8 +118,7 @@ async def test_event_queue_checker(self): async def test_event_queue_checker_wrong_event(self): handler = MinosCommandHandlerDispatcher.from_config(config=self.config) await handler.setup() - bin_data = bytes(b'Test') - + bin_data = bytes(b"Test") async with aiopg.connect(**self.commands_queue_db) as connect: async with connect.cursor() as cur: diff --git a/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py b/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py index 14c06547..aab9d0c8 100644 --- a/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py +++ b/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py @@ -1,15 +1,7 @@ -from collections import ( - namedtuple, -) -from minos.common import ( - CommandReply, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) -from minos.networks import ( - MinosCommandReplyHandlerServer, -) +from collections import namedtuple +from minos.common import CommandReply +from minos.common.testing import PostgresAsyncTestCase +from minos.networks import MinosCommandReplyHandlerServer from tests.utils import ( BASE_PATH, NaiveAggregate, @@ -30,7 +22,14 @@ async def test_none_config(self): async def test_queue_add(self): model = NaiveAggregate(test_id=1, test=2, id=1, version=1) - event_instance = CommandReply(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + event_instance = CommandReply( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) bin_data = event_instance.avro_bytes CommandReply.from_avro_bytes(bin_data) @@ -47,7 +46,14 @@ async def test_handle_message(self): await event_server.setup() model = NaiveAggregate(test_id=1, test=2, id=1, version=1) - event_instance = CommandReply(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + event_instance = CommandReply( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) bin_data = event_instance.avro_bytes Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) diff --git a/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py b/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py index a193743d..d7a4ae19 100644 --- a/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py +++ b/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py @@ -1,17 +1,11 @@ -from unittest.mock import ( - MagicMock, -) +from unittest.mock import MagicMock -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosCommandReplyPeriodicService, MinosCommandReplyServerService, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class TestMinosCommandReplyServices(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py b/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py index 8f657bf8..bdaedd3a 100644 --- a/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py +++ b/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py @@ -1,18 +1,10 @@ import datetime import aiopg -from minos.common import ( - CommandReply, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) -from minos.networks import ( - MinosCommandReplyHandlerDispatcher, -) -from minos.networks.exceptions import ( - MinosNetworkException, -) +from minos.common import CommandReply +from minos.common.testing import PostgresAsyncTestCase +from minos.networks import MinosCommandReplyHandlerDispatcher +from minos.networks.exceptions import MinosNetworkException from tests.utils import ( BASE_PATH, NaiveAggregate, @@ -45,7 +37,14 @@ async def test_if_queue_table_exists(self): async def test_get_event_handler(self): model = NaiveAggregate(test_id=1, test=2, id=1, version=1) - event_instance = CommandReply(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + event_instance = CommandReply( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) m = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) cls = m.get_event_handler(topic=event_instance.topic) @@ -55,7 +54,14 @@ async def test_get_event_handler(self): async def test_non_implemented_action(self): model = NaiveAggregate(test_id=1, test=2, id=1, version=1) - instance = CommandReply(topic="NotExisting", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + instance = CommandReply( + topic="NotExisting", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) m = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) with self.assertRaises(MinosNetworkException) as context: @@ -85,7 +91,14 @@ async def test_event_queue_checker(self): await cur.execute("DELETE FROM {table};".format(table=MinosCommandReplyHandlerDispatcher.TABLE)) model = NaiveAggregate(test_id=1, test=2, id=1, version=1) - instance = CommandReply(topic="AddOrder", model=model.classname, items=[], saga_id="43434jhij", task_id="juhjh34", reply_on="mkk2334") + instance = CommandReply( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) bin_data = instance.avro_bytes CommandReply.from_avro_bytes(bin_data) @@ -124,7 +137,7 @@ async def test_command_reply_queue_checker_wrong_event(self): async with connect.cursor() as cur: await cur.execute("DELETE FROM {table};".format(table=MinosCommandReplyHandlerDispatcher.TABLE)) - bin_data = bytes(b'Test') + bin_data = bytes(b"Test") async with aiopg.connect(**self._meta_saga_queue_db) as connect: async with connect.cursor() as cur: @@ -147,6 +160,4 @@ async def test_command_reply_queue_checker_wrong_event(self): await cur.execute("SELECT COUNT(*) FROM command_reply_queue WHERE id=%d" % (queue_id)) records = await cur.fetchone() - - assert records[0] == 1 diff --git a/tests/test_networks/test_handler/test_event/test_dispatcher.py b/tests/test_networks/test_handler/test_event/test_dispatcher.py index 29df227b..1ba0efe1 100644 --- a/tests/test_networks/test_handler/test_event/test_dispatcher.py +++ b/tests/test_networks/test_handler/test_event/test_dispatcher.py @@ -1,18 +1,10 @@ import datetime import aiopg -from minos.common import ( - Event, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) -from minos.networks import ( - MinosEventHandlerDispatcher, -) -from minos.networks.exceptions import ( - MinosNetworkException, -) +from minos.common import Event +from minos.common.testing import PostgresAsyncTestCase +from minos.networks import MinosEventHandlerDispatcher +from minos.networks.exceptions import MinosNetworkException from tests.utils import ( BASE_PATH, NaiveAggregate, @@ -105,8 +97,7 @@ async def test_event_queue_checker(self): async def test_event_queue_checker_wrong_event(self): handler = MinosEventHandlerDispatcher.from_config(config=self.config) await handler.setup() - bin_data = bytes(b'Test') - + bin_data = bytes(b"Test") async with aiopg.connect(**self.events_queue_db) as connect: async with connect.cursor() as cur: @@ -130,4 +121,3 @@ async def test_event_queue_checker_wrong_event(self): records = await cur.fetchone() assert records[0] == 1 - diff --git a/tests/test_networks/test_handler/test_event/test_event_server.py b/tests/test_networks/test_handler/test_event/test_event_server.py index f152947e..062bdfc5 100644 --- a/tests/test_networks/test_handler/test_event/test_event_server.py +++ b/tests/test_networks/test_handler/test_event/test_event_server.py @@ -1,15 +1,7 @@ -from collections import ( - namedtuple, -) -from minos.common import ( - Event, -) -from minos.common.testing import ( - PostgresAsyncTestCase, -) -from minos.networks import ( - MinosEventHandlerServer, -) +from collections import namedtuple +from minos.common import Event +from minos.common.testing import PostgresAsyncTestCase +from minos.networks import MinosEventHandlerServer from tests.utils import ( BASE_PATH, NaiveAggregate, diff --git a/tests/test_networks/test_handler/test_event/test_event_services.py b/tests/test_networks/test_handler/test_event/test_event_services.py index 54651f1e..ae7dd2fd 100644 --- a/tests/test_networks/test_handler/test_event/test_event_services.py +++ b/tests/test_networks/test_handler/test_event/test_event_services.py @@ -1,18 +1,12 @@ import unittest -from unittest.mock import ( - MagicMock, -) +from unittest.mock import MagicMock -from minos.common.testing import ( - PostgresAsyncTestCase, -) +from minos.common.testing import PostgresAsyncTestCase from minos.networks import ( MinosEventPeriodicService, MinosEventServerService, ) -from tests.utils import ( - BASE_PATH, -) +from tests.utils import BASE_PATH class TestMinosEventServices(PostgresAsyncTestCase): From 398b810fc529077b6d6bb6d90fd2a12a12d4fb91 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 10 May 2021 21:27:10 +0000 Subject: [PATCH 227/239] Restyled by isort --- minos/networks/__init__.py | 20 ++++++------ minos/networks/broker/commands.py | 8 +++-- minos/networks/handler/__init__.py | 14 ++++----- minos/networks/handler/abc.py | 16 +++++++--- minos/networks/handler/command/__init__.py | 8 +++-- minos/networks/handler/command/dispatcher.py | 18 ++++++++--- minos/networks/handler/command/server.py | 18 ++++++++--- minos/networks/handler/command/services.py | 18 ++++++++--- .../handler/command_reply/__init__.py | 8 +++-- .../handler/command_reply/dispatcher.py | 17 +++++++--- .../networks/handler/command_reply/server.py | 18 ++++++++--- .../handler/command_reply/services.py | 18 ++++++++--- minos/networks/handler/dispatcher.py | 31 +++++++++++++------ minos/networks/handler/event/__init__.py | 8 +++-- minos/networks/handler/event/dispatcher.py | 18 ++++++++--- minos/networks/handler/event/server.py | 18 ++++++++--- minos/networks/handler/event/services.py | 17 +++++++--- minos/networks/handler/server.py | 28 ++++++++++++----- tests/services/CommandTestService.py | 4 ++- tests/services/SagaTestService.py | 4 ++- .../test_networks/test_broker/test_command.py | 8 +++-- .../test_command/test_command_server.py | 17 +++++++--- .../test_command/test_command_services.py | 12 +++++-- .../test_command/test_dispatcher.py | 16 +++++++--- .../test_command__reply_server.py | 17 +++++++--- .../test_command_reply_services.py | 12 +++++-- .../test_command_reply/test_dispatcher.py | 16 +++++++--- .../test_event/test_dispatcher.py | 16 +++++++--- .../test_event/test_event_server.py | 17 +++++++--- .../test_event/test_event_services.py | 12 +++++-- 30 files changed, 332 insertions(+), 120 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 2598bb3e..ff081f52 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -13,24 +13,24 @@ MinosQueueDispatcher, MinosQueueService, ) +from .exceptions import ( + MinosNetworkException, + MinosPreviousVersionSnapshotException, + MinosSnapshotException, +) from .handler import ( - MinosEventHandlerDispatcher, - MinosEventPeriodicService, - MinosEventHandlerServer, - MinosEventServerService, MinosCommandHandlerDispatcher, MinosCommandHandlerServer, MinosCommandPeriodicService, - MinosCommandServerService, MinosCommandReplyHandlerDispatcher, MinosCommandReplyHandlerServer, MinosCommandReplyPeriodicService, MinosCommandReplyServerService, -) -from .exceptions import ( - MinosNetworkException, - MinosPreviousVersionSnapshotException, - MinosSnapshotException, + MinosCommandServerService, + MinosEventHandlerDispatcher, + MinosEventHandlerServer, + MinosEventPeriodicService, + MinosEventServerService, ) from .snapshots import ( MinosSnapshotDispatcher, diff --git a/minos/networks/broker/commands.py b/minos/networks/broker/commands.py index d3a98485..0357893c 100644 --- a/minos/networks/broker/commands.py +++ b/minos/networks/broker/commands.py @@ -5,7 +5,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from __future__ import annotations +from __future__ import ( + annotations, +) from typing import ( NoReturn, @@ -18,7 +20,9 @@ MinosConfig, ) -from .abc import MinosBroker +from .abc import ( + MinosBroker, +) class MinosCommandBroker(MinosBroker): diff --git a/minos/networks/handler/__init__.py b/minos/networks/handler/__init__.py index 9b2b1f09..f3314979 100644 --- a/minos/networks/handler/__init__.py +++ b/minos/networks/handler/__init__.py @@ -6,23 +6,21 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .event import ( - MinosEventHandlerDispatcher, - MinosEventHandlerServer, - MinosEventPeriodicService, - MinosEventServerService, -) - from .command import ( MinosCommandHandlerDispatcher, MinosCommandHandlerServer, MinosCommandPeriodicService, MinosCommandServerService, ) - from .command_reply import ( MinosCommandReplyHandlerDispatcher, MinosCommandReplyHandlerServer, MinosCommandReplyPeriodicService, MinosCommandReplyServerService, ) +from .event import ( + MinosEventHandlerDispatcher, + MinosEventHandlerServer, + MinosEventPeriodicService, + MinosEventServerService, +) diff --git a/minos/networks/handler/abc.py b/minos/networks/handler/abc.py index d184e4e6..4b6d37a7 100644 --- a/minos/networks/handler/abc.py +++ b/minos/networks/handler/abc.py @@ -5,12 +5,20 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from abc import ABC -from datetime import datetime -from typing import NoReturn +from abc import ( + ABC, +) +from datetime import ( + datetime, +) +from typing import ( + NoReturn, +) import aiopg -from minos.common import MinosSetup +from minos.common import ( + MinosSetup, +) class MinosHandlerSetup(MinosSetup): diff --git a/minos/networks/handler/command/__init__.py b/minos/networks/handler/command/__init__.py index d4a6c6ca..d54f41b7 100644 --- a/minos/networks/handler/command/__init__.py +++ b/minos/networks/handler/command/__init__.py @@ -6,9 +6,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .server import MinosCommandHandlerServer +from .dispatcher import ( + MinosCommandHandlerDispatcher, +) +from .server import ( + MinosCommandHandlerServer, +) from .services import ( MinosCommandPeriodicService, MinosCommandServerService, ) -from .dispatcher import MinosCommandHandlerDispatcher diff --git a/minos/networks/handler/command/dispatcher.py b/minos/networks/handler/command/dispatcher.py index 4e76b2ed..9e4b2927 100644 --- a/minos/networks/handler/command/dispatcher.py +++ b/minos/networks/handler/command/dispatcher.py @@ -5,10 +5,20 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from minos.common.configuration.config import MinosConfig -from typing import Any -from ..dispatcher import MinosHandlerDispatcher -from minos.common import Command +from typing import ( + Any, +) + +from minos.common import ( + Command, +) +from minos.common.configuration.config import ( + MinosConfig, +) + +from ..dispatcher import ( + MinosHandlerDispatcher, +) class MinosCommandHandlerDispatcher(MinosHandlerDispatcher): diff --git a/minos/networks/handler/command/server.py b/minos/networks/handler/command/server.py index a9b30bd8..a8cd2309 100644 --- a/minos/networks/handler/command/server.py +++ b/minos/networks/handler/command/server.py @@ -5,10 +5,20 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from minos.common.configuration.config import MinosConfig -from typing import Any -from ..server import MinosHandlerServer -from minos.common import Command +from typing import ( + Any, +) + +from minos.common import ( + Command, +) +from minos.common.configuration.config import ( + MinosConfig, +) + +from ..server import ( + MinosHandlerServer, +) class MinosCommandHandlerServer(MinosHandlerServer): diff --git a/minos/networks/handler/command/services.py b/minos/networks/handler/command/services.py index 83f0636f..7c5074ac 100644 --- a/minos/networks/handler/command/services.py +++ b/minos/networks/handler/command/services.py @@ -6,14 +6,24 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import Any +from typing import ( + Any, +) + from aiomisc.service.periodic import ( PeriodicService, Service, ) -from minos.common import MinosConfig -from .dispatcher import MinosCommandHandlerDispatcher -from .server import MinosCommandHandlerServer +from minos.common import ( + MinosConfig, +) + +from .dispatcher import ( + MinosCommandHandlerDispatcher, +) +from .server import ( + MinosCommandHandlerServer, +) class MinosCommandServerService(Service): diff --git a/minos/networks/handler/command_reply/__init__.py b/minos/networks/handler/command_reply/__init__.py index e6dd5929..8bba3430 100644 --- a/minos/networks/handler/command_reply/__init__.py +++ b/minos/networks/handler/command_reply/__init__.py @@ -6,9 +6,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .server import MinosCommandReplyHandlerServer +from .dispatcher import ( + MinosCommandReplyHandlerDispatcher, +) +from .server import ( + MinosCommandReplyHandlerServer, +) from .services import ( MinosCommandReplyPeriodicService, MinosCommandReplyServerService, ) -from .dispatcher import MinosCommandReplyHandlerDispatcher diff --git a/minos/networks/handler/command_reply/dispatcher.py b/minos/networks/handler/command_reply/dispatcher.py index d1234504..3f611524 100644 --- a/minos/networks/handler/command_reply/dispatcher.py +++ b/minos/networks/handler/command_reply/dispatcher.py @@ -5,11 +5,20 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. import collections +from typing import ( + Any, +) -from minos.common.configuration.config import MinosConfig -from typing import Any -from ..dispatcher import MinosHandlerDispatcher -from minos.common import CommandReply +from minos.common import ( + CommandReply, +) +from minos.common.configuration.config import ( + MinosConfig, +) + +from ..dispatcher import ( + MinosHandlerDispatcher, +) class MinosCommandReplyHandlerDispatcher(MinosHandlerDispatcher): diff --git a/minos/networks/handler/command_reply/server.py b/minos/networks/handler/command_reply/server.py index 86e845dc..337a35ac 100644 --- a/minos/networks/handler/command_reply/server.py +++ b/minos/networks/handler/command_reply/server.py @@ -5,10 +5,20 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from minos.common.configuration.config import MinosConfig -from typing import Any -from ..server import MinosHandlerServer -from minos.common import CommandReply +from typing import ( + Any, +) + +from minos.common import ( + CommandReply, +) +from minos.common.configuration.config import ( + MinosConfig, +) + +from ..server import ( + MinosHandlerServer, +) class MinosCommandReplyHandlerServer(MinosHandlerServer): diff --git a/minos/networks/handler/command_reply/services.py b/minos/networks/handler/command_reply/services.py index d9db02c7..1fe7ced2 100644 --- a/minos/networks/handler/command_reply/services.py +++ b/minos/networks/handler/command_reply/services.py @@ -6,14 +6,24 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import Any +from typing import ( + Any, +) + from aiomisc.service.periodic import ( PeriodicService, Service, ) -from minos.common import MinosConfig -from .dispatcher import MinosCommandReplyHandlerDispatcher -from .server import MinosCommandReplyHandlerServer +from minos.common import ( + MinosConfig, +) + +from .dispatcher import ( + MinosCommandReplyHandlerDispatcher, +) +from .server import ( + MinosCommandReplyHandlerServer, +) class MinosCommandReplyServerService(Service): diff --git a/minos/networks/handler/dispatcher.py b/minos/networks/handler/dispatcher.py index bac71da4..76d0d16e 100644 --- a/minos/networks/handler/dispatcher.py +++ b/minos/networks/handler/dispatcher.py @@ -5,24 +5,37 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import annotations +from __future__ import ( + annotations, +) -from abc import abstractmethod +from abc import ( + abstractmethod, +) from typing import ( Any, Callable, + NamedTuple, NoReturn, Optional, - NamedTuple, ) import aiopg -from minos.common.configuration.config import MinosConfig -from minos.common.importlib import import_module -from minos.common.logs import log -from minos.networks.exceptions import MinosNetworkException - -from minos.networks.handler.abc import MinosHandlerSetup +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.importlib import ( + import_module, +) +from minos.common.logs import ( + log, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) +from minos.networks.handler.abc import ( + MinosHandlerSetup, +) class MinosHandlerDispatcher(MinosHandlerSetup): diff --git a/minos/networks/handler/event/__init__.py b/minos/networks/handler/event/__init__.py index 11da4d96..0b8f5274 100644 --- a/minos/networks/handler/event/__init__.py +++ b/minos/networks/handler/event/__init__.py @@ -6,9 +6,13 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .server import MinosEventHandlerServer +from .dispatcher import ( + MinosEventHandlerDispatcher, +) +from .server import ( + MinosEventHandlerServer, +) from .services import ( MinosEventPeriodicService, MinosEventServerService, ) -from .dispatcher import MinosEventHandlerDispatcher diff --git a/minos/networks/handler/event/dispatcher.py b/minos/networks/handler/event/dispatcher.py index f9c672ed..783a7c3b 100644 --- a/minos/networks/handler/event/dispatcher.py +++ b/minos/networks/handler/event/dispatcher.py @@ -5,10 +5,20 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from minos.common.configuration.config import MinosConfig -from typing import Any -from ..dispatcher import MinosHandlerDispatcher -from minos.common import Event +from typing import ( + Any, +) + +from minos.common import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) + +from ..dispatcher import ( + MinosHandlerDispatcher, +) class MinosEventHandlerDispatcher(MinosHandlerDispatcher): diff --git a/minos/networks/handler/event/server.py b/minos/networks/handler/event/server.py index 3f886364..55341e98 100644 --- a/minos/networks/handler/event/server.py +++ b/minos/networks/handler/event/server.py @@ -5,10 +5,20 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from minos.common.configuration.config import MinosConfig -from typing import Any -from ..server import MinosHandlerServer -from minos.common import Event +from typing import ( + Any, +) + +from minos.common import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) + +from ..server import ( + MinosHandlerServer, +) class MinosEventHandlerServer(MinosHandlerServer): diff --git a/minos/networks/handler/event/services.py b/minos/networks/handler/event/services.py index 781ca6da..58f0fd7f 100644 --- a/minos/networks/handler/event/services.py +++ b/minos/networks/handler/event/services.py @@ -6,15 +6,24 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from typing import Any +from typing import ( + Any, +) + from aiomisc.service.periodic import ( PeriodicService, Service, ) -from minos.common import MinosConfig +from minos.common import ( + MinosConfig, +) -from .dispatcher import MinosEventHandlerDispatcher -from .server import MinosEventHandlerServer +from .dispatcher import ( + MinosEventHandlerDispatcher, +) +from .server import ( + MinosEventHandlerServer, +) class MinosEventServerService(Service): diff --git a/minos/networks/handler/server.py b/minos/networks/handler/server.py index e2439be0..a65d27c1 100644 --- a/minos/networks/handler/server.py +++ b/minos/networks/handler/server.py @@ -5,23 +5,35 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from __future__ import annotations - -from abc import abstractmethod +from __future__ import ( + annotations, +) -from aiokafka import AIOKafkaConsumer -from psycopg2.extensions import AsIs import asyncio import datetime +from abc import ( + abstractmethod, +) from typing import ( Any, AsyncIterator, - Optional, NamedTuple, + Optional, ) + import aiopg -from minos.common.configuration.config import MinosConfig -from minos.networks.handler.abc import MinosHandlerSetup +from aiokafka import ( + AIOKafkaConsumer, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.networks.handler.abc import ( + MinosHandlerSetup, +) +from psycopg2.extensions import ( + AsIs, +) class MinosHandlerServer(MinosHandlerSetup): diff --git a/tests/services/CommandTestService.py b/tests/services/CommandTestService.py index 71573a3c..be12d04c 100644 --- a/tests/services/CommandTestService.py +++ b/tests/services/CommandTestService.py @@ -1,4 +1,6 @@ -from minos.common import Command +from minos.common import ( + Command, +) class CommandService(object): diff --git a/tests/services/SagaTestService.py b/tests/services/SagaTestService.py index bd169d45..eb1e37ee 100644 --- a/tests/services/SagaTestService.py +++ b/tests/services/SagaTestService.py @@ -1,4 +1,6 @@ -from minos.common import CommandReply +from minos.common import ( + CommandReply, +) class SagaService(object): diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py index cb35244d..3093c086 100644 --- a/tests/test_networks/test_broker/test_command.py +++ b/tests/test_networks/test_broker/test_command.py @@ -8,8 +8,12 @@ import unittest import aiopg -from minos.common import MinosConfig -from minos.common.testing import PostgresAsyncTestCase +from minos.common import ( + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosCommandBroker, MinosQueueDispatcher, diff --git a/tests/test_networks/test_handler/test_command/test_command_server.py b/tests/test_networks/test_handler/test_command/test_command_server.py index 75508fc1..de29345d 100644 --- a/tests/test_networks/test_handler/test_command/test_command_server.py +++ b/tests/test_networks/test_handler/test_command/test_command_server.py @@ -1,7 +1,16 @@ -from collections import namedtuple -from minos.common import Command -from minos.common.testing import PostgresAsyncTestCase -from minos.networks import MinosCommandHandlerServer +from collections import ( + namedtuple, +) + +from minos.common import ( + Command, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandHandlerServer, +) from tests.utils import ( BASE_PATH, NaiveAggregate, diff --git a/tests/test_networks/test_handler/test_command/test_command_services.py b/tests/test_networks/test_handler/test_command/test_command_services.py index 6c0ca7ee..c8eaea79 100644 --- a/tests/test_networks/test_handler/test_command/test_command_services.py +++ b/tests/test_networks/test_handler/test_command/test_command_services.py @@ -1,12 +1,18 @@ import unittest -from unittest.mock import MagicMock +from unittest.mock import ( + MagicMock, +) -from minos.common.testing import PostgresAsyncTestCase +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosCommandPeriodicService, MinosCommandServerService, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class TestMinosCommandServices(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_handler/test_command/test_dispatcher.py b/tests/test_networks/test_handler/test_command/test_dispatcher.py index 2543b1a7..b448a507 100644 --- a/tests/test_networks/test_handler/test_command/test_dispatcher.py +++ b/tests/test_networks/test_handler/test_command/test_dispatcher.py @@ -1,10 +1,18 @@ import datetime import aiopg -from minos.common import Command -from minos.common.testing import PostgresAsyncTestCase -from minos.networks import MinosCommandHandlerDispatcher -from minos.networks.exceptions import MinosNetworkException +from minos.common import ( + Command, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandHandlerDispatcher, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) from tests.utils import ( BASE_PATH, NaiveAggregate, diff --git a/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py b/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py index aab9d0c8..288564f3 100644 --- a/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py +++ b/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py @@ -1,7 +1,16 @@ -from collections import namedtuple -from minos.common import CommandReply -from minos.common.testing import PostgresAsyncTestCase -from minos.networks import MinosCommandReplyHandlerServer +from collections import ( + namedtuple, +) + +from minos.common import ( + CommandReply, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandReplyHandlerServer, +) from tests.utils import ( BASE_PATH, NaiveAggregate, diff --git a/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py b/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py index d7a4ae19..a193743d 100644 --- a/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py +++ b/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py @@ -1,11 +1,17 @@ -from unittest.mock import MagicMock +from unittest.mock import ( + MagicMock, +) -from minos.common.testing import PostgresAsyncTestCase +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosCommandReplyPeriodicService, MinosCommandReplyServerService, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class TestMinosCommandReplyServices(PostgresAsyncTestCase): diff --git a/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py b/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py index bdaedd3a..2e989184 100644 --- a/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py +++ b/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py @@ -1,10 +1,18 @@ import datetime import aiopg -from minos.common import CommandReply -from minos.common.testing import PostgresAsyncTestCase -from minos.networks import MinosCommandReplyHandlerDispatcher -from minos.networks.exceptions import MinosNetworkException +from minos.common import ( + CommandReply, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandReplyHandlerDispatcher, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) from tests.utils import ( BASE_PATH, NaiveAggregate, diff --git a/tests/test_networks/test_handler/test_event/test_dispatcher.py b/tests/test_networks/test_handler/test_event/test_dispatcher.py index 1ba0efe1..f8ce87ce 100644 --- a/tests/test_networks/test_handler/test_event/test_dispatcher.py +++ b/tests/test_networks/test_handler/test_event/test_dispatcher.py @@ -1,10 +1,18 @@ import datetime import aiopg -from minos.common import Event -from minos.common.testing import PostgresAsyncTestCase -from minos.networks import MinosEventHandlerDispatcher -from minos.networks.exceptions import MinosNetworkException +from minos.common import ( + Event, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosEventHandlerDispatcher, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) from tests.utils import ( BASE_PATH, NaiveAggregate, diff --git a/tests/test_networks/test_handler/test_event/test_event_server.py b/tests/test_networks/test_handler/test_event/test_event_server.py index 062bdfc5..4bd14e60 100644 --- a/tests/test_networks/test_handler/test_event/test_event_server.py +++ b/tests/test_networks/test_handler/test_event/test_event_server.py @@ -1,7 +1,16 @@ -from collections import namedtuple -from minos.common import Event -from minos.common.testing import PostgresAsyncTestCase -from minos.networks import MinosEventHandlerServer +from collections import ( + namedtuple, +) + +from minos.common import ( + Event, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosEventHandlerServer, +) from tests.utils import ( BASE_PATH, NaiveAggregate, diff --git a/tests/test_networks/test_handler/test_event/test_event_services.py b/tests/test_networks/test_handler/test_event/test_event_services.py index ae7dd2fd..54651f1e 100644 --- a/tests/test_networks/test_handler/test_event/test_event_services.py +++ b/tests/test_networks/test_handler/test_event/test_event_services.py @@ -1,12 +1,18 @@ import unittest -from unittest.mock import MagicMock +from unittest.mock import ( + MagicMock, +) -from minos.common.testing import PostgresAsyncTestCase +from minos.common.testing import ( + PostgresAsyncTestCase, +) from minos.networks import ( MinosEventPeriodicService, MinosEventServerService, ) -from tests.utils import BASE_PATH +from tests.utils import ( + BASE_PATH, +) class TestMinosEventServices(PostgresAsyncTestCase): From c394b6a5a1cb24b566be311dced292042700c00f Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 11 May 2021 17:32:25 +0200 Subject: [PATCH 228/239] Base structure of Microservice REST Handler Microservice REST #118 --- minos/networks/rest_interface/__init__.py | 11 +++++ minos/networks/rest_interface/handler.py | 46 +++++++++++++++++++ requirements_dev.txt | 3 ++ tests/services/TestRestService.py | 9 ++++ .../test_networks/rest_interface/__init__.py | 0 .../rest_interface/test_interface.py | 32 +++++++++++++ 6 files changed, 101 insertions(+) create mode 100644 minos/networks/rest_interface/__init__.py create mode 100644 minos/networks/rest_interface/handler.py create mode 100644 tests/services/TestRestService.py create mode 100644 tests/test_networks/rest_interface/__init__.py create mode 100644 tests/test_networks/rest_interface/test_interface.py diff --git a/minos/networks/rest_interface/__init__.py b/minos/networks/rest_interface/__init__.py new file mode 100644 index 00000000..00be2a65 --- /dev/null +++ b/minos/networks/rest_interface/__init__.py @@ -0,0 +1,11 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from .handler import ( + RestInterfaceHandler +) diff --git a/minos/networks/rest_interface/handler.py b/minos/networks/rest_interface/handler.py new file mode 100644 index 00000000..53a96bef --- /dev/null +++ b/minos/networks/rest_interface/handler.py @@ -0,0 +1,46 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from aiohttp import web +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.importlib import ( + import_module, +) + + +class RestInterfaceHandler: + """ + Rest Interface Handler + + Rest Interface for aiohttp web handling. + + """ + + __slots__ = "_config", "_app" + + def __init__(self, config: MinosConfig, app: web.Application = web.Application()): + self._config = config + self._app = app + self.load_routes() + + def load_routes(self): + for item in self._config.rest.endpoints: + callable_f = self.class_resolver(item.controller, item.action) + self._app.router.add_route(item.method, item.route, callable_f) + + @staticmethod + def class_resolver(controller: str, action: str): + object_class = import_module(controller) + instance_class = object_class() + class_method = getattr(instance_class, action) + + return class_method + + def get_app(self): + return self._app diff --git a/requirements_dev.txt b/requirements_dev.txt index 0323040a..47b03ed9 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,3 +1,4 @@ +aiohttp==3.7.4.post0 aiokafka==0.7.0 aiomisc==12.1.0 aiopg==1.2.1 @@ -34,6 +35,7 @@ MarkupSafe==1.1.1 mccabe==0.6.1 minos-microservice-common==0.0.9 more-itertools==8.7.0 +multidict==5.1.0 mypy-extensions==0.4.3 orjson==3.5.1 packaging==20.9 @@ -78,4 +80,5 @@ virtualenv==20.4.3 watchdog==2.0.2 wcwidth==0.2.5 webencodings==0.5.1 +yarl==1.6.3 zipp==3.4.1 diff --git a/tests/services/TestRestService.py b/tests/services/TestRestService.py new file mode 100644 index 00000000..d578c004 --- /dev/null +++ b/tests/services/TestRestService.py @@ -0,0 +1,9 @@ +from aiohttp import web + + +class RestService(object): + async def add_order(self, request): + return web.Response(text="Order added") + + async def get_order(self, request): + return web.Response(text="Order get") diff --git a/tests/test_networks/rest_interface/__init__.py b/tests/test_networks/rest_interface/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_networks/rest_interface/test_interface.py b/tests/test_networks/rest_interface/test_interface.py new file mode 100644 index 00000000..b6103065 --- /dev/null +++ b/tests/test_networks/rest_interface/test_interface.py @@ -0,0 +1,32 @@ +from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop +from aiohttp import web +from minos.networks.rest_interface import RestInterfaceHandler +from minos.common.configuration.config import ( + MinosConfig, +) +from tests.utils import BASE_PATH + + +class MyAppTestCase(AioHTTPTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def get_application(self): + """ + Override the get_app method to return your application. + """ + rest_interface = RestInterfaceHandler(config=MinosConfig(self.CONFIG_FILE_PATH)) + + return rest_interface.get_app() + + @unittest_run_loop + async def test_methods(self): + url = "/order" + resp = await self.client.request("GET", url) + assert resp.status == 200 + text = await resp.text() + assert "Order get" in text + + resp = await self.client.request("POST", url) + assert resp.status == 200 + text = await resp.text() + assert "Order added" in text From 3ea378d2215a1f6b4a6a2325e10bb27cea935f9b Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 11 May 2021 17:43:48 +0200 Subject: [PATCH 229/239] Update handler.py --- minos/networks/rest_interface/handler.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/minos/networks/rest_interface/handler.py b/minos/networks/rest_interface/handler.py index 53a96bef..fa9ac261 100644 --- a/minos/networks/rest_interface/handler.py +++ b/minos/networks/rest_interface/handler.py @@ -30,12 +30,18 @@ def __init__(self, config: MinosConfig, app: web.Application = web.Application() self.load_routes() def load_routes(self): + """Load routes from config file.""" for item in self._config.rest.endpoints: callable_f = self.class_resolver(item.controller, item.action) self._app.router.add_route(item.method, item.route, callable_f) @staticmethod def class_resolver(controller: str, action: str): + """Load controller class and action method. + :param controller: Controller string. Example: "tests.service.CommandTestService.CommandService" + :param action: Config instance. Example: "get_order" + :return: A class method callable instance. + """ object_class = import_module(controller) instance_class = object_class() class_method = getattr(instance_class, action) @@ -43,4 +49,7 @@ def class_resolver(controller: str, action: str): return class_method def get_app(self): + """Return rest application instance. + :return: A `web.Application` instance. + """ return self._app From f07215ae83b2048394836914db612c220bcf539c Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Tue, 11 May 2021 17:45:57 +0200 Subject: [PATCH 230/239] Update test_config.yml --- tests/test_config.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_config.yml b/tests/test_config.yml index fb2f609f..810c6cd7 100644 --- a/tests/test_config.yml +++ b/tests/test_config.yml @@ -7,8 +7,13 @@ rest: - name: AddOrder route: /order method: POST - controller: minos.services.OrderService + controller: tests.services.TestRestService.RestService action: add_order + - name: GetOrder + route: /order + method: GET + controller: tests.services.TestRestService.RestService + action: get_order repository: database: order_db user: minos From a8209a69e8ff1804bd5db2985d7e7b92530e603a Mon Sep 17 00:00:00 2001 From: vladyslav-fenchak Date: Wed, 12 May 2021 13:37:21 +0200 Subject: [PATCH 231/239] Microservice REST Microservice REST #118 The REST service must have a default route called /system/health #119 --- minos/networks/__init__.py | 4 +++ minos/networks/rest_interface/__init__.py | 3 ++ minos/networks/rest_interface/handler.py | 14 ++++++++ minos/networks/rest_interface/service.py | 24 +++++++++++++ tests/test_config.yml | 2 +- .../rest_interface/test_interface.py | 9 ++--- .../rest_interface/test_service.py | 35 +++++++++++++++++++ 7 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 minos/networks/rest_interface/service.py create mode 100644 tests/test_networks/rest_interface/test_service.py diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index ff081f52..4ecff408 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -37,3 +37,7 @@ MinosSnapshotEntry, MinosSnapshotService, ) +from .rest_interface import ( + RestInterfaceHandler, + REST +) diff --git a/minos/networks/rest_interface/__init__.py b/minos/networks/rest_interface/__init__.py index 00be2a65..e7d9aeb0 100644 --- a/minos/networks/rest_interface/__init__.py +++ b/minos/networks/rest_interface/__init__.py @@ -9,3 +9,6 @@ from .handler import ( RestInterfaceHandler ) +from .service import ( + REST +) diff --git a/minos/networks/rest_interface/handler.py b/minos/networks/rest_interface/handler.py index fa9ac261..577422bf 100644 --- a/minos/networks/rest_interface/handler.py +++ b/minos/networks/rest_interface/handler.py @@ -35,6 +35,9 @@ def load_routes(self): callable_f = self.class_resolver(item.controller, item.action) self._app.router.add_route(item.method, item.route, callable_f) + # Load default routes + self._mount_system_health() + @staticmethod def class_resolver(controller: str, action: str): """Load controller class and action method. @@ -53,3 +56,14 @@ def get_app(self): :return: A `web.Application` instance. """ return self._app + + def _mount_system_health(self): + """Mount System Health Route.""" + self._app.router.add_get("/system/health", self._system_health_handler) + + @staticmethod + async def _system_health_handler(request): + """System Health Route Handler. + :return: A `web.json_response` response. + """ + return web.json_response({"host": request.host}, status=200) diff --git a/minos/networks/rest_interface/service.py b/minos/networks/rest_interface/service.py new file mode 100644 index 00000000..b229f8f1 --- /dev/null +++ b/minos/networks/rest_interface/service.py @@ -0,0 +1,24 @@ +import typing as t +from aiomisc.service.aiohttp import AIOHTTPService +from minos.common import MinosConfig + +from .handler import RestInterfaceHandler + + +class REST(AIOHTTPService): + """ + Rest Interface + + Expose REST Interface handler using aiomisc AIOHTTPService. + + """ + def __init__(self, config: MinosConfig, **kwds: t.Any): + address = config.rest.broker.host + port = config.rest.broker.port + super().__init__(address=address, port=port,**kwds) + self._config = config + self.rest_interface = RestInterfaceHandler(config=self._config) + + async def create_application(self): + return self.rest_interface.get_app() # pragma: no cover + diff --git a/tests/test_config.yml b/tests/test_config.yml index 810c6cd7..37f46669 100644 --- a/tests/test_config.yml +++ b/tests/test_config.yml @@ -2,7 +2,7 @@ service: name: Order rest: host: localhost - port: 8900 + port: 8080 endpoints: - name: AddOrder route: /order diff --git a/tests/test_networks/rest_interface/test_interface.py b/tests/test_networks/rest_interface/test_interface.py index b6103065..af061985 100644 --- a/tests/test_networks/rest_interface/test_interface.py +++ b/tests/test_networks/rest_interface/test_interface.py @@ -1,19 +1,15 @@ from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop -from aiohttp import web from minos.networks.rest_interface import RestInterfaceHandler from minos.common.configuration.config import ( MinosConfig, ) from tests.utils import BASE_PATH - -class MyAppTestCase(AioHTTPTestCase): +""" +class TestInterface(AioHTTPTestCase): CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" async def get_application(self): - """ - Override the get_app method to return your application. - """ rest_interface = RestInterfaceHandler(config=MinosConfig(self.CONFIG_FILE_PATH)) return rest_interface.get_app() @@ -30,3 +26,4 @@ async def test_methods(self): assert resp.status == 200 text = await resp.text() assert "Order added" in text +""" diff --git a/tests/test_networks/rest_interface/test_service.py b/tests/test_networks/rest_interface/test_service.py new file mode 100644 index 00000000..17b5acad --- /dev/null +++ b/tests/test_networks/rest_interface/test_service.py @@ -0,0 +1,35 @@ +from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop +from minos.networks import REST +from minos.common.configuration.config import ( + MinosConfig, +) +from tests.utils import BASE_PATH + + +class TestRestInterfaceService(AioHTTPTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def get_application(self): + """ + Override the get_app method to return your application. + """ + config = MinosConfig(self.CONFIG_FILE_PATH) + rest_interface = REST(config=config) + + return await rest_interface.create_application() + + @unittest_run_loop + async def test_methods(self): + url = "/order" + resp = await self.client.request("GET", url) + assert resp.status == 200 + text = await resp.text() + assert "Order get" in text + + resp = await self.client.request("POST", url) + assert resp.status == 200 + text = await resp.text() + assert "Order added" in text + + resp = await self.client.request("GET", "/system/health") + assert resp.status == 200 From 63fde755a8e64c0f5159fd8fdb0dc74a00643407 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 12 May 2021 11:37:37 +0000 Subject: [PATCH 232/239] Restyled by black --- minos/networks/__init__.py | 5 +---- minos/networks/rest_interface/__init__.py | 8 ++------ minos/networks/rest_interface/handler.py | 8 ++------ minos/networks/rest_interface/service.py | 4 ++-- tests/test_networks/rest_interface/test_interface.py | 4 +--- tests/test_networks/rest_interface/test_service.py | 4 +--- 6 files changed, 9 insertions(+), 24 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 4ecff408..c05abc18 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -37,7 +37,4 @@ MinosSnapshotEntry, MinosSnapshotService, ) -from .rest_interface import ( - RestInterfaceHandler, - REST -) +from .rest_interface import RestInterfaceHandler, REST diff --git a/minos/networks/rest_interface/__init__.py b/minos/networks/rest_interface/__init__.py index e7d9aeb0..f512e80e 100644 --- a/minos/networks/rest_interface/__init__.py +++ b/minos/networks/rest_interface/__init__.py @@ -6,9 +6,5 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .handler import ( - RestInterfaceHandler -) -from .service import ( - REST -) +from .handler import RestInterfaceHandler +from .service import REST diff --git a/minos/networks/rest_interface/handler.py b/minos/networks/rest_interface/handler.py index 577422bf..b6dd3d46 100644 --- a/minos/networks/rest_interface/handler.py +++ b/minos/networks/rest_interface/handler.py @@ -6,12 +6,8 @@ # permission of Clariteia SL. from aiohttp import web -from minos.common.configuration.config import ( - MinosConfig, -) -from minos.common.importlib import ( - import_module, -) +from minos.common.configuration.config import MinosConfig +from minos.common.importlib import import_module class RestInterfaceHandler: diff --git a/minos/networks/rest_interface/service.py b/minos/networks/rest_interface/service.py index b229f8f1..47df188d 100644 --- a/minos/networks/rest_interface/service.py +++ b/minos/networks/rest_interface/service.py @@ -12,13 +12,13 @@ class REST(AIOHTTPService): Expose REST Interface handler using aiomisc AIOHTTPService. """ + def __init__(self, config: MinosConfig, **kwds: t.Any): address = config.rest.broker.host port = config.rest.broker.port - super().__init__(address=address, port=port,**kwds) + super().__init__(address=address, port=port, **kwds) self._config = config self.rest_interface = RestInterfaceHandler(config=self._config) async def create_application(self): return self.rest_interface.get_app() # pragma: no cover - diff --git a/tests/test_networks/rest_interface/test_interface.py b/tests/test_networks/rest_interface/test_interface.py index af061985..8518d23d 100644 --- a/tests/test_networks/rest_interface/test_interface.py +++ b/tests/test_networks/rest_interface/test_interface.py @@ -1,8 +1,6 @@ from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop from minos.networks.rest_interface import RestInterfaceHandler -from minos.common.configuration.config import ( - MinosConfig, -) +from minos.common.configuration.config import MinosConfig from tests.utils import BASE_PATH """ diff --git a/tests/test_networks/rest_interface/test_service.py b/tests/test_networks/rest_interface/test_service.py index 17b5acad..f19826e4 100644 --- a/tests/test_networks/rest_interface/test_service.py +++ b/tests/test_networks/rest_interface/test_service.py @@ -1,8 +1,6 @@ from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop from minos.networks import REST -from minos.common.configuration.config import ( - MinosConfig, -) +from minos.common.configuration.config import MinosConfig from tests.utils import BASE_PATH From 740cbf9126ef9300e52ea2490012def58e1d77be Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 12 May 2021 11:37:41 +0000 Subject: [PATCH 233/239] Restyled by isort --- minos/networks/__init__.py | 5 ++++- minos/networks/rest_interface/__init__.py | 8 ++++++-- minos/networks/rest_interface/handler.py | 12 +++++++++--- minos/networks/rest_interface/service.py | 13 ++++++++++--- tests/services/TestRestService.py | 4 +++- .../rest_interface/test_interface.py | 17 +++++++++++++---- .../rest_interface/test_service.py | 17 +++++++++++++---- 7 files changed, 58 insertions(+), 18 deletions(-) diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index c05abc18..d01e6e92 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -32,9 +32,12 @@ MinosEventPeriodicService, MinosEventServerService, ) +from .rest_interface import ( + REST, + RestInterfaceHandler, +) from .snapshots import ( MinosSnapshotDispatcher, MinosSnapshotEntry, MinosSnapshotService, ) -from .rest_interface import RestInterfaceHandler, REST diff --git a/minos/networks/rest_interface/__init__.py b/minos/networks/rest_interface/__init__.py index f512e80e..524ba7a0 100644 --- a/minos/networks/rest_interface/__init__.py +++ b/minos/networks/rest_interface/__init__.py @@ -6,5 +6,9 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -from .handler import RestInterfaceHandler -from .service import REST +from .handler import ( + RestInterfaceHandler, +) +from .service import ( + REST, +) diff --git a/minos/networks/rest_interface/handler.py b/minos/networks/rest_interface/handler.py index b6dd3d46..9ba76668 100644 --- a/minos/networks/rest_interface/handler.py +++ b/minos/networks/rest_interface/handler.py @@ -5,9 +5,15 @@ # Minos framework can not be copied and/or distributed without the express # permission of Clariteia SL. -from aiohttp import web -from minos.common.configuration.config import MinosConfig -from minos.common.importlib import import_module +from aiohttp import ( + web, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.importlib import ( + import_module, +) class RestInterfaceHandler: diff --git a/minos/networks/rest_interface/service.py b/minos/networks/rest_interface/service.py index 47df188d..1a30e28c 100644 --- a/minos/networks/rest_interface/service.py +++ b/minos/networks/rest_interface/service.py @@ -1,8 +1,15 @@ import typing as t -from aiomisc.service.aiohttp import AIOHTTPService -from minos.common import MinosConfig -from .handler import RestInterfaceHandler +from aiomisc.service.aiohttp import ( + AIOHTTPService, +) +from minos.common import ( + MinosConfig, +) + +from .handler import ( + RestInterfaceHandler, +) class REST(AIOHTTPService): diff --git a/tests/services/TestRestService.py b/tests/services/TestRestService.py index d578c004..4b214303 100644 --- a/tests/services/TestRestService.py +++ b/tests/services/TestRestService.py @@ -1,4 +1,6 @@ -from aiohttp import web +from aiohttp import ( + web, +) class RestService(object): diff --git a/tests/test_networks/rest_interface/test_interface.py b/tests/test_networks/rest_interface/test_interface.py index 8518d23d..e1a46844 100644 --- a/tests/test_networks/rest_interface/test_interface.py +++ b/tests/test_networks/rest_interface/test_interface.py @@ -1,7 +1,16 @@ -from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop -from minos.networks.rest_interface import RestInterfaceHandler -from minos.common.configuration.config import MinosConfig -from tests.utils import BASE_PATH +from aiohttp.test_utils import ( + AioHTTPTestCase, + unittest_run_loop, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.networks.rest_interface import ( + RestInterfaceHandler, +) +from tests.utils import ( + BASE_PATH, +) """ class TestInterface(AioHTTPTestCase): diff --git a/tests/test_networks/rest_interface/test_service.py b/tests/test_networks/rest_interface/test_service.py index f19826e4..ecb13f2c 100644 --- a/tests/test_networks/rest_interface/test_service.py +++ b/tests/test_networks/rest_interface/test_service.py @@ -1,7 +1,16 @@ -from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop -from minos.networks import REST -from minos.common.configuration.config import MinosConfig -from tests.utils import BASE_PATH +from aiohttp.test_utils import ( + AioHTTPTestCase, + unittest_run_loop, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.networks import ( + REST, +) +from tests.utils import ( + BASE_PATH, +) class TestRestInterfaceService(AioHTTPTestCase): From 5e3fc9f347fd76babe9ec85454f83f4e00de1a27 Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Wed, 12 May 2021 14:38:27 +0200 Subject: [PATCH 234/239] modified authors, history, setup and init for new version --- AUTHORS.rst => AUTHORS.md | 4 ++-- HISTORY.md | 11 +++++++++-- minos/networks/__init__.py | 2 +- setup.py | 4 ++-- 4 files changed, 14 insertions(+), 7 deletions(-) rename AUTHORS.rst => AUTHORS.md (52%) diff --git a/AUTHORS.rst b/AUTHORS.md similarity index 52% rename from AUTHORS.rst rename to AUTHORS.md index 19183d0e..43c91703 100644 --- a/AUTHORS.rst +++ b/AUTHORS.md @@ -1,4 +1,3 @@ -======= Credits ======= @@ -10,4 +9,5 @@ Development Lead Contributors ------------ -None yet. Why not be the first? +* Sergio Garcia Prado +* Vladyslav Fenchak ` diff --git a/HISTORY.md b/HISTORY.md index 529ee21e..6b27a9fb 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,7 +1,14 @@ History ======= -0.0.1-alpha (2021-03-29) +0.0.1 (2021-03-29) ------------------ -* First release on PyPI. +* Added Event Handler +* Added Command Handler +* Added Event Recurring Service +* Added Command Recurring Service +* Added Event Broker with Recurrent Service +* Added Command Broker with Recurrent Service +* Added REST Service with Health handler + diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index d01e6e92..31f90883 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -5,7 +5,7 @@ Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. """ -__version__ = "0.0.1.1-alpha" +__version__ = "0.0.1" from .broker import ( MinosCommandBroker, diff --git a/setup.py b/setup.py index 86970804..8ad9ebb5 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( author="Andrea Mucci", author_email="andrea@clariteia.com", - python_requires=">=3.5", + python_requires=">=3.7", classifiers=[ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", @@ -49,6 +49,6 @@ setup_requires=setup_requirements, test_suite="tests", tests_require=test_requirements, - version="0.0.1-alpha", + version="0.0.1", zip_safe=False, ) From 39e5f9b1267683bf7110d342afc8cd388446c55e Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Wed, 12 May 2021 14:39:20 +0200 Subject: [PATCH 235/239] Update minos_microservice_networks.iml --- .idea/minos_microservice_networks.iml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.idea/minos_microservice_networks.iml b/.idea/minos_microservice_networks.iml index 765f3db6..de5fcde3 100644 --- a/.idea/minos_microservice_networks.iml +++ b/.idea/minos_microservice_networks.iml @@ -14,4 +14,4 @@ - + \ No newline at end of file From e9266a6bc040397c0d953756b778907886960f9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 12 May 2021 14:53:12 +0200 Subject: [PATCH 236/239] Delete minos_microservice_networks.iml --- .idea/minos_microservice_networks.iml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .idea/minos_microservice_networks.iml diff --git a/.idea/minos_microservice_networks.iml b/.idea/minos_microservice_networks.iml deleted file mode 100644 index de5fcde3..00000000 --- a/.idea/minos_microservice_networks.iml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file From ec8b9dadeb82bbe7a390e2e51caa534e52eef6eb Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Wed, 12 May 2021 15:08:49 +0200 Subject: [PATCH 237/239] removed whitespaces in README file --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index bc770c2c..6e13b102 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ Minos Microservice Networks Python Package with the common network classes and utlities used in Minos Microservice - Credits ------- From b04f452c0aca1a213aeb736bad81a957eb69ed85 Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Wed, 12 May 2021 16:22:17 +0200 Subject: [PATCH 238/239] Update HISTORY.md --- HISTORY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 6b27a9fb..71a32842 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,7 +1,7 @@ History ======= -0.0.1 (2021-03-29) +0.0.1 (2021-05-12) ------------------ * Added Event Handler From 99b680fe321690d7a17b0560d23e6583312cf606 Mon Sep 17 00:00:00 2001 From: andrea-mucci <81492948+andrea-mucci@users.noreply.github.com> Date: Wed, 12 May 2021 16:23:03 +0200 Subject: [PATCH 239/239] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e13b102..0f3492b6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Minos Microservice Networks =========================== -[![codecov](https://codecov.io/gh/Clariteia/minos_microservice_networks/branch/main/graph/badge.svg)](https://codecov.io/gh/Clariteia/minos_microservice_common) +[![codecov](https://codecov.io/gh/Clariteia/minos_microservice_networks/branch/main/graph/badge.svg)](https://codecov.io/gh/Clariteia/minos_microservice_networks) ![Tests](https://github.com/Clariteia/minos_microservice_networks/actions/workflows/python-tests.yml/badge.svg)