diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 95a2e1d5..9030e1fe 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -2,46 +2,71 @@ on: push jobs: build: + runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.9] - container: - image: python:latest + container: python:3.9-buster + + services: - zookeeper: - image: bitnami/zookeeper:latest - ports: - - "2181:2181" - kafka: - image: bitnami/kafka:latest - ports: - - "9092:9092" - env: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_HOST_NAME: localhost + postgres: + image: postgres + env: + POSTGRES_USER: minos + POSTGRES_PASSWORD: min0s + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + zookeeper: + image: wurstmeister/zookeeper:latest + ports: + - 2181:2181 + + kafka: + image: wurstmeister/kafka:latest + ports: + - 9092:9092 + env: + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_HOST_NAME: kafka + env: + MINOS_EVENTS_QUEUE_HOST: postgres + MINOS_EVENTS_BROKER: kafka + MINOS_COMMANDS_QUEUE_HOST: postgres + MINOS_COMMANDS_BROKER: kafka + MINOS_REPOSITORY_HOST: postgres + MINOS_SNAPSHOT_HOST: postgres + steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - apt-get install -y libc6 build-essential + - name: Check out repository code + uses: actions/checkout@v2 + + - name: Install dependencies + run: | python -m pip install --upgrade pip if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi python setup.py install - - name: Test with pytest - run: | + + - name: Install Postgresql client + run: | + apt-get update + apt-get install --yes postgresql-client + + - name: Create database + run: | + PGPASSWORD=min0s psql -U minos -h postgres -tc "CREATE DATABASE order_db OWNER minos;" + + - name: Test with pytest + run: | make test - - name: Generate coverage report + + - name: Generate coverage report run: | make coverage - - name: Codecov + + - name: Codecov uses: codecov/codecov-action@v1.3.1 with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + fail_ci_if_error: true diff --git a/.gitignore b/.gitignore index 43091aa9..23ae5f8a 100644 --- a/.gitignore +++ b/.gitignore @@ -102,4 +102,7 @@ ENV/ .mypy_cache/ # IDE settings -.vscode/ \ No newline at end of file +.vscode/ + +# Intellij IDEa / PyCharm / etc. +.idea diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 73f69e09..00000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 06f7730f..00000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2da..00000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/minos_microservice_networks.iml b/.idea/minos_microservice_networks.iml deleted file mode 100644 index de5fcde3..00000000 --- a/.idea/minos_microservice_networks.iml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 1cc5b395..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 70bc1b36..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.restyled.yaml b/.restyled.yaml new file mode 100644 index 00000000..e8e0a9d3 --- /dev/null +++ b/.restyled.yaml @@ -0,0 +1,28 @@ +--- +enabled: true +exclude: + - "**/*.md" + - ".idea/**/*" + - "docs/**/*" + - "**/*.in" + - "Makefile" + - ".github/workflows/**/*" +restylers: + - name: black + image: restyled/restyler-black:v19.10b0 + command: + - black + arguments: ["--line-length", "120"] + include: + - "**/*.py" + interpreters: + - python + - name: isort + image: restyled/restyler-isort:v4.3.21 + command: + - isort + arguments: [] + include: + - "**/*.py" + interpreters: + - python diff --git a/AUTHORS.rst b/AUTHORS.md similarity index 52% rename from AUTHORS.rst rename to AUTHORS.md index 19183d0e..43c91703 100644 --- a/AUTHORS.rst +++ b/AUTHORS.md @@ -1,4 +1,3 @@ -======= Credits ======= @@ -10,4 +9,5 @@ Development Lead Contributors ------------ -None yet. Why not be the first? +* Sergio Garcia Prado +* Vladyslav Fenchak ` diff --git a/HISTORY.md b/HISTORY.md new file mode 100644 index 00000000..71a32842 --- /dev/null +++ b/HISTORY.md @@ -0,0 +1,14 @@ +History +======= + +0.0.1 (2021-05-12) +------------------ + +* Added Event Handler +* Added Command Handler +* Added Event Recurring Service +* Added Command Recurring Service +* Added Event Broker with Recurrent Service +* Added Command Broker with Recurrent Service +* Added REST Service with Health handler + diff --git a/HISTORY.rst b/HISTORY.rst deleted file mode 100644 index d32b71d6..00000000 --- a/HISTORY.rst +++ /dev/null @@ -1,8 +0,0 @@ -======= -History -======= - -0.0.1-alpha (2021-03-29) ------------------- - -* First release on PyPI. diff --git a/Makefile b/Makefile index bc9d0a2d..f28a0731 100644 --- a/Makefile +++ b/Makefile @@ -59,8 +59,8 @@ test-all: ## run tests on every Python version with tox coverage: ## check code coverage quickly with the default Python coverage run --source minos -m pytest coverage report -m - coverage html - $(BROWSER) htmlcov/index.html + coverage xml + ## $(BROWSER) htmlcov/index.html docs: ## generate Sphinx HTML documentation, including API docs rm -f docs/minos_microservice_networks.rst diff --git a/README.md b/README.md new file mode 100644 index 00000000..0f3492b6 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +Minos Microservice Networks +=========================== + +[![codecov](https://codecov.io/gh/Clariteia/minos_microservice_networks/branch/main/graph/badge.svg)](https://codecov.io/gh/Clariteia/minos_microservice_networks) + +![Tests](https://github.com/Clariteia/minos_microservice_networks/actions/workflows/python-tests.yml/badge.svg) + +Python Package with the common network classes and utlities used in Minos Microservice + + +Credits +------- + +This package was created with ![Cookiecutter](https://github.com/audreyr/cookiecutter) +and the ![Minos Package](https://github.com/Clariteia/minos-pypackage) project template. diff --git a/README.rst b/README.rst deleted file mode 100644 index 15b60fc3..00000000 --- a/README.rst +++ /dev/null @@ -1,18 +0,0 @@ -=========================== -Minos Microservice Networks -=========================== - -Python Package with the common network classes and utlities used in Minos Microservice - -Features --------- - -* TODO - -Credits -------- - -This package was created with Cookiecutter_ and the `clariteia/minos-pypackage`_ project template. - -.. _Cookiecutter: https://github.com/audreyr/cookiecutter -.. _`clariteia/minos-pypackage`: https://bitbucket.org/clariteia-devs/minos-pypackage/src/master/ diff --git a/minos/networks/__init__.py b/minos/networks/__init__.py index 57ffe7ee..31f90883 100644 --- a/minos/networks/__init__.py +++ b/minos/networks/__init__.py @@ -1 +1,43 @@ -__version__ = '0.0.1.1-alpha' +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +__version__ = "0.0.1" + +from .broker import ( + MinosCommandBroker, + MinosEventBroker, + MinosQueueDispatcher, + MinosQueueService, +) +from .exceptions import ( + MinosNetworkException, + MinosPreviousVersionSnapshotException, + MinosSnapshotException, +) +from .handler import ( + MinosCommandHandlerDispatcher, + MinosCommandHandlerServer, + MinosCommandPeriodicService, + MinosCommandReplyHandlerDispatcher, + MinosCommandReplyHandlerServer, + MinosCommandReplyPeriodicService, + MinosCommandReplyServerService, + MinosCommandServerService, + MinosEventHandlerDispatcher, + MinosEventHandlerServer, + MinosEventPeriodicService, + MinosEventServerService, +) +from .rest_interface import ( + REST, + RestInterfaceHandler, +) +from .snapshots import ( + MinosSnapshotDispatcher, + MinosSnapshotEntry, + MinosSnapshotService, +) diff --git a/minos/networks/broker/__init__.py b/minos/networks/broker/__init__.py new file mode 100644 index 00000000..e229ea70 --- /dev/null +++ b/minos/networks/broker/__init__.py @@ -0,0 +1,19 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from .commands import ( + MinosCommandBroker, +) +from .dispatchers import ( + MinosQueueDispatcher, +) +from .events import ( + MinosEventBroker, +) +from .services import ( + MinosQueueService, +) diff --git a/minos/networks/broker/abc.py b/minos/networks/broker/abc.py new file mode 100644 index 00000000..b2c01d9e --- /dev/null +++ b/minos/networks/broker/abc.py @@ -0,0 +1,65 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from abc import ( + ABC, +) +from datetime import ( + datetime, +) +from typing import ( + NoReturn, +) + +from minos.common import ( + MinosBaseBroker, + PostgreSqlMinosDatabase, +) + + +class MinosBrokerSetup(PostgreSqlMinosDatabase): + """Minos Broker Setup Class""" + + async def _setup(self) -> NoReturn: + await self._create_broker_table() + + async def _create_broker_table(self) -> NoReturn: + await self.submit_query(_CREATE_TABLE_QUERY) + + +class MinosBroker(MinosBaseBroker, MinosBrokerSetup, ABC): + """Minos Broker Class.""" + + ACTION: str + + def __init__(self, topic: str, *args, **kwargs): + MinosBaseBroker.__init__(self, topic) + MinosBrokerSetup.__init__(self, *args, **kwargs) + + async def _send_bytes(self, topic: str, raw: bytes) -> int: + params = (topic, raw, 0, self.ACTION, datetime.now(), datetime.now()) + raw = await self.submit_query_and_fetchone(_INSERT_ENTRY_QUERY, params) + return raw[0] + + +_CREATE_TABLE_QUERY = """ +CREATE TABLE IF NOT EXISTS producer_queue ( + id BIGSERIAL NOT NULL PRIMARY KEY, + topic VARCHAR(255) NOT NULL, + model BYTEA NOT NULL, + retry INTEGER NOT NULL, + action VARCHAR(255) NOT NULL, + creation_date TIMESTAMP NOT NULL, + update_date TIMESTAMP NOT NULL +); +""".strip() + +_INSERT_ENTRY_QUERY = """ +INSERT INTO producer_queue (topic, model, retry, action, creation_date, update_date) +VALUES (%s, %s, %s, %s, %s, %s) +RETURNING id; +""".strip() diff --git a/minos/networks/broker/commands.py b/minos/networks/broker/commands.py new file mode 100644 index 00000000..0357893c --- /dev/null +++ b/minos/networks/broker/commands.py @@ -0,0 +1,61 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from __future__ import ( + annotations, +) + +from typing import ( + NoReturn, + Optional, +) + +from minos.common import ( + Aggregate, + Command, + MinosConfig, +) + +from .abc import ( + MinosBroker, +) + + +class MinosCommandBroker(MinosBroker): + """Minos Command Broker Class.""" + + ACTION = "command" + + def __init__(self, *args, saga_id: str, task_id: str, reply_on: str, **kwargs): + super().__init__(*args, **kwargs) + self.reply_on = reply_on + self.saga_id = saga_id + self.task_id = task_id + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosCommandBroker]: + """Build a new repository from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + return None + # noinspection PyProtectedMember + return cls(*args, **config.commands.queue._asdict(), **kwargs) + + async def send(self, items: list[Aggregate]) -> int: + """Send a list of ``Aggregate`` instances. + + :param items: A list of aggregates. + :return: This method does not return anything. + """ + command = Command(self.topic, items, self.saga_id, self.task_id, self.reply_on) + return await self._send_bytes(command.topic, command.avro_bytes) diff --git a/minos/networks/broker/dispatchers.py b/minos/networks/broker/dispatchers.py new file mode 100644 index 00000000..2b358b87 --- /dev/null +++ b/minos/networks/broker/dispatchers.py @@ -0,0 +1,128 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from __future__ import ( + annotations, +) + +from typing import ( + AsyncIterator, + NamedTuple, + NoReturn, + Optional, +) + +from aiokafka import ( + AIOKafkaProducer, +) +from minos.common import ( + MinosConfig, + MinosConfigException, +) + +from .abc import ( + MinosBrokerSetup, +) + + +class MinosQueueDispatcher(MinosBrokerSetup): + """Minos Queue Dispatcher Class.""" + + # noinspection PyUnresolvedReferences + def __init__(self, *args, queue: NamedTuple, broker, **kwargs): + # noinspection PyProtectedMember + super().__init__(*args, **queue._asdict(), **kwargs) + self.retry = queue.retry + self.records = queue.records + self.broker = broker + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosQueueDispatcher]: + """Build a new repository from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + raise MinosConfigException("The config object must be setup.") + # noinspection PyProtectedMember + return cls(*args, **config.events._asdict(), **kwargs) + + async def dispatch(self) -> NoReturn: + """Dispatch the items in the publishing queue. + + :return: This method does not return anything. + """ + async for row in self.select(): + await self._dispatch_one(row) + + # noinspection PyUnusedLocal + async def select(self, *args, **kwargs) -> AsyncIterator[tuple]: + """Select a sequence of ``MinosSnapshotEntry`` objects. + + :param args: Additional positional arguments. + :param kwargs: Additional named arguments. + :return: A sequence of ``MinosSnapshotEntry`` objects. + """ + params = (self.retry, self.records) + async for row in self.submit_query_and_iter(_SELECT_NON_PROCESSED_ROWS_QUERY, params): + yield row + + async def _dispatch_one(self, row: tuple) -> NoReturn: + # noinspection PyBroadException + published = await self.publish(topic=row[1], message=row[2]) + await self._update_queue_state(row, published) + + async def _update_queue_state(self, row: tuple, published: bool): + update_query = _DELETE_PROCESSED_QUERY if published else _UPDATE_NON_PROCESSED_QUERY + await self.submit_query(update_query, (row[0],)) + + async def publish(self, topic: str, message: bytes) -> bool: + """Publish a new item in in the broker (kafka). + + :param topic: The topic in which the message will be published. + :param message: The message to be published. + :return: A boolean flag, ``True`` when the message is properly published or ``False`` otherwise. + """ + producer = AIOKafkaProducer(bootstrap_servers=f"{self.broker.host}:{self.broker.port}") + # noinspection PyBroadException + try: + # Get cluster layout and initial topic/partition leadership information + await producer.start() + # Produce message + await producer.send_and_wait(topic, message) + published = True + except Exception: + published = False + finally: + # Wait for all pending messages to be delivered or expire. + await producer.stop() + + return published + + +_SELECT_NON_PROCESSED_ROWS_QUERY = """ +SELECT * +FROM producer_queue +WHERE retry <= %s +ORDER BY creation_date +LIMIT %s; +""".strip() + +_DELETE_PROCESSED_QUERY = """ +DELETE FROM producer_queue +WHERE id = %s; +""".strip() + +_UPDATE_NON_PROCESSED_QUERY = """ +UPDATE producer_queue + SET retry = retry + 1 +WHERE id = %s; +""".strip() diff --git a/minos/networks/broker/events.py b/minos/networks/broker/events.py new file mode 100644 index 00000000..b19b587a --- /dev/null +++ b/minos/networks/broker/events.py @@ -0,0 +1,54 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from __future__ import ( + annotations, +) + +from typing import ( + Optional, +) + +from minos.common import ( + Aggregate, + Event, + MinosConfig, +) + +from .abc import ( + MinosBroker, +) + + +class MinosEventBroker(MinosBroker): + """Minos Event broker class.""" + + ACTION = "event" + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosEventBroker]: + """Build a new repository from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + return None + # noinspection PyProtectedMember + return cls(*args, **config.events.queue._asdict(), **kwargs) + + async def send(self, items: list[Aggregate]) -> int: + """Send a list of ``Aggregate`` instances. + + :param items: A list of aggregates. + :return: This method does not return anything. + """ + event = Event(self.topic, items) + return await self._send_bytes(event.topic, event.avro_bytes) diff --git a/minos/networks/broker/services.py b/minos/networks/broker/services.py new file mode 100644 index 00000000..39e808c9 --- /dev/null +++ b/minos/networks/broker/services.py @@ -0,0 +1,53 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from __future__ import ( + annotations, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosQueueDispatcher, +) + + +class MinosQueueService(PeriodicService): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosQueueDispatcher.from_config(config=config) + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await self.dispatcher.setup() + await super().start() + + async def callback(self) -> None: + """Method to be called periodically by the internal ``aiomisc`` logic. + + :return:This method does not return anything. + """ + await self.dispatcher.dispatch() + + async def stop(self, err: Exception = None) -> None: + """Stop the service execution. + + :param err: Optional exception that stopped the execution. + :return: This method does not return anything. + """ + await super().stop(err) + await self.dispatcher.destroy() diff --git a/minos/networks/event.py b/minos/networks/event.py deleted file mode 100644 index 9dcaaf52..00000000 --- a/minos/networks/event.py +++ /dev/null @@ -1,179 +0,0 @@ -import asyncio -import functools -import typing as t -from collections import Counter - -from aiokafka import AIOKafkaConsumer, ConsumerRebalanceListener -from aiomisc import Service -from kafka.errors import OffsetOutOfRangeError -from minos.common.configuration.config import MinosConfig -from minos.common.importlib import import_module -from minos.common.logs import log -from minos.common.storage.abstract import MinosStorage -from minos.common.storage.lmdb import MinosStorageLmdb - -from minos.networks.exceptions import MinosNetworkException - - -class MinosLocalState: - __slots__ = "_counts", "_offsets", "_storage", "_dbname" - - def __init__(self, storage: MinosStorage, db_name="LocalState"): - self._counts = {} - self._offsets = {} - self._dbname = db_name - self._storage = storage - - def dump_local_state(self): - - for tp in self._counts: - key = f"{tp.topic}:{tp.partition}" - state = { - "last_offset": self._offsets[tp], - "counts": dict(self._counts[tp]) - } - actual_state = self._storage.get(self._dbname, key) - if actual_state is not None: - self._storage.update(self._dbname, key, state) - else: - self._storage.add(self._dbname, key, state) - - def load_local_state(self, partitions): - self._counts.clear() - self._offsets.clear() - for tp in partitions: - # prepare the default state - state = { - "last_offset": -1, # Non existing, will reset - "counts": {} - } - key = f"{tp.topic}:{tp.partition}" - returned_val = self._storage.get(self._dbname, key) - if returned_val is not None: - state = returned_val - self._counts[tp] = Counter(state['counts']) - self._offsets[tp] = state['last_offset'] - - def discard_state(self, tps): - """ - reset the entire memory database with the default values - """ - for tp in tps: - self._offsets[tp] = -1 - self._counts[tp] = Counter() - - def get_last_offset(self, tp): - return self._offsets[tp] - - def add_counts(self, tp, counts, las_offset): - self._counts[tp] += counts - self._offsets[tp] = las_offset - - -class MinosRebalanceListener(ConsumerRebalanceListener): - - def __init__(self, consumer, database_state: MinosLocalState): - self._consumer = consumer - self._state = database_state - - async def on_partitions_revoked(self, revoked): - log.debug("KAFKA Consumer: Revoked %s", revoked) - self._state.dump_local_state() - - async def on_partitions_assigned(self, assigned): - log.debug("KAFKA Consumer: Assigned %s", assigned) - self._state.load_local_state(partitions=assigned) - for tp in assigned: - last_offset = self._state.get_last_offset(tp) - if last_offset < 0: - await self._consumer.seek_to_beginning(tp) - else: - self._consumer.seek(tp, last_offset + 1) - - -class MinosEventServer(Service): - """ - Event Manager - - Consumer for the Broker ( at the moment only Kafka is supported ) - - """ - __slots__ = "_tasks", "_local_state", "_storage", "_handlers", "_broker_host", "_broker_port", "_broker_group" - - def __init__(self, *, conf: MinosConfig, storage: MinosStorage = MinosStorageLmdb, **kwargs: t.Any): - self._tasks = set() # type: t.Set[asyncio.Task] - self._broker_host = conf.events.broker.host - self._broker_port = conf.events.broker.port - self._broker_group = f"{conf.service.name}_group_id" - self._handler = conf.events.items - self._storage = storage.build(conf.events.database.path) - self._local_state = MinosLocalState(storage=self._storage, db_name=conf.events.database.name) - - super().__init__(**kwargs) - - def create_task(self, coro: t.Awaitable[t.Any]): - task = self.loop.create_task(coro) - self._tasks.add(task) - task.add_done_callback(self._tasks.remove) - - async def save_state_every_second(self): - while True: - try: - await asyncio.sleep(1) - except asyncio.CancelledError: - break - self._local_state.dump_local_state() - - async def handle_message(self, consumer: t.Any): - while True: - try: - msg_set = await consumer.getmany(timeout_ms=1000) - except OffsetOutOfRangeError as err: - # this is the case that the database is not updated and must be reset - tps = err.args[0].keys() - self._local_state.discard_state(tps) - await consumer.seek_to_beginning(*tps) - continue - for tp, msgs in msg_set.items(): - log.debug(f"EVENT Manager topic: {tp}") - log.debug(f"EVENT Manager msg: {msgs}") - # check if the topic is managed by the handler - if tp.topic in self._handlers: - counts = Counter() - for msg in msgs: - counts[msg.key] += 1 - func = self._get_event_handler(tp.topic) - func(msg.value) - self._local_state.add_counts(tp, counts, msg.offset) - - async def start(self) -> t.Any: - self.start_event.set() - log.debug("Event Consumer Manager: Started") - # start the Service Event Consumer for Kafka - consumer = AIOKafkaConsumer(loop=self.loop, - enable_auto_commit=False, - auto_offset_reset="none", - group_id=self._broker_group, - bootstrap_servers=f"{self._broker_host}:{self._broker_port}", - key_deserializer=lambda key: key.decode("utf-8") if key else "", ) - - await consumer.start() - # prepare the database interface - listener = MinosRebalanceListener(consumer, self._local_state) - topics: t.List = list(self._handlers.keys()) - consumer.subscribe(topics=topics, listener=listener) - - self.create_task(self.save_state_every_second()) - self.create_task(self.handle_message(consumer)) - - def _get_event_handler(self, topic: str) -> t.Callable: - for event in self._handlers: - if event.name == topic: - # the topic exist, get the controller and the action - controller = event.controller - action = event.action - object_class = import_module(controller) - instance_class = object_class() - return functools.partial(instance_class.action) - raise MinosNetworkException(f"topic {topic} have no controller/action configured, " - f"please review th configuration file") diff --git a/minos/networks/exceptions.py b/minos/networks/exceptions.py index 8e573e1f..593d3924 100644 --- a/minos/networks/exceptions.py +++ b/minos/networks/exceptions.py @@ -1,4 +1,31 @@ -from minos.common.exceptions import MinosException +""" +Copyright (C) 2021 Clariteia SL +This file is part of minos framework. -class MinosNetworkException(MinosException): pass +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from minos.common import ( + Aggregate, + MinosException, +) + + +class MinosNetworkException(MinosException): + """Base network exception.""" + + +class MinosSnapshotException(MinosNetworkException): + """Base snapshot exception""" + + +class MinosPreviousVersionSnapshotException(MinosSnapshotException): + """Exception to be raised when current version is newer than the one to be processed.""" + + def __init__(self, previous: Aggregate, new: Aggregate): + self.previous = previous + self.new = new + super().__init__( + f"Version for {repr(previous.classname)} aggregate must be " + f"greater than {previous.version}. Obtained: {new.version}" + ) diff --git a/minos/networks/handler/__init__.py b/minos/networks/handler/__init__.py new file mode 100644 index 00000000..f3314979 --- /dev/null +++ b/minos/networks/handler/__init__.py @@ -0,0 +1,26 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from .command import ( + MinosCommandHandlerDispatcher, + MinosCommandHandlerServer, + MinosCommandPeriodicService, + MinosCommandServerService, +) +from .command_reply import ( + MinosCommandReplyHandlerDispatcher, + MinosCommandReplyHandlerServer, + MinosCommandReplyPeriodicService, + MinosCommandReplyServerService, +) +from .event import ( + MinosEventHandlerDispatcher, + MinosEventHandlerServer, + MinosEventPeriodicService, + MinosEventServerService, +) diff --git a/minos/networks/handler/abc.py b/minos/networks/handler/abc.py new file mode 100644 index 00000000..4b6d37a7 --- /dev/null +++ b/minos/networks/handler/abc.py @@ -0,0 +1,65 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from abc import ( + ABC, +) +from datetime import ( + datetime, +) +from typing import ( + NoReturn, +) + +import aiopg +from minos.common import ( + MinosSetup, +) + + +class MinosHandlerSetup(MinosSetup): + """Minos Broker Setup Class""" + + def __init__( + self, + table_name: str, + *args, + host: str = None, + port: int = None, + database: str = None, + user: str = None, + password: str = None, + **kwargs + ): + super().__init__(*args, **kwargs) + + self.host = host + self.port = port + self.database = database + self.user = user + self.password = password + self.table_name = table_name + + async def _setup(self) -> NoReturn: + await self._create_event_queue_table() + + async def _create_event_queue_table(self) -> NoReturn: + async with self._connection() as connect: + async with connect.cursor() as cur: + await cur.execute( + 'CREATE TABLE IF NOT EXISTS "%s" (' + '"id" BIGSERIAL NOT NULL PRIMARY KEY, ' + '"topic" VARCHAR(255) NOT NULL, ' + '"partition_id" INTEGER,' + '"binary_data" BYTEA NOT NULL, ' + '"creation_date" TIMESTAMP NOT NULL);' % (self.table_name) + ) + + def _connection(self): + return aiopg.connect( + host=self.host, port=self.port, dbname=self.database, user=self.user, password=self.password, + ) diff --git a/minos/networks/handler/command/__init__.py b/minos/networks/handler/command/__init__.py new file mode 100644 index 00000000..d54f41b7 --- /dev/null +++ b/minos/networks/handler/command/__init__.py @@ -0,0 +1,18 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from .dispatcher import ( + MinosCommandHandlerDispatcher, +) +from .server import ( + MinosCommandHandlerServer, +) +from .services import ( + MinosCommandPeriodicService, + MinosCommandServerService, +) diff --git a/minos/networks/handler/command/dispatcher.py b/minos/networks/handler/command/dispatcher.py new file mode 100644 index 00000000..9e4b2927 --- /dev/null +++ b/minos/networks/handler/command/dispatcher.py @@ -0,0 +1,37 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from typing import ( + Any, +) + +from minos.common import ( + Command, +) +from minos.common.configuration.config import ( + MinosConfig, +) + +from ..dispatcher import ( + MinosHandlerDispatcher, +) + + +class MinosCommandHandlerDispatcher(MinosHandlerDispatcher): + + TABLE = "command_queue" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(table_name=self.TABLE, config=config.commands, **kwargs) + self._broker_group_name = f"event_{config.service.name}" + + def _is_valid_instance(self, value: bytes): + try: + instance = Command.from_avro_bytes(value) + return True, instance + except: + return False, None diff --git a/minos/networks/handler/command/server.py b/minos/networks/handler/command/server.py new file mode 100644 index 00000000..a8cd2309 --- /dev/null +++ b/minos/networks/handler/command/server.py @@ -0,0 +1,38 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from typing import ( + Any, +) + +from minos.common import ( + Command, +) +from minos.common.configuration.config import ( + MinosConfig, +) + +from ..server import ( + MinosHandlerServer, +) + + +class MinosCommandHandlerServer(MinosHandlerServer): + + TABLE = "command_queue" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(table_name=self.TABLE, config=config.commands, **kwargs) + self._kafka_conn_data = f"{config.commands.broker.host}:{config.commands.broker.port}" + self._broker_group_name = f"event_{config.service.name}" + + def _is_valid_instance(self, value: bytes): + try: + Command.from_avro_bytes(value) + return True + except: + return False diff --git a/minos/networks/handler/command/services.py b/minos/networks/handler/command/services.py new file mode 100644 index 00000000..7c5074ac --- /dev/null +++ b/minos/networks/handler/command/services.py @@ -0,0 +1,74 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from typing import ( + Any, +) + +from aiomisc.service.periodic import ( + PeriodicService, + Service, +) +from minos.common import ( + MinosConfig, +) + +from .dispatcher import ( + MinosCommandHandlerDispatcher, +) +from .server import ( + MinosCommandHandlerServer, +) + + +class MinosCommandServerService(Service): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosCommandHandlerServer.from_config(config=config) + self.consumer = None + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await self.dispatcher.setup() + + self.consumer = await self.dispatcher.kafka_consumer( + self.dispatcher._topics, self.dispatcher._broker_group_name, self.dispatcher._kafka_conn_data + ) + await self.dispatcher.handle_message(self.consumer) + + async def stop(self, exception: Exception = None) -> Any: + if self.consumer is not None: + await self.consumer.stop() + + +class MinosCommandPeriodicService(PeriodicService): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosCommandHandlerDispatcher.from_config(config=config) + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await super().start() + await self.dispatcher.setup() + + async def callback(self) -> None: + """Method to be called periodically by the internal ``aiomisc`` logic. + + :return:This method does not return anything. + """ + await self.dispatcher.queue_checker() diff --git a/minos/networks/handler/command_reply/__init__.py b/minos/networks/handler/command_reply/__init__.py new file mode 100644 index 00000000..8bba3430 --- /dev/null +++ b/minos/networks/handler/command_reply/__init__.py @@ -0,0 +1,18 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from .dispatcher import ( + MinosCommandReplyHandlerDispatcher, +) +from .server import ( + MinosCommandReplyHandlerServer, +) +from .services import ( + MinosCommandReplyPeriodicService, + MinosCommandReplyServerService, +) diff --git a/minos/networks/handler/command_reply/dispatcher.py b/minos/networks/handler/command_reply/dispatcher.py new file mode 100644 index 00000000..3f611524 --- /dev/null +++ b/minos/networks/handler/command_reply/dispatcher.py @@ -0,0 +1,37 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. +import collections +from typing import ( + Any, +) + +from minos.common import ( + CommandReply, +) +from minos.common.configuration.config import ( + MinosConfig, +) + +from ..dispatcher import ( + MinosHandlerDispatcher, +) + + +class MinosCommandReplyHandlerDispatcher(MinosHandlerDispatcher): + + TABLE = "command_reply_queue" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(table_name=self.TABLE, config=config.saga, **kwargs) + self._broker_group_name = f"event_{config.service.name}" + + def _is_valid_instance(self, value: bytes): + try: + instance = CommandReply.from_avro_bytes(value) + return True, instance + except: + return False, None diff --git a/minos/networks/handler/command_reply/server.py b/minos/networks/handler/command_reply/server.py new file mode 100644 index 00000000..337a35ac --- /dev/null +++ b/minos/networks/handler/command_reply/server.py @@ -0,0 +1,38 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from typing import ( + Any, +) + +from minos.common import ( + CommandReply, +) +from minos.common.configuration.config import ( + MinosConfig, +) + +from ..server import ( + MinosHandlerServer, +) + + +class MinosCommandReplyHandlerServer(MinosHandlerServer): + + TABLE = "command_reply_queue" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(table_name=self.TABLE, config=config.saga, **kwargs) + self._kafka_conn_data = f"{config.commands.broker.host}:{config.commands.broker.port}" + self._broker_group_name = f"event_{config.service.name}" + + def _is_valid_instance(self, value: bytes): + try: + CommandReply.from_avro_bytes(value) + return True + except: + return False diff --git a/minos/networks/handler/command_reply/services.py b/minos/networks/handler/command_reply/services.py new file mode 100644 index 00000000..1fe7ced2 --- /dev/null +++ b/minos/networks/handler/command_reply/services.py @@ -0,0 +1,74 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from typing import ( + Any, +) + +from aiomisc.service.periodic import ( + PeriodicService, + Service, +) +from minos.common import ( + MinosConfig, +) + +from .dispatcher import ( + MinosCommandReplyHandlerDispatcher, +) +from .server import ( + MinosCommandReplyHandlerServer, +) + + +class MinosCommandReplyServerService(Service): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosCommandReplyHandlerServer.from_config(config=config) + self.consumer = None + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await self.dispatcher.setup() + + self.consumer = await self.dispatcher.kafka_consumer( + self.dispatcher._topics, self.dispatcher._broker_group_name, self.dispatcher._kafka_conn_data + ) + await self.dispatcher.handle_message(self.consumer) + + async def stop(self, exception: Exception = None) -> Any: + if self.consumer is not None: + await self.consumer.stop() + + +class MinosCommandReplyPeriodicService(PeriodicService): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosCommandReplyHandlerDispatcher.from_config(config=config) + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await super().start() + await self.dispatcher.setup() + + async def callback(self) -> None: + """Method to be called periodically by the internal ``aiomisc`` logic. + + :return:This method does not return anything. + """ + await self.dispatcher.queue_checker() diff --git a/minos/networks/handler/dispatcher.py b/minos/networks/handler/dispatcher.py new file mode 100644 index 00000000..76d0d16e --- /dev/null +++ b/minos/networks/handler/dispatcher.py @@ -0,0 +1,144 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from __future__ import ( + annotations, +) + +from abc import ( + abstractmethod, +) +from typing import ( + Any, + Callable, + NamedTuple, + NoReturn, + Optional, +) + +import aiopg +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.importlib import ( + import_module, +) +from minos.common.logs import ( + log, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) +from minos.networks.handler.abc import ( + MinosHandlerSetup, +) + + +class MinosHandlerDispatcher(MinosHandlerSetup): + """ + Event Handler + + """ + + __slots__ = "_db_dsn", "_handlers", "_event_items", "_topics", "_conf" + + def __init__(self, *, table_name: str, config: NamedTuple, **kwargs: Any): + super().__init__(table_name=table_name, **kwargs, **config.queue._asdict()) + self._db_dsn = ( + f"dbname={config.queue.database} user={config.queue.user} " + f"password={config.queue.password} host={config.queue.host}" + ) + self._handlers = {item.name: {"controller": item.controller, "action": item.action} for item in config.items} + self._event_items = config.items + self._topics = list(self._handlers.keys()) + self._conf = config + self._table_name = table_name + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosHandlerDispatcher]: + """Build a new repository from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + return None + # noinspection PyProtectedMember + return cls(*args, config=config, **kwargs) + + def get_event_handler(self, topic: str) -> Callable: + + """Get Event instance to call. + + Gets the instance of the class and method to call. + + Args: + topic: Kafka topic. Example: "TicketAdded" + + Raises: + MinosNetworkException: topic TicketAdded have no controller/action configured, please review th configuration file. + """ + for event in self._event_items: + if event.name == topic: + # the topic exist, get the controller and the action + controller = event.controller + action = event.action + + object_class = import_module(controller) + log.debug(object_class()) + instance_class = object_class() + class_method = getattr(instance_class, action) + + return class_method + raise MinosNetworkException( + f"topic {topic} have no controller/action configured, " f"please review th configuration file" + ) + + async def queue_checker(self) -> NoReturn: + """Event Queue Checker and dispatcher. + + It is in charge of querying the database and calling the action according to the topic. + + 1. Get periodically 10 records (or as many as defined in config > queue > records). + 2. Instantiate the action (asynchronous) by passing it the model. + 3. If the invoked function terminates successfully, remove the event from the database. + + Raises: + Exception: An error occurred inserting record. + """ + db_dsn = ( + f"dbname={self._conf.queue.database} user={self._conf.queue.user} " + f"password={self._conf.queue.password} host={self._conf.queue.host}" + ) + async with aiopg.create_pool(db_dsn) as pool: + async with pool.acquire() as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT * FROM %s ORDER BY creation_date ASC LIMIT %d;" + % (self._table_name, self._conf.queue.records), + ) + async for row in cur: + call_ok = False + try: + reply_on = self.get_event_handler(topic=row[1]) + valid_instance, instance = self._is_valid_instance(row[3]) + if not valid_instance: + return + await reply_on(row[1], instance) + call_ok = True + finally: + if call_ok: + # Delete from database If the event was sent successfully to Kafka. + async with connect.cursor() as cur2: + await cur2.execute("DELETE FROM %s WHERE id=%d;" % (self._table_name, row[0])) + + @abstractmethod + def _is_valid_instance(self, value: bytes): # pragma: no cover + raise Exception("Method not implemented") diff --git a/minos/networks/handler/event/__init__.py b/minos/networks/handler/event/__init__.py new file mode 100644 index 00000000..0b8f5274 --- /dev/null +++ b/minos/networks/handler/event/__init__.py @@ -0,0 +1,18 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from .dispatcher import ( + MinosEventHandlerDispatcher, +) +from .server import ( + MinosEventHandlerServer, +) +from .services import ( + MinosEventPeriodicService, + MinosEventServerService, +) diff --git a/minos/networks/handler/event/dispatcher.py b/minos/networks/handler/event/dispatcher.py new file mode 100644 index 00000000..783a7c3b --- /dev/null +++ b/minos/networks/handler/event/dispatcher.py @@ -0,0 +1,37 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from typing import ( + Any, +) + +from minos.common import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) + +from ..dispatcher import ( + MinosHandlerDispatcher, +) + + +class MinosEventHandlerDispatcher(MinosHandlerDispatcher): + + TABLE = "event_queue" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(table_name=self.TABLE, config=config.events, **kwargs) + self._broker_group_name = f"event_{config.service.name}" + + def _is_valid_instance(self, value: bytes): + try: + instance = Event.from_avro_bytes(value) + return True, instance + except: + return False, None diff --git a/minos/networks/handler/event/server.py b/minos/networks/handler/event/server.py new file mode 100644 index 00000000..55341e98 --- /dev/null +++ b/minos/networks/handler/event/server.py @@ -0,0 +1,38 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from typing import ( + Any, +) + +from minos.common import ( + Event, +) +from minos.common.configuration.config import ( + MinosConfig, +) + +from ..server import ( + MinosHandlerServer, +) + + +class MinosEventHandlerServer(MinosHandlerServer): + + TABLE = "event_queue" + + def __init__(self, *, config: MinosConfig, **kwargs: Any): + super().__init__(table_name=self.TABLE, config=config.events, **kwargs) + self._kafka_conn_data = f"{config.events.broker.host}:{config.events.broker.port}" + self._broker_group_name = f"event_{config.service.name}" + + def _is_valid_instance(self, value: bytes): + try: + Event.from_avro_bytes(value) + return True + except: + return False diff --git a/minos/networks/handler/event/services.py b/minos/networks/handler/event/services.py new file mode 100644 index 00000000..58f0fd7f --- /dev/null +++ b/minos/networks/handler/event/services.py @@ -0,0 +1,74 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from typing import ( + Any, +) + +from aiomisc.service.periodic import ( + PeriodicService, + Service, +) +from minos.common import ( + MinosConfig, +) + +from .dispatcher import ( + MinosEventHandlerDispatcher, +) +from .server import ( + MinosEventHandlerServer, +) + + +class MinosEventServerService(Service): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosEventHandlerServer.from_config(config=config) + self.consumer = None + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await self.dispatcher.setup() + + self.consumer = await self.dispatcher.kafka_consumer( + self.dispatcher._topics, self.dispatcher._broker_group_name, self.dispatcher._kafka_conn_data + ) + await self.dispatcher.handle_message(self.consumer) + + async def stop(self, exception: Exception = None) -> Any: + if self.consumer is not None: + await self.consumer.stop() + + +class MinosEventPeriodicService(PeriodicService): + """Minos QueueDispatcherService class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosEventHandlerDispatcher.from_config(config=config) + + async def start(self) -> None: + """Method to be called at the startup by the internal ``aiomisc`` loigc. + + :return: This method does not return anything. + """ + await super().start() + await self.dispatcher.setup() + + async def callback(self) -> None: + """Method to be called periodically by the internal ``aiomisc`` logic. + + :return:This method does not return anything. + """ + await self.dispatcher.queue_checker() diff --git a/minos/networks/handler/server.py b/minos/networks/handler/server.py new file mode 100644 index 00000000..a65d27c1 --- /dev/null +++ b/minos/networks/handler/server.py @@ -0,0 +1,150 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from __future__ import ( + annotations, +) + +import asyncio +import datetime +from abc import ( + abstractmethod, +) +from typing import ( + Any, + AsyncIterator, + NamedTuple, + Optional, +) + +import aiopg +from aiokafka import ( + AIOKafkaConsumer, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.networks.handler.abc import ( + MinosHandlerSetup, +) +from psycopg2.extensions import ( + AsIs, +) + + +class MinosHandlerServer(MinosHandlerSetup): + """ + Handler Server + + Generic insert for queue_* table. (Support Command, CommandReply and Event) + + """ + + __slots__ = "_tasks", "_db_dsn", "_handlers", "_topics", "_broker_group_name" + + def __init__(self, *, table_name: str, config: NamedTuple, **kwargs: Any): + super().__init__(table_name=table_name, **kwargs, **config.queue._asdict()) + self._tasks = set() # type: t.Set[asyncio.Task] + self._db_dsn = ( + f"dbname={config.queue.database} user={config.queue.user} " + f"password={config.queue.password} host={config.queue.host}" + ) + self._handler = {item.name: {"controller": item.controller, "action": item.action} for item in config.items} + self._topics = list(self._handler.keys()) + self._table_name = table_name + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> Optional[MinosHandlerServer]: + """Build a new repository from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + return None + # noinspection PyProtectedMember + return cls(*args, config=config, **kwargs) + + async def queue_add(self, topic: str, partition: int, binary: bytes): + """Insert row to event_queue table. + + Retrieves number of affected rows and row ID. + + Args: + topic: Kafka topic. Example: "TicketAdded" + partition: Kafka partition number. + binary: Event Model in bytes. + + Returns: + Affected rows and queue ID. + + Example: 1, 12 + + Raises: + Exception: An error occurred inserting record. + """ + + async with aiopg.create_pool(self._db_dsn) as pool: + async with pool.acquire() as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO %s (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + (AsIs(self._table_name), topic, partition, binary, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + return affected_rows, queue_id[0] + + async def handle_single_message(self, msg): + """Handle Kafka messages. + + Evaluate if the binary of message is an Event instance. + Add Event instance to the event_queue table. + + Args: + msg: Kafka message. + + Raises: + Exception: An error occurred inserting record. + """ + # the handler receive a message and store in the queue database + # check if the event binary string is well formatted + if not self._is_valid_instance(msg.value): + return + affected_rows, id = await self.queue_add(msg.topic, msg.partition, msg.value) + return affected_rows, id + + @abstractmethod + def _is_valid_instance(self, value: bytes): # pragma: no cover + raise Exception("Method not implemented") + + async def handle_message(self, consumer: AsyncIterator): + """Message consumer. + + It consumes the messages and sends them for processing. + + Args: + consumer: Kafka Consumer instance (at the moment only Kafka consumer is supported). + """ + + async for msg in consumer: + await self.handle_single_message(msg) + + @staticmethod + async def kafka_consumer(topics: list, group_name: str, conn: str): + # start the Service Event Consumer for Kafka + consumer = AIOKafkaConsumer(group_id=group_name, auto_offset_reset="latest", bootstrap_servers=conn,) + + await consumer.start() + consumer.subscribe(topics) + + return consumer diff --git a/minos/networks/rest_interface/__init__.py b/minos/networks/rest_interface/__init__.py new file mode 100644 index 00000000..524ba7a0 --- /dev/null +++ b/minos/networks/rest_interface/__init__.py @@ -0,0 +1,14 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from .handler import ( + RestInterfaceHandler, +) +from .service import ( + REST, +) diff --git a/minos/networks/rest_interface/handler.py b/minos/networks/rest_interface/handler.py new file mode 100644 index 00000000..9ba76668 --- /dev/null +++ b/minos/networks/rest_interface/handler.py @@ -0,0 +1,71 @@ +# Copyright (C) 2020 Clariteia SL +# +# This file is part of minos framework. +# +# Minos framework can not be copied and/or distributed without the express +# permission of Clariteia SL. + +from aiohttp import ( + web, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.common.importlib import ( + import_module, +) + + +class RestInterfaceHandler: + """ + Rest Interface Handler + + Rest Interface for aiohttp web handling. + + """ + + __slots__ = "_config", "_app" + + def __init__(self, config: MinosConfig, app: web.Application = web.Application()): + self._config = config + self._app = app + self.load_routes() + + def load_routes(self): + """Load routes from config file.""" + for item in self._config.rest.endpoints: + callable_f = self.class_resolver(item.controller, item.action) + self._app.router.add_route(item.method, item.route, callable_f) + + # Load default routes + self._mount_system_health() + + @staticmethod + def class_resolver(controller: str, action: str): + """Load controller class and action method. + :param controller: Controller string. Example: "tests.service.CommandTestService.CommandService" + :param action: Config instance. Example: "get_order" + :return: A class method callable instance. + """ + object_class = import_module(controller) + instance_class = object_class() + class_method = getattr(instance_class, action) + + return class_method + + def get_app(self): + """Return rest application instance. + :return: A `web.Application` instance. + """ + return self._app + + def _mount_system_health(self): + """Mount System Health Route.""" + self._app.router.add_get("/system/health", self._system_health_handler) + + @staticmethod + async def _system_health_handler(request): + """System Health Route Handler. + :return: A `web.json_response` response. + """ + return web.json_response({"host": request.host}, status=200) diff --git a/minos/networks/rest_interface/service.py b/minos/networks/rest_interface/service.py new file mode 100644 index 00000000..1a30e28c --- /dev/null +++ b/minos/networks/rest_interface/service.py @@ -0,0 +1,31 @@ +import typing as t + +from aiomisc.service.aiohttp import ( + AIOHTTPService, +) +from minos.common import ( + MinosConfig, +) + +from .handler import ( + RestInterfaceHandler, +) + + +class REST(AIOHTTPService): + """ + Rest Interface + + Expose REST Interface handler using aiomisc AIOHTTPService. + + """ + + def __init__(self, config: MinosConfig, **kwds: t.Any): + address = config.rest.broker.host + port = config.rest.broker.port + super().__init__(address=address, port=port, **kwds) + self._config = config + self.rest_interface = RestInterfaceHandler(config=self._config) + + async def create_application(self): + return self.rest_interface.get_app() # pragma: no cover diff --git a/minos/networks/snapshots/__init__.py b/minos/networks/snapshots/__init__.py new file mode 100644 index 00000000..ab7b0571 --- /dev/null +++ b/minos/networks/snapshots/__init__.py @@ -0,0 +1,16 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from .dispatchers import ( + MinosSnapshotDispatcher, +) +from .entries import ( + MinosSnapshotEntry, +) +from .services import ( + MinosSnapshotService, +) diff --git a/minos/networks/snapshots/dispatchers.py b/minos/networks/snapshots/dispatchers.py new file mode 100644 index 00000000..39dd3a00 --- /dev/null +++ b/minos/networks/snapshots/dispatchers.py @@ -0,0 +1,237 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from __future__ import ( + annotations, +) + +from typing import ( + Any, + AsyncIterator, + NoReturn, + Optional, + Type, +) + +from minos.common import ( + Aggregate, + MinosConfig, + MinosConfigException, + MinosRepositoryAction, + MinosRepositoryEntry, + PostgreSqlMinosDatabase, + PostgreSqlMinosRepository, + import_module, +) + +from ..exceptions import ( + MinosPreviousVersionSnapshotException, +) +from .entries import ( + MinosSnapshotEntry, +) + + +class MinosSnapshotDispatcher(PostgreSqlMinosDatabase): + """Minos Snapshot Dispatcher class.""" + + def __init__(self, *args, repository: dict[str, Any] = None, **kwargs): + super().__init__(*args, **kwargs) + self.repository = PostgreSqlMinosRepository(**repository) + + async def _destroy(self) -> NoReturn: + await super()._destroy() + await self.repository.destroy() + + @classmethod + def from_config(cls, *args, config: MinosConfig = None, **kwargs) -> MinosSnapshotDispatcher: + """Build a new Snapshot Dispatcher from config. + :param args: Additional positional arguments. + :param config: Config instance. If `None` is provided, default config is chosen. + :param kwargs: Additional named arguments. + :return: A `MinosRepository` instance. + """ + if config is None: + config = MinosConfig.get_default() + if config is None: + raise MinosConfigException("The config object must be setup.") + # noinspection PyProtectedMember + return cls(*args, **config.snapshot._asdict(), repository=config.repository._asdict(), **kwargs) + + async def _setup(self) -> NoReturn: + await self.submit_query(_CREATE_TABLE_QUERY) + await self.submit_query(_CREATE_OFFSET_TABLE_QUERY) + + async def dispatch(self) -> NoReturn: + """Perform a dispatching step, based on the sequence of non already processed ``MinosRepositoryEntry`` objects. + + :return: This method does not return anything. + """ + offset = await self._load_offset() + + ids = set() + + def _update_offset(e: MinosRepositoryEntry, o: int): + ids.add(e.id) + while o + 1 in ids: + o += 1 + ids.remove(o) + return o + + async for entry in self.repository.select(id_ge=offset): + try: + await self._dispatch_one(entry) + except MinosPreviousVersionSnapshotException: + pass + offset = _update_offset(entry, offset) + + await self._store_offset(offset) + + async def _load_offset(self) -> int: + # noinspection PyBroadException + try: + raw = await self.submit_query_and_fetchone(_SELECT_OFFSET_QUERY) + return raw[0] + except Exception: + return 0 + + async def _store_offset(self, offset: int) -> NoReturn: + await self.submit_query(_INSERT_OFFSET_QUERY, {"value": offset}) + + # noinspection PyUnusedLocal + async def select(self, *args, **kwargs) -> AsyncIterator[MinosSnapshotEntry]: + """Select a sequence of ``MinosSnapshotEntry`` objects. + + :param args: Additional positional arguments. + :param kwargs: Additional named arguments. + :return: A sequence of ``MinosSnapshotEntry`` objects. + """ + async for row in self.submit_query_and_iter(_SELECT_ALL_ENTRIES_QUERY): + yield MinosSnapshotEntry(*row) + + async def _dispatch_one(self, event_entry: MinosRepositoryEntry) -> Optional[MinosSnapshotEntry]: + if event_entry.action is MinosRepositoryAction.DELETE: + await self._submit_delete(event_entry) + return + instance = await self._build_instance(event_entry) + return await self._submit_instance(instance) + + async def _submit_delete(self, entry: MinosRepositoryEntry) -> NoReturn: + params = {"aggregate_id": entry.aggregate_id, "aggregate_name": entry.aggregate_name} + await self.submit_query(_DELETE_ONE_SNAPSHOT_ENTRY_QUERY, params) + + async def _build_instance(self, event_entry: MinosRepositoryEntry) -> Aggregate: + # noinspection PyTypeChecker + cls: Type[Aggregate] = import_module(event_entry.aggregate_name) + instance = cls.from_avro_bytes(event_entry.data, id=event_entry.aggregate_id, version=event_entry.version) + instance = await self._update_if_exists(instance) + return instance + + async def _update_if_exists(self, new: Aggregate) -> Aggregate: + # noinspection PyBroadException + try: + # noinspection PyTypeChecker + previous = await self._select_one_aggregate(new.id, new.classname) + except Exception: + return new + + if previous.version >= new.version: + raise MinosPreviousVersionSnapshotException(previous, new) + + new._fields = previous.fields | new.fields + return new + + async def _select_one_aggregate(self, aggregate_id: int, aggregate_name: str) -> Aggregate: + snapshot_entry = await self._select_one(aggregate_id, aggregate_name) + return snapshot_entry.aggregate + + async def _select_one(self, aggregate_id: int, aggregate_name: str) -> MinosSnapshotEntry: + raw = await self.submit_query_and_fetchone(_SELECT_ONE_SNAPSHOT_ENTRY_QUERY, (aggregate_id, aggregate_name)) + return MinosSnapshotEntry(aggregate_id, aggregate_name, *raw) + + async def _submit_instance(self, aggregate: Aggregate) -> MinosSnapshotEntry: + snapshot_entry = MinosSnapshotEntry.from_aggregate(aggregate) + snapshot_entry = await self._submit_update_or_create(snapshot_entry) + return snapshot_entry + + async def _submit_update_or_create(self, entry: MinosSnapshotEntry) -> MinosSnapshotEntry: + params = { + "aggregate_id": entry.aggregate_id, + "aggregate_name": entry.aggregate_name, + "version": entry.version, + "data": entry.data, + } + response = await self.submit_query_and_fetchone(_INSERT_ONE_SNAPSHOT_ENTRY_QUERY, params) + + entry.created_at, entry.updated_at = response + + return entry + + +_CREATE_TABLE_QUERY = """ +CREATE TABLE IF NOT EXISTS snapshot ( + aggregate_id BIGINT NOT NULL, + aggregate_name TEXT NOT NULL, + version INT NOT NULL, + data BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (aggregate_id, aggregate_name) +); +""".strip() + +_SELECT_ALL_ENTRIES_QUERY = """ +SELECT aggregate_id, aggregate_name, version, data, created_at, updated_at +FROM snapshot; +""".strip() + +_DELETE_ONE_SNAPSHOT_ENTRY_QUERY = """ +DELETE FROM snapshot +WHERE aggregate_id = %(aggregate_id)s and aggregate_name = %(aggregate_name)s; +""".strip() + +_SELECT_ONE_SNAPSHOT_ENTRY_QUERY = """ +SELECT version, data, created_at, updated_at +FROM snapshot +WHERE aggregate_id = %s and aggregate_name = %s; +""".strip() + +_INSERT_ONE_SNAPSHOT_ENTRY_QUERY = """ +INSERT INTO snapshot (aggregate_id, aggregate_name, version, data, created_at, updated_at) +VALUES ( + %(aggregate_id)s, + %(aggregate_name)s, + %(version)s, + %(data)s, + default, + default +) +ON CONFLICT (aggregate_id, aggregate_name) +DO + UPDATE SET version = %(version)s, data = %(data)s, updated_at = NOW() +RETURNING created_at, updated_at; +""".strip() + +_CREATE_OFFSET_TABLE_QUERY = """ +CREATE TABLE IF NOT EXISTS snapshot_aux_offset ( + id bool PRIMARY KEY DEFAULT TRUE, + value BIGINT NOT NULL, + CONSTRAINT id_uni CHECK (id) +); +""".strip() + +_SELECT_OFFSET_QUERY = """ +SELECT value +FROM snapshot_aux_offset +WHERE id = TRUE; +""" +_INSERT_OFFSET_QUERY = """ +INSERT INTO snapshot_aux_offset (id, value) +VALUES (TRUE, %(value)s) +ON CONFLICT (id) +DO UPDATE SET value = %(value)s; +""".strip() diff --git a/minos/networks/snapshots/entries.py b/minos/networks/snapshots/entries.py new file mode 100644 index 00000000..973b351e --- /dev/null +++ b/minos/networks/snapshots/entries.py @@ -0,0 +1,103 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from __future__ import ( + annotations, +) + +from datetime import ( + datetime, +) +from typing import ( + Iterable, + Optional, + Type, + Union, +) + +from minos.common import ( + Aggregate, + import_module, +) + + +class MinosSnapshotEntry(object): + """Minos Snapshot Entry class. + + Is the python object representation of a row in the ``snapshot`` storage system. + """ + + __slots__ = "aggregate_id", "aggregate_name", "version", "data", "created_at", "updated_at" + + # noinspection PyShadowingBuiltins + def __init__( + self, + aggregate_id: int, + aggregate_name: str, + version: int, + data: Union[bytes, memoryview] = bytes(), + created_at: Optional[datetime] = None, + updated_at: Optional[datetime] = None, + ): + if isinstance(data, memoryview): + data = data.tobytes() + + self.aggregate_id = aggregate_id + self.aggregate_name = aggregate_name + self.version = version + self.data = data + + self.created_at = created_at + self.updated_at = updated_at + + @classmethod + def from_aggregate(cls, aggregate: Aggregate) -> MinosSnapshotEntry: + """Build a new instance from an ``Aggregate``. + + :param aggregate: The aggregate instance. + :return: A new ``MinosSnapshotEntry`` instance. + """ + # noinspection PyTypeChecker + return cls(aggregate.id, aggregate.classname, aggregate.version, aggregate.avro_bytes) + + @property + def aggregate(self) -> Aggregate: + """Rebuild the stored ``Aggregate`` object instance from the internal state. + + :return: A ``Aggregate`` instance. + """ + cls = self.aggregate_cls + instance = cls.from_avro_bytes(self.data, id=self.aggregate_id, version=self.version) + return instance + + @property + def aggregate_cls(self) -> Type[Aggregate]: + """Load the concrete ``Aggregate`` class. + + :return: A ``Type`` object. + """ + # noinspection PyTypeChecker + return import_module(self.aggregate_name) + + def __eq__(self, other: MinosSnapshotEntry) -> bool: + return type(self) == type(other) and tuple(self) == tuple(other) + + def __hash__(self) -> int: + return hash(tuple(self)) + + def __iter__(self) -> Iterable: + # noinspection PyRedundantParentheses + yield from (self.aggregate_name, self.version, self.data, self.created_at, self.updated_at) + + def __repr__(self): + name = type(self).__name__ + return ( + f"{name}(aggregate_id={repr(self.aggregate_id)}, aggregate_name={repr(self.aggregate_name)}, " + f"version={repr(self.version)}, data={repr(self.data)}, " + f"created_at={repr(self.created_at)}, updated_at={repr(self.updated_at)})" + ) diff --git a/minos/networks/snapshots/services.py b/minos/networks/snapshots/services.py new file mode 100644 index 00000000..4113f9e2 --- /dev/null +++ b/minos/networks/snapshots/services.py @@ -0,0 +1,50 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfig, +) + +from .dispatchers import ( + MinosSnapshotDispatcher, +) + + +class MinosSnapshotService(PeriodicService): + """Minos Snapshot Service class.""" + + def __init__(self, config: MinosConfig = None, **kwargs): + super().__init__(**kwargs) + self.dispatcher = MinosSnapshotDispatcher.from_config(config=config) + + async def start(self) -> None: + """Start the service execution. + + :return: This method does not return anything. + """ + await self.dispatcher.setup() + await super().start() + + async def callback(self) -> None: + """Callback implementation to be executed periodically. + + :return: This method does not return anything. + """ + await self.dispatcher.dispatch() + + async def stop(self, err: Exception = None) -> None: + """Stop the service execution. + + :param err: Optional exception that stopped the execution. + :return: This method does not return anything. + """ + await super().stop(err) + await self.dispatcher.destroy() diff --git a/requirements_dev.txt b/requirements_dev.txt index aae4484e..47b03ed9 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,12 +1,84 @@ -pip -bump2version -wheel -watchdog -flake8 -tox -coverage -Sphinx -twine - -pytest==4.6.5 +aiohttp==3.7.4.post0 +aiokafka==0.7.0 +aiomisc==12.1.0 +aiopg==1.2.1 +alabaster==0.7.12 +appdirs==1.4.4 +async-timeout==3.0.1 +atomicwrites==1.4.0 +attrs==20.3.0 +avro==1.10.2 +Babel==2.9.0 +black==20.8b1 +bleach==3.3.0 +bump2version==1.0.1 +certifi==2020.12.5 +chardet==4.0.0 +click==7.1.2 +colorama==0.4.4 +colorlog==5.0.1 +coverage==5.5 +distlib==0.3.1 +docutils==0.16 +fastavro==1.4.0 +filelock==3.0.12 +flake8==3.9.0 +idna==2.10 +imagesize==1.2.0 +importlib-metadata==3.10.1 +iniconfig==1.1.1 +Jinja2==2.11.3 +kafka-python==2.0.2 +keyring==23.0.1 +lmdb==1.1.1 +MarkupSafe==1.1.1 +mccabe==0.6.1 +minos-microservice-common==0.0.9 +more-itertools==8.7.0 +multidict==5.1.0 +mypy-extensions==0.4.3 +orjson==3.5.1 +packaging==20.9 +pathspec==0.8.1 +peewee==3.14.4 +pkginfo==1.7.0 +pluggy==0.13.1 +psycopg2==2.8.6 +psycopg2-binary==2.8.6 +py==1.10.0 +pycodestyle==2.7.0 +pyflakes==2.3.1 +Pygments==2.8.1 +pyparsing==2.4.7 +pytest==6.2.3 +pytest-asyncio==0.15.1 pytest-runner==5.1 +pytz==2021.1 +PyYAML==5.4.1 +readme-renderer==29.0 +regex==2021.4.4 +requests==2.25.1 +requests-toolbelt==0.9.1 +rfc3986==1.4.0 +six==1.15.0 +snowballstemmer==2.1.0 +Sphinx==3.5.4 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +toml==0.10.2 +tox==3.23.0 +tqdm==4.60.0 +twine==3.4.1 +typed-ast==1.4.3 +typing-extensions==3.7.4.3 +urllib3==1.26.4 +virtualenv==20.4.3 +watchdog==2.0.2 +wcwidth==0.2.5 +webencodings==0.5.1 +yarl==1.6.3 +zipp==3.4.1 diff --git a/setup.cfg b/setup.cfg index f549cb1b..38353861 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,12 +14,28 @@ replace = __version__ = '{new_version}' [bdist_wheel] universal = 1 -[flake8] -exclude = docs - [aliases] # Define setup.py command aliases here test = pytest [tool:pytest] collect_ignore = ['setup.py'] + +[coverage:report] +exclude_lines = + pragma: no cover + raise NotImplementedError + if TYPE_CHECKING: + pass +precision = 2 + +[flake8] +exclude = docs +max-line-length = 120 + +[isort] +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 1 +use_parentheses = True +line_length = 120 diff --git a/setup.py b/setup.py index 600da3b6..8ad9ebb5 100644 --- a/setup.py +++ b/setup.py @@ -2,44 +2,53 @@ """The setup script.""" -from setuptools import setup, find_namespace_packages +from setuptools import ( + find_namespace_packages, + setup, +) -with open('README.rst') as readme_file: +with open("README.md") as readme_file: readme = readme_file.read() -with open('HISTORY.rst') as history_file: +with open("HISTORY.md") as history_file: history = history_file.read() -requirements = ['minos-microservice-common', 'aiokafka', 'aiomisc'] +requirements = ["minos-microservice-common", "aiokafka", "aiomisc", "aiopg"] -setup_requirements = ['pytest-runner',] +setup_requirements = [ + "pytest-runner", +] -test_requirements = ['pytest>=3', 'pytest-asyncio', ] +test_requirements = [ + "pytest>=3", + "pytest-asyncio", +] setup( author="Andrea Mucci", - author_email='andrea@clariteia.com', - python_requires='>=3.5', + author_email="andrea@clariteia.com", + python_requires=">=3.7", classifiers=[ - 'Development Status :: 2 - Pre-Alpha', - 'Intended Audience :: Developers', - 'Natural Language :: English', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], description="Python Package with the common network classes and utlities used in Minos Microservice", install_requires=requirements, - long_description=readme + '\n\n' + history, + long_description_content_type="text/markdown", + long_description=readme + "\n\n" + history, include_package_data=True, - keywords='minos_microservice_networks', - name='minos_microservice_networks', - packages=find_namespace_packages(include=['minos.*']), + keywords="minos_microservice_networks", + name="minos_microservice_networks", + packages=find_namespace_packages(include=["minos.*"]), setup_requires=setup_requirements, - test_suite='tests', + test_suite="tests", tests_require=test_requirements, - version='0.0.1-alpha', + version="0.0.1", zip_safe=False, ) diff --git a/tests/aggregate_classes.py b/tests/aggregate_classes.py new file mode 100644 index 00000000..501537a6 --- /dev/null +++ b/tests/aggregate_classes.py @@ -0,0 +1,31 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from typing import ( + Optional, +) + +from minos.common import ( + Aggregate, + ModelRef, +) + + +class Owner(Aggregate): + """Aggregate ``Owner`` class for testing purposes.""" + + name: str + surname: str + age: Optional[int] + + +class Car(Aggregate): + """Aggregate ``Car`` class for testing purposes.""" + + doors: int + color: str + owner: Optional[list[ModelRef[Owner]]] diff --git a/tests/kafka_container/.travis.yml b/tests/kafka_container/.travis.yml deleted file mode 100644 index 30595423..00000000 --- a/tests/kafka_container/.travis.yml +++ /dev/null @@ -1,89 +0,0 @@ -sudo: required - -language: scala - -services: - - docker - -# This version will be also tagged as 'latest' -env: - global: - - LATEST="2.13-2.7.0" - -# Build recommended versions based on: http://kafka.apache.org/downloads -matrix: - include: - - scala: "2.10" - env: KAFKA_VERSION=0.8.2.2 - - scala: 2.11 - env: KAFKA_VERSION=0.9.0.1 - - scala: 2.11 - env: KAFKA_VERSION=0.10.2.2 - - scala: 2.11 - env: KAFKA_VERSION=0.11.0.3 - - scala: 2.11 - env: KAFKA_VERSION=1.0.2 - - scala: 2.11 - env: KAFKA_VERSION=1.1.1 - - scala: 2.12 - env: KAFKA_VERSION=2.0.1 - - scala: 2.12 - env: KAFKA_VERSION=2.1.1 - - scala: 2.12 - env: KAFKA_VERSION=2.2.2 - - scala: 2.12 - env: KAFKA_VERSION=2.3.1 - - scala: 2.12 - env: KAFKA_VERSION=2.4.1 - - scala: 2.12 - env: KAFKA_VERSION=2.5.0 - - scala: 2.13 - env: KAFKA_VERSION=2.6.0 - - scala: 2.13 - env: KAFKA_VERSION=2.7.0 - -install: - - docker --version - - docker-compose --version - - echo "KAFKA VERSION $KAFKA_VERSION" - - echo "SCALA VERSION $TRAVIS_SCALA_VERSION" - - echo "LATEST VERSION $LATEST" - - export CURRENT=${TRAVIS_SCALA_VERSION}-${KAFKA_VERSION} - - docker build --build-arg kafka_version=$KAFKA_VERSION --build-arg scala_version=$TRAVIS_SCALA_VERSION --build-arg vcs_ref=$TRAVIS_COMMIT --build-arg build_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -t wurstmeister/kafka . - - docker pull confluentinc/cp-kafkacat - -before_script: - - docker-compose -f test/docker-compose.yml up -d zookeeper kafka_1 kafka_2 - -script: - # Shellcheck main source files - - shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh versions.sh - - cd test - # Shellcheck the tests - - shellcheck -x -e SC1090 -s bash *.sh **/*.sh - - ./verifyImageLabels.sh # Verify docker image's label - - sleep 5 # Wait for containers to start - - docker-compose logs - - docker ps -a - - ./runAllTests.sh - # End-to-End scenario tests - - cd scenarios - - ./runJmxScenario.sh - - cd $TRAVIS_BUILD_DIR - -after_script: - - docker-compose stop - -# This will deploy from master. Might want to have a single release branch for a little more control -deploy: - - provider: script - script: bash docker_push latest - on: - repo: wurstmeister/kafka-docker - branch: master - condition: $CURRENT = $LATEST - - provider: script - script: bash docker_push "${TRAVIS_SCALA_VERSION}-${KAFKA_VERSION}" - on: - repo: wurstmeister/kafka-docker - # branch: release diff --git a/tests/kafka_container/CHANGELOG.md b/tests/kafka_container/CHANGELOG.md deleted file mode 100644 index 0e0fce52..00000000 --- a/tests/kafka_container/CHANGELOG.md +++ /dev/null @@ -1,120 +0,0 @@ -Changelog -========= - -Kafka features are not tied to a specific kafka-docker version (ideally all changes will be merged into all branches). Therefore, this changelog will track changes to the image by date. - -30-Dec-2020 ------------ - -- Add support for Kafka `2.7.0` - -06-Aug-2020 ------------ - -- Add support for Kafka `2.6.0` - -20-Apr-2020 ------------ - -- Add support for Kafka `2.5.0` - -16-Mar-2020 ------------ - -- Add support for Kafka `2.4.1` -- Update glibc to `2.31-r0` - -20-Dec-2019 ------------ - -- Add support for Kafka `2.2.2` -- Update glibc to 2.30-r0 - -17-Dec-2019 ------------ - -- Add support for Kafka `2.4.0` - -26-Oct-2019 ------------ - -- Add support for Kafka `2.3.1` - -28-Jun-2019 ------------ - -- Add support for Kafka `2.3.0` - -04-Jun-2019 ------------ - -- Updated `2.2.x` version to Kafka `2.2.1` -- Update base image to openjdk:8u212-jre-alpine - -15-Apr-2019 ------------ - -- Update base image to openjdk:8u201-jre-alpine - -27-Mar-2019 ------------ - -- Add support for Kafka `2.2.0` - -21-Feb-2019 ------------ - -- Update to latest Kafka: `2.1.1` -- Update glibc to `2.29-r0` - -21-Nov-2018 ------------ - -- Update to latest Kafka: `2.1.0` -- Set scala version for Kafka `2.1.0` and `2.0.1` to recommended `2.12` - -10-Nov-2018 ------------ - -- Update to Kafka `2.0.0` -> `2.0.1`. -- Update glibc to `2.28-r0` -- Update base image to openjdk:8u181-jre-alpine - -29-Jun-2018 ------------ - -- **MAJOR:** Use new docker image labelling (`-`) and use travis to publish images. -- Update base image to openjdk:8u171-jre-alpine - -20-Apr-2018 ------------ - -- Issue #312 - Fix conflict between KAFKA_xxx broker config values (e.g. KAFKA_JMX_OPTS) and container configuration options (e.g. KAFKA_CREATE_TOPICS) - -19-Apr-2018 ------------ - -- Issue #310 - Only return Apache download mirrors that can supply required kafka/scala version - -11-Apr-2018 ------------ - -- Issue #313 - Fix parsing of environment value substitution when spaces included. - -08-Apr-2018 ------------ - -- Issue #208 - Add `KAFKA_CREATE_TOPICS_SEPARATOR` to allow custom input, such as multi-line YAML. -- Issue #298 - Fix SNAPPY compression support by adding glibc port back into image (removed when switching to openjdk base image in #7a25ade) - -04-Apr-2018 ------------ - -- Support `_{PORT_COMMAND}` placeholder. - -03-Apr-2018 ------------ - -- **BREAKING:** removed `KAFKA_ADVERTISED_PROTOCOL_NAME` and `KAFKA_PROTOCOL_NAME`. Use the canonical [Kafka Configuration](http://kafka.apache.org/documentation.html#brokerconfigs) instead. -- Support `_{HOSTNAME_COMMAND}` placeholder. -- **BREAKING:** Make `KAFKA_ZOOKEEPER_CONNECT` mandatory diff --git a/tests/kafka_container/Dockerfile b/tests/kafka_container/Dockerfile deleted file mode 100644 index 5eca3982..00000000 --- a/tests/kafka_container/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -FROM openjdk:8u212-jre-alpine - -ARG kafka_version=2.7.0 -ARG scala_version=2.13 -ARG glibc_version=2.31-r0 -ARG vcs_ref=unspecified -ARG build_date=unspecified - -LABEL org.label-schema.name="kafka" \ - org.label-schema.description="Apache Kafka" \ - org.label-schema.build-date="${build_date}" \ - org.label-schema.vcs-url="https://github.com/wurstmeister/kafka-docker" \ - org.label-schema.vcs-ref="${vcs_ref}" \ - org.label-schema.version="${scala_version}_${kafka_version}" \ - org.label-schema.schema-version="1.0" \ - maintainer="wurstmeister" - -ENV KAFKA_VERSION=$kafka_version \ - SCALA_VERSION=$scala_version \ - KAFKA_HOME=/opt/kafka \ - GLIBC_VERSION=$glibc_version - -ENV PATH=${PATH}:${KAFKA_HOME}/bin - -COPY download-kafka.sh start-kafka.sh broker-list.sh create-topics.sh versions.sh /tmp/ - -RUN apk add --no-cache bash curl jq docker \ - && chmod a+x /tmp/*.sh \ - && mv /tmp/start-kafka.sh /tmp/broker-list.sh /tmp/create-topics.sh /tmp/versions.sh /usr/bin \ - && sync && /tmp/download-kafka.sh \ - && tar xfz /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt \ - && rm /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz \ - && ln -s /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} ${KAFKA_HOME} \ - && rm /tmp/* \ - && wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \ - && apk add --no-cache --allow-untrusted glibc-${GLIBC_VERSION}.apk \ - && rm glibc-${GLIBC_VERSION}.apk - -COPY overrides /opt/overrides - -VOLUME ["/kafka"] - -# Use "exec" form so that it runs as PID 1 (useful for graceful shutdown) -CMD ["start-kafka.sh"] diff --git a/tests/kafka_container/LICENSE b/tests/kafka_container/LICENSE deleted file mode 100644 index e06d2081..00000000 --- a/tests/kafka_container/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/tests/kafka_container/README.md b/tests/kafka_container/README.md deleted file mode 100644 index d55939ad..00000000 --- a/tests/kafka_container/README.md +++ /dev/null @@ -1,222 +0,0 @@ -[![Docker Pulls](https://img.shields.io/docker/pulls/wurstmeister/kafka.svg)](https://hub.docker.com/r/wurstmeister/kafka/) -[![Docker Stars](https://img.shields.io/docker/stars/wurstmeister/kafka.svg)](https://hub.docker.com/r/wurstmeister/kafka/) -[![](https://images.microbadger.com/badges/version/wurstmeister/kafka.svg)](https://microbadger.com/images/wurstmeister/kafka "Get your own version badge on microbadger.com") -[![](https://images.microbadger.com/badges/image/wurstmeister/kafka.svg)](https://microbadger.com/images/wurstmeister/kafka "Get your own image badge on microbadger.com") -[![Build Status](https://travis-ci.org/wurstmeister/kafka-docker.svg?branch=master)](https://travis-ci.org/wurstmeister/kafka-docker) - -kafka-docker -============ - -Dockerfile for [Apache Kafka](http://kafka.apache.org/) - -The image is available directly from [Docker Hub](https://hub.docker.com/r/wurstmeister/kafka/) - -Tags and releases ------------------ - -All versions of the image are built from the same set of scripts with only minor variations (i.e. certain features are not supported on older versions). The version format mirrors the Kafka format, `-`. Initially, all images are built with the recommended version of scala documented on [http://kafka.apache.org/downloads](http://kafka.apache.org/downloads). Available tags are: - -- `2.13-2.7.0` -- `2.13-2.6.0` -- `2.12-2.5.0` -- `2.12-2.4.1` -- `2.12-2.3.1` -- `2.12-2.2.2` -- `2.12-2.1.1` -- `2.12-2.0.1` -- `2.11-1.1.1` -- `2.11-1.0.2` -- `2.11-0.11.0.3` -- `2.11-0.10.2.2` -- `2.11-0.9.0.1` -- `2.10-0.8.2.2` - -Everytime the image is updated, all tags will be pushed with the latest updates. This should allow for greater consistency across tags, as well as any security updates that have been made to the base image. - ---- - -## Announcements - -* **04-Jun-2019** - Update base image to openjdk 212 ([Release notes](https://www.oracle.com/technetwork/java/javase/8u212-relnotes-5292913.html). Please force pull to get these latest updates - including security patches etc. - ---- - -## Pre-Requisites - -- install docker-compose [https://docs.docker.com/compose/install/](https://docs.docker.com/compose/install/) -- modify the ```KAFKA_ADVERTISED_HOST_NAME``` in [docker-compose.yml](https://raw.githubusercontent.com/wurstmeister/kafka-docker/master/docker-compose.yml) to match your docker host IP (Note: Do not use localhost or 127.0.0.1 as the host ip if you want to run multiple brokers.) -- if you want to customize any Kafka parameters, simply add them as environment variables in ```docker-compose.yml```, e.g. in order to increase the ```message.max.bytes``` parameter set the environment to ```KAFKA_MESSAGE_MAX_BYTES: 2000000```. To turn off automatic topic creation set ```KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'``` -- Kafka's log4j usage can be customized by adding environment variables prefixed with ```LOG4J_```. These will be mapped to ```log4j.properties```. For example: ```LOG4J_LOGGER_KAFKA_AUTHORIZER_LOGGER=DEBUG, authorizerAppender``` - -**NOTE:** There are several 'gotchas' with configuring networking. If you are not sure about what the requirements are, please check out the [Connectivity Guide](https://github.com/wurstmeister/kafka-docker/wiki/Connectivity) in the [Wiki](https://github.com/wurstmeister/kafka-docker/wiki) - -## Usage - -Start a cluster: - -- ```docker-compose up -d ``` - -Add more brokers: - -- ```docker-compose scale kafka=3``` - -Destroy a cluster: - -- ```docker-compose stop``` - -## Note - -The default ```docker-compose.yml``` should be seen as a starting point. By default each broker will get a new port number and broker id on restart. Depending on your use case this might not be desirable. If you need to use specific ports and broker ids, modify the docker-compose configuration accordingly, e.g. [docker-compose-single-broker.yml](https://github.com/wurstmeister/kafka-docker/blob/master/docker-compose-single-broker.yml): - -- ```docker-compose -f docker-compose-single-broker.yml up``` - -## Broker IDs - -You can configure the broker id in different ways - -1. explicitly, using ```KAFKA_BROKER_ID``` -2. via a command, using ```BROKER_ID_COMMAND```, e.g. ```BROKER_ID_COMMAND: "hostname | awk -F'-' '{print $$2}'"``` - -If you don't specify a broker id in your docker-compose file, it will automatically be generated (see [https://issues.apache.org/jira/browse/KAFKA-1070](https://issues.apache.org/jira/browse/KAFKA-1070). This allows scaling up and down. In this case it is recommended to use the ```--no-recreate``` option of docker-compose to ensure that containers are not re-created and thus keep their names and ids. - - -## Automatically create topics - -If you want to have kafka-docker automatically create topics in Kafka during -creation, a ```KAFKA_CREATE_TOPICS``` environment variable can be -added in ```docker-compose.yml```. - -Here is an example snippet from ```docker-compose.yml```: - - environment: - KAFKA_CREATE_TOPICS: "Topic1:1:3,Topic2:1:1:compact" - -```Topic 1``` will have 1 partition and 3 replicas, ```Topic 2``` will have 1 partition, 1 replica and a `cleanup.policy` set to `compact`. Also, see FAQ: [Topic compaction does not work](https://github.com/wurstmeister/kafka-docker/wiki#topic-compaction-does-not-work) - -If you wish to use multi-line YAML or some other delimiter between your topic definitions, override the default `,` separator by specifying the `KAFKA_CREATE_TOPICS_SEPARATOR` environment variable. - -For example, `KAFKA_CREATE_TOPICS_SEPARATOR: "$$'\n'"` would use a newline to split the topic definitions. Syntax has to follow docker-compose escaping rules, and [ANSI-C](https://www.gnu.org/software/bash/manual/html_node/ANSI_002dC-Quoting.html) quoting. - -## Advertised hostname - -You can configure the advertised hostname in different ways - -1. explicitly, using ```KAFKA_ADVERTISED_HOST_NAME``` -2. via a command, using ```HOSTNAME_COMMAND```, e.g. ```HOSTNAME_COMMAND: "route -n | awk '/UG[ \t]/{print $$2}'"``` - -When using commands, make sure you review the "Variable Substitution" section in [https://docs.docker.com/compose/compose-file/](https://docs.docker.com/compose/compose-file/#variable-substitution) - -If ```KAFKA_ADVERTISED_HOST_NAME``` is specified, it takes precedence over ```HOSTNAME_COMMAND``` - -For AWS deployment, you can use the Metadata service to get the container host's IP: -``` -HOSTNAME_COMMAND=wget -t3 -T2 -qO- http://169.254.169.254/latest/meta-data/local-ipv4 -``` -Reference: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html - -### Injecting HOSTNAME_COMMAND into configuration - -If you require the value of `HOSTNAME_COMMAND` in any of your other `KAFKA_XXX` variables, use the `_{HOSTNAME_COMMAND}` string in your variable value, i.e. - -``` -KAFKA_ADVERTISED_LISTENERS=SSL://_{HOSTNAME_COMMAND}:9093,PLAINTEXT://9092 -``` - -## Advertised port - -If the required advertised port is not static, it may be necessary to determine this programatically. This can be done with the `PORT_COMMAND` environment variable. - -``` -PORT_COMMAND: "docker port $$(hostname) 9092/tcp | cut -d: -f2" -``` - -This can be then interpolated in any other `KAFKA_XXX` config using the `_{PORT_COMMAND}` string, i.e. - -``` -KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://1.2.3.4:_{PORT_COMMAND} -``` - -## Listener Configuration - -It may be useful to have the [Kafka Documentation](https://kafka.apache.org/documentation/) open, to understand the various broker listener configuration options. - -Since 0.9.0, Kafka has supported [multiple listener configurations](https://issues.apache.org/jira/browse/KAFKA-1809) for brokers to help support different protocols and discriminate between internal and external traffic. Later versions of Kafka have deprecated ```advertised.host.name``` and ```advertised.port```. - -**NOTE:** ```advertised.host.name``` and ```advertised.port``` still work as expected, but should not be used if configuring the listeners. - -### Example - -The example environment below: - -``` -HOSTNAME_COMMAND: curl http://169.254.169.254/latest/meta-data/public-hostname -KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094 -KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094 -KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT -KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE -``` - -Will result in the following broker config: - -``` -advertised.listeners = OUTSIDE://ec2-xx-xx-xxx-xx.us-west-2.compute.amazonaws.com:9094,INSIDE://:9092 -listeners = OUTSIDE://:9094,INSIDE://:9092 -inter.broker.listener.name = INSIDE -``` - -### Rules - -* No listeners may share a port number. -* An advertised.listener must be present by protocol name and port number in the list of listeners. - -## Broker Rack - -You can configure the broker rack affinity in different ways - -1. explicitly, using ```KAFKA_BROKER_RACK``` -2. via a command, using ```RACK_COMMAND```, e.g. ```RACK_COMMAND: "curl http://169.254.169.254/latest/meta-data/placement/availability-zone"``` - -In the above example the AWS metadata service is used to put the instance's availability zone in the ```broker.rack``` property. - -## JMX - -For monitoring purposes you may wish to configure JMX. Additional to the standard JMX parameters, problems could arise from the underlying RMI protocol used to connect - -* java.rmi.server.hostname - interface to bind listening port -* com.sun.management.jmxremote.rmi.port - The port to service RMI requests - -For example, to connect to a kafka running locally (assumes exposing port 1099) - - KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.rmi.port=1099" - JMX_PORT: 1099 - -Jconsole can now connect at ```jconsole 192.168.99.100:1099``` - -## Docker Swarm Mode - -The listener configuration above is necessary when deploying Kafka in a Docker Swarm using an overlay network. By separating OUTSIDE and INSIDE listeners, a host can communicate with clients outside the overlay network while still benefiting from it from within the swarm. - -In addition to the multiple-listener configuration, additional best practices for operating Kafka in a Docker Swarm include: - -* Use "deploy: global" in a compose file to launch one and only one Kafka broker per swarm node. -* Use compose file version '3.2' (minimum Docker version 16.04) and the "long" port definition with the port in "host" mode instead of the default "ingress" load-balanced port binding. This ensures that outside requests are always routed to the correct broker. For example: - -``` -ports: - - target: 9094 - published: 9094 - protocol: tcp - mode: host -``` - -Older compose files using the short-version of port mapping may encounter Kafka client issues if their connection to individual brokers cannot be guaranteed. - -See the included sample compose file ```docker-compose-swarm.yml``` - -## Release process - -See the [wiki](https://github.com/wurstmeister/kafka-docker/wiki/ReleaseProcess) for information on adding or updating versions to release to Dockerhub. - -## Tutorial - -[http://wurstmeister.github.io/kafka-docker/](http://wurstmeister.github.io/kafka-docker/) diff --git a/tests/kafka_container/broker-list.sh b/tests/kafka_container/broker-list.sh deleted file mode 100755 index 73aa8220..00000000 --- a/tests/kafka_container/broker-list.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -CONTAINERS=$(docker ps | grep 9092 | awk '{print $1}') -BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done) -echo "${BROKERS/$'\n'/,}" diff --git a/tests/kafka_container/create-topics.sh b/tests/kafka_container/create-topics.sh deleted file mode 100755 index 0bacf7b5..00000000 --- a/tests/kafka_container/create-topics.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -if [[ -z "$KAFKA_CREATE_TOPICS" ]]; then - exit 0 -fi - -if [[ -z "$START_TIMEOUT" ]]; then - START_TIMEOUT=600 -fi - -start_timeout_exceeded=false -count=0 -step=10 -while netstat -lnt | awk '$4 ~ /:'"$KAFKA_PORT"'$/ {exit 1}'; do - echo "waiting for kafka to be ready" - sleep $step; - count=$((count + step)) - if [ $count -gt $START_TIMEOUT ]; then - start_timeout_exceeded=true - break - fi -done - -if $start_timeout_exceeded; then - echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)" - exit 1 -fi - -# introduced in 0.10. In earlier versions, this will fail because the topic already exists. -# shellcheck disable=SC1091 -source "/usr/bin/versions.sh" -if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -gt "9" ]] || [[ "$MAJOR_VERSION" -gt "0" ]]; then - KAFKA_0_10_OPTS="--if-not-exists" -fi - -# Expected format: -# name:partitions:replicas:cleanup.policy -IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOPICS; do - echo "creating topics: $topicToCreate" - IFS=':' read -r -a topicConfig <<< "$topicToCreate" - config= - if [ -n "${topicConfig[3]}" ]; then - config="--config=cleanup.policy=${topicConfig[3]}" - fi - - COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\ - --create \\ - --zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\ - --topic ${topicConfig[0]} \\ - --partitions ${topicConfig[1]} \\ - --replication-factor ${topicConfig[2]} \\ - ${config} \\ - ${KAFKA_0_10_OPTS} &" - eval "${COMMAND}" -done - -wait diff --git a/tests/kafka_container/docker-compose-single-broker.yml b/tests/kafka_container/docker-compose-single-broker.yml deleted file mode 100644 index 94128d5c..00000000 --- a/tests/kafka_container/docker-compose-single-broker.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: '2' -services: - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181:2181" - kafka: - build: . - ports: - - "9092:9092" - environment: - KAFKA_ADVERTISED_HOST_NAME: localhost - KAFKA_CREATE_TOPICS: "test:1:1" - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - volumes: - - /var/run/docker.sock:/var/run/docker.sock diff --git a/tests/kafka_container/docker-compose-swarm.yml b/tests/kafka_container/docker-compose-swarm.yml deleted file mode 100644 index 86e63eb1..00000000 --- a/tests/kafka_container/docker-compose-swarm.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: '3.2' -services: - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181:2181" - kafka: - image: wurstmeister/kafka:latest - ports: - - target: 9094 - published: 9094 - protocol: tcp - mode: host - environment: - HOSTNAME_COMMAND: "docker info | grep ^Name: | cut -d' ' -f 2" - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094 - KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094 - KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE - volumes: - - /var/run/docker.sock:/var/run/docker.sock diff --git a/tests/kafka_container/docker-compose.yml b/tests/kafka_container/docker-compose.yml deleted file mode 100644 index bc270dc6..00000000 --- a/tests/kafka_container/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '2' -services: - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181:2181" - kafka: - build: . - ports: - - "9092:9092" - environment: - KAFKA_ADVERTISED_HOST_NAME: localhost - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - volumes: - - /var/run/docker.sock:/var/run/docker.sock diff --git a/tests/kafka_container/docker_push b/tests/kafka_container/docker_push deleted file mode 100755 index 99975bb9..00000000 --- a/tests/kafka_container/docker_push +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -e - -BASE_IMAGE="wurstmeister/kafka" -IMAGE_VERSION="$1" - -if [ -z "$IMAGE_VERSION" ]; then - echo "No IMAGE_VERSION var specified" - exit 1 -fi - -echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin -TARGET="$BASE_IMAGE:$IMAGE_VERSION" -docker tag "$BASE_IMAGE" "$TARGET" -docker push "$TARGET" diff --git a/tests/kafka_container/download-kafka.sh b/tests/kafka_container/download-kafka.sh deleted file mode 100755 index 00bf4511..00000000 --- a/tests/kafka_container/download-kafka.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -e - -# shellcheck disable=SC1091 -source "/usr/bin/versions.sh" - -FILENAME="kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" - -url=$(curl --stderr /dev/null "https://www.apache.org/dyn/closer.cgi?path=/kafka/${KAFKA_VERSION}/${FILENAME}&as_json=1" | jq -r '"\(.preferred)\(.path_info)"') - -# Test to see if the suggested mirror has this version, currently pre 2.1.1 versions -# do not appear to be actively mirrored. This may also be useful if closer.cgi is down. -if [[ ! $(curl -s -f -I "${url}") ]]; then - echo "Mirror does not have desired version, downloading direct from Apache" - url="https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/${FILENAME}" -fi - -echo "Downloading Kafka from $url" -wget "${url}" -O "/tmp/${FILENAME}" diff --git a/tests/kafka_container/overrides/0.9.0.1.sh b/tests/kafka_container/overrides/0.9.0.1.sh deleted file mode 100755 index d5e85611..00000000 --- a/tests/kafka_container/overrides/0.9.0.1.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -e - -# Kafka 0.9.x.x has a 'listeners' config by default. We need to remove this -# as the user may be configuring via the host.name / advertised.host.name properties -echo "Removing 'listeners' from server.properties pre-bootstrap" -sed -i -e '/^listeners=/d' "$KAFKA_HOME/config/server.properties" diff --git a/tests/kafka_container/start-kafka-shell.sh b/tests/kafka_container/start-kafka-shell.sh deleted file mode 100755 index 62663e49..00000000 --- a/tests/kafka_container/start-kafka-shell.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e HOST_IP=$1 -e ZK=$2 -i -t wurstmeister/kafka /bin/bash diff --git a/tests/kafka_container/start-kafka.sh b/tests/kafka_container/start-kafka.sh deleted file mode 100755 index 85359118..00000000 --- a/tests/kafka_container/start-kafka.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash -e - -# Allow specific kafka versions to perform any unique bootstrap operations -OVERRIDE_FILE="/opt/overrides/${KAFKA_VERSION}.sh" -if [[ -x "$OVERRIDE_FILE" ]]; then - echo "Executing override file $OVERRIDE_FILE" - eval "$OVERRIDE_FILE" -fi - -# Store original IFS config, so we can restore it at various stages -ORIG_IFS=$IFS - -if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then - echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT" - exit 1 -fi - -if [[ -z "$KAFKA_PORT" ]]; then - export KAFKA_PORT=9092 -fi - -create-topics.sh & -unset KAFKA_CREATE_TOPICS - -if [[ -z "$KAFKA_ADVERTISED_PORT" && \ - -z "$KAFKA_LISTENERS" && \ - -z "$KAFKA_ADVERTISED_LISTENERS" && \ - -S /var/run/docker.sock ]]; then - KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g') - export KAFKA_ADVERTISED_PORT -fi - -if [[ -z "$KAFKA_BROKER_ID" ]]; then - if [[ -n "$BROKER_ID_COMMAND" ]]; then - KAFKA_BROKER_ID=$(eval "$BROKER_ID_COMMAND") - export KAFKA_BROKER_ID - else - # By default auto allocate broker ID - export KAFKA_BROKER_ID=-1 - fi -fi - -if [[ -z "$KAFKA_LOG_DIRS" ]]; then - export KAFKA_LOG_DIRS="/kafka/kafka-logs-$HOSTNAME" -fi - -if [[ -n "$KAFKA_HEAP_OPTS" ]]; then - sed -r -i 's/(export KAFKA_HEAP_OPTS)="(.*)"/\1="'"$KAFKA_HEAP_OPTS"'"/g' "$KAFKA_HOME/bin/kafka-server-start.sh" - unset KAFKA_HEAP_OPTS -fi - -if [[ -n "$HOSTNAME_COMMAND" ]]; then - HOSTNAME_VALUE=$(eval "$HOSTNAME_COMMAND") - - # Replace any occurences of _{HOSTNAME_COMMAND} with the value - IFS=$'\n' - for VAR in $(env); do - if [[ $VAR =~ ^KAFKA_ && "$VAR" =~ "_{HOSTNAME_COMMAND}" ]]; then - eval "export ${VAR//_\{HOSTNAME_COMMAND\}/$HOSTNAME_VALUE}" - fi - done - IFS=$ORIG_IFS -fi - -if [[ -n "$PORT_COMMAND" ]]; then - PORT_VALUE=$(eval "$PORT_COMMAND") - - # Replace any occurences of _{PORT_COMMAND} with the value - IFS=$'\n' - for VAR in $(env); do - if [[ $VAR =~ ^KAFKA_ && "$VAR" =~ "_{PORT_COMMAND}" ]]; then - eval "export ${VAR//_\{PORT_COMMAND\}/$PORT_VALUE}" - fi - done - IFS=$ORIG_IFS -fi - -if [[ -n "$RACK_COMMAND" && -z "$KAFKA_BROKER_RACK" ]]; then - KAFKA_BROKER_RACK=$(eval "$RACK_COMMAND") - export KAFKA_BROKER_RACK -fi - -# Try and configure minimal settings or exit with error if there isn't enough information -if [[ -z "$KAFKA_ADVERTISED_HOST_NAME$KAFKA_LISTENERS" ]]; then - if [[ -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then - echo "ERROR: Missing environment variable KAFKA_LISTENERS. Must be specified when using KAFKA_ADVERTISED_LISTENERS" - exit 1 - elif [[ -z "$HOSTNAME_VALUE" ]]; then - echo "ERROR: No listener or advertised hostname configuration provided in environment." - echo " Please define KAFKA_LISTENERS / (deprecated) KAFKA_ADVERTISED_HOST_NAME" - exit 1 - fi - - # Maintain existing behaviour - # If HOSTNAME_COMMAND is provided, set that to the advertised.host.name value if listeners are not defined. - export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE" -fi - -#Issue newline to config file in case there is not one already -echo "" >> "$KAFKA_HOME/config/server.properties" - -( - function updateConfig() { - key=$1 - value=$2 - file=$3 - - # Omit $value here, in case there is sensitive information - echo "[Configuring] '$key' in '$file'" - - # If config exists in file, replace it. Otherwise, append to file. - if grep -E -q "^#?$key=" "$file"; then - sed -r -i "s@^#?$key=.*@$key=$value@g" "$file" #note that no config values may contain an '@' char - else - echo "$key=$value" >> "$file" - fi - } - - # Fixes #312 - # KAFKA_VERSION + KAFKA_HOME + grep -rohe KAFKA[A-Z0-0_]* /opt/kafka/bin | sort | uniq | tr '\n' '|' - EXCLUSIONS="|KAFKA_VERSION|KAFKA_HOME|KAFKA_DEBUG|KAFKA_GC_LOG_OPTS|KAFKA_HEAP_OPTS|KAFKA_JMX_OPTS|KAFKA_JVM_PERFORMANCE_OPTS|KAFKA_LOG|KAFKA_OPTS|" - - # Read in env as a new-line separated array. This handles the case of env variables have spaces and/or carriage returns. See #313 - IFS=$'\n' - for VAR in $(env) - do - env_var=$(echo "$VAR" | cut -d= -f1) - if [[ "$EXCLUSIONS" = *"|$env_var|"* ]]; then - echo "Excluding $env_var from broker config" - continue - fi - - if [[ $env_var =~ ^KAFKA_ ]]; then - kafka_name=$(echo "$env_var" | cut -d_ -f2- | tr '[:upper:]' '[:lower:]' | tr _ .) - updateConfig "$kafka_name" "${!env_var}" "$KAFKA_HOME/config/server.properties" - fi - - if [[ $env_var =~ ^LOG4J_ ]]; then - log4j_name=$(echo "$env_var" | tr '[:upper:]' '[:lower:]' | tr _ .) - updateConfig "$log4j_name" "${!env_var}" "$KAFKA_HOME/config/log4j.properties" - fi - done -) - -if [[ -n "$CUSTOM_INIT_SCRIPT" ]] ; then - eval "$CUSTOM_INIT_SCRIPT" -fi - -exec "$KAFKA_HOME/bin/kafka-server-start.sh" "$KAFKA_HOME/config/server.properties" diff --git a/tests/kafka_container/test/0.0/test.broker-list.kafka.sh b/tests/kafka_container/test/0.0/test.broker-list.kafka.sh deleted file mode 100755 index f557e8f9..00000000 --- a/tests/kafka_container/test/0.0/test.broker-list.kafka.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e - -testBrokerList() { - # Need to get the proxied ports for kafka - PORT1=$(docker inspect -f '{{ index .NetworkSettings.Ports "9092/tcp" 0 "HostPort" }}' test_kafka_1) - PORT2=$(docker inspect -f '{{ index .NetworkSettings.Ports "9092/tcp" 0 "HostPort" }}' test_kafka_2) - - RESULT=$(HOST_IP=1.2.3.4 broker-list.sh) - - echo "$RESULT" - - if [[ "$RESULT" == "1.2.3.4:$PORT1,1.2.3.4:$PORT2" || "$RESULT" == "1.2.3.4:$PORT2,1.2.3.4:$PORT1" ]]; then - return 0 - else - return 1 - fi -} - -testBrokerList diff --git a/tests/kafka_container/test/0.0/test.create-topics-custom-separator.kafka.sh b/tests/kafka_container/test/0.0/test.create-topics-custom-separator.kafka.sh deleted file mode 100755 index abd031c1..00000000 --- a/tests/kafka_container/test/0.0/test.create-topics-custom-separator.kafka.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -e - -# NOTE: create-topics.sh requires KAFKA_PORT and KAFKA_ZOOKEEPER_CONNECT to be set (see docker-compose.yml) -testCreateTopicsCustomSeparator() { - NOW=$(date +%s) - - # TOPICS array contains the topic name to create / validate - TOPICS[0]="one-$NOW" - TOPICS[1]="two-$NOW" - TOPICS[2]="three-$NOW" - - export KAFKA_CREATE_TOPICS_SEPARATOR=$'\n' - KAFKA_CREATE_TOPICS=$(cat <<-EOF - ${TOPICS[0]}:1:1 - ${TOPICS[1]}:1:1 - ${TOPICS[2]}:1:1 - EOF - ) - export KAFKA_CREATE_TOPICS - - create-topics.sh - - # Loop through each array, validate that topic exists - for i in "${!TOPICS[@]}"; do - TOPIC=${TOPICS[i]} - - echo "Validating topic '$TOPIC'" - - EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC") - if [[ "$EXISTS" != "$TOPIC" ]]; then - echo "$TOPIC topic not created" - return 1 - fi - done - - return 0 -} - -# mock the netstat call as made by the create-topics.sh script -function netstat() { echo "1 2 3 :$KAFKA_PORT"; } -export -f netstat - -testCreateTopicsCustomSeparator diff --git a/tests/kafka_container/test/0.0/test.path.kafka.sh b/tests/kafka_container/test/0.0/test.path.kafka.sh deleted file mode 100755 index ce02401b..00000000 --- a/tests/kafka_container/test/0.0/test.path.kafka.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -e - -# NOTE: this tests to see if the /opt/kafka/bin is existing in the path within the docker container - -testPath() { - echo "Checking PATH '$PATH'" - if [[ ! "$PATH" =~ "/opt/kafka/bin" ]]; then - echo "path is not set correctly: $PATH" - return 1 - fi - - return 0 -} - -testPath diff --git a/tests/kafka_container/test/0.0/test.read-write.kafkacat.sh b/tests/kafka_container/test/0.0/test.read-write.kafkacat.sh deleted file mode 100755 index 325455bb..00000000 --- a/tests/kafka_container/test/0.0/test.read-write.kafkacat.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e - -source version.functions - -testReadWrite() { - echo 'foo,bar' | eval "kafkacat -b $BROKER_LIST $KAFKACAT_OPTS -P -D, -t readwrite" - eval "kafkacat -b $BROKER_LIST $KAFKACAT_OPTS -C -e -t readwrite" -} - -testReadWrite diff --git a/tests/kafka_container/test/0.0/test.start-kafka-advertised-host.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-advertised-host.kafka.sh deleted file mode 100755 index 1791d60b..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-advertised-host.kafka.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testAdvertisedHost() { - # Given a hostname is provided - export KAFKA_ADVERTISED_HOST_NAME=monkey - export KAFKA_ADVERTISED_PORT=8888 - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration file is correct - assertExpectedConfig "advertised.host.name=monkey" - assertExpectedConfig "advertised.port=8888" - assertAbsent 'advertised.listeners' - assertAbsent 'listeners' -} - -testAdvertisedHost diff --git a/tests/kafka_container/test/0.0/test.start-kafka-broker-id.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-broker-id.kafka.sh deleted file mode 100755 index 4b132648..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-broker-id.kafka.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testManualBrokerId() { - echo "testManualBrokerId" - - # Given a Broker Id is provided - export KAFKA_LISTENERS=PLAINTEXT://:9092 - export KAFKA_BROKER_ID=57 - - # When the script is invoked - source "$START_KAFKA" - - # Then the broker Id is set - assertExpectedConfig 'broker.id=57' -} - -testAutomaticBrokerId() { - echo "testAutomaticBrokerId" - - # Given no Broker Id is provided - export KAFKA_LISTENERS=PLAINTEXT://:9092 - unset KAFKA_BROKER_ID - - # When the script is invoked - source "$START_KAFKA" - - # Then the broker Id is configured to automatic - assertExpectedConfig 'broker.id=-1' -} - -testBrokerIdCommand() { - echo "testBrokerIdCommand" - - # Given a Broker Id command is provided - export KAFKA_LISTENERS=PLAINTEXT://:9092 - unset KAFKA_BROKER_ID - export BROKER_ID_COMMAND='f() { echo "23"; }; f' - - # When the script is invoked - source "$START_KAFKA" - - # Then the broker Id is the result of the command - assertExpectedConfig 'broker.id=23' -} - - -testManualBrokerId \ - && testAutomaticBrokerId \ - && testBrokerIdCommand diff --git a/tests/kafka_container/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh deleted file mode 100755 index 5f0167c3..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testKafkaEnv() { - # Given required settings are provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" - export KAFKA_OPTS="-Djava.security.auth.login.config=/kafka_server_jaas.conf" - - # When the script is invoked - source "$START_KAFKA" - - # Then env should remain untouched - if [[ ! "$KAFKA_OPTS" == "-Djava.security.auth.login.config=/kafka_server_jaas.conf" ]]; then - echo "KAFKA_OPTS not set to expected value. $KAFKA_OPTS" - exit 1 - fi - - # And the broker config should not be set - assertAbsent 'opts' - - echo " > Set KAFKA_OPTS=$KAFKA_OPTS" -} - -testKafkaEnv diff --git a/tests/kafka_container/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh deleted file mode 100755 index df7b4018..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testKafkaOpts() { - # Given required settings are provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" - # .. and a CUSTOM_INIT_SCRIPT with spaces - export CUSTOM_INIT_SCRIPT="export KAFKA_OPTS=-Djava.security.auth.login.config=/kafka_server_jaas.conf" - - # When the script is invoked - source "$START_KAFKA" - - # Then the custom init script should be evaluated - if [[ ! "$KAFKA_OPTS" == "-Djava.security.auth.login.config=/kafka_server_jaas.conf" ]]; then - echo "KAFKA_OPTS not set to expected value. $KAFKA_OPTS" - exit 1 - fi - - echo " > Set KAFKA_OPTS=$KAFKA_OPTS" -} - -testKafkaOpts diff --git a/tests/kafka_container/test/0.0/test.start-kafka-host-name.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-host-name.kafka.sh deleted file mode 100755 index 08dc2113..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-host-name.kafka.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testHostnameCommand() { - # Given a hostname command is provided - export HOSTNAME_COMMAND='f() { echo "my-host"; }; f' - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration uses the value from the command - assertExpectedConfig 'advertised.host.name=my-host' - assertAbsent 'advertised.listeners' - assertAbsent 'listeners' -} - -testHostnameCommand diff --git a/tests/kafka_container/test/0.0/test.start-kafka-log4j-config.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-log4j-config.kafka.sh deleted file mode 100755 index da4ff28e..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-log4j-config.kafka.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testLog4jConfig() { - # Given Log4j overrides are provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" - export LOG4J_LOGGER_KAFKA=DEBUG - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration file is correct - assertExpectedLog4jConfig "log4j.logger.kafka=DEBUG" -} - -testLog4jConfig diff --git a/tests/kafka_container/test/0.0/test.start-kafka-port-command.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-port-command.kafka.sh deleted file mode 100755 index 4c4d8833..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-port-command.kafka.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testPortCommand() { - # Given a port command is provided - export PORT_COMMAND='f() { echo "12345"; }; f' - export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://1.2.3.4:_{PORT_COMMAND}" - export KAFKA_LISTENERS="PLAINTEXT://:9092" - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration uses the value from the command - assertExpectedConfig 'advertised.listeners=PLAINTEXT://1.2.3.4:12345' - assertExpectedConfig 'listeners=PLAINTEXT://:9092' - assertAbsent 'advertised.host.name' - assertAbsent 'advertised.port' -} - -testPortCommand diff --git a/tests/kafka_container/test/0.0/test.start-kafka-restart.kafka.sh b/tests/kafka_container/test/0.0/test.start-kafka-restart.kafka.sh deleted file mode 100755 index 68a25598..00000000 --- a/tests/kafka_container/test/0.0/test.start-kafka-restart.kafka.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testRestart() { - # Given a hostname is provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" - - # When the container is restarted (Script invoked multiple times) - source "$START_KAFKA" - source "$START_KAFKA" - - # Then the configuration file only has one instance of the config - assertExpectedConfig 'advertised.host.name=testhost' - assertAbsent 'listeners' -} - -testRestart diff --git a/tests/kafka_container/test/0.10/test.create-topics.kafka.sh b/tests/kafka_container/test/0.10/test.create-topics.kafka.sh deleted file mode 100755 index e9dda7bd..00000000 --- a/tests/kafka_container/test/0.10/test.create-topics.kafka.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -e - -# NOTE: create-topics.sh requires KAFKA_PORT and KAFKA_ZOOKEEPER_CONNECT to be set (see docker-compose.yml) - -testCreateTopics() { - NOW=$(date +%s) - - # TOPICS array contains the topic name to create / validate - # CLEANUP array contains the expected cleanup policy configuration for the topic - TOPICS[0]="default-$NOW" - CLEANUP[0]="" - - TOPICS[1]="compact-$NOW" - CLEANUP[1]="compact,compression.type=snappy" - - KAFKA_CREATE_TOPICS="${TOPICS[0]}:1:1,${TOPICS[1]}:2:1:compact --config=compression.type=snappy" create-topics.sh - - # Loop through each array, validate that topic exists, and correct cleanup policy is set - for i in "${!TOPICS[@]}"; do - TOPIC=${TOPICS[i]} - - echo "Validating topic '$TOPIC'" - - EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC") - POLICY=$(/opt/kafka/bin/kafka-configs.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --entity-type topics --entity-name "$TOPIC" --describe | grep 'Configs for topic' | awk -F'cleanup.policy=' '{print $2}') - - RESULT="$EXISTS:$POLICY" - EXPECTED="$TOPIC:${CLEANUP[i]}" - - if [[ "$RESULT" != "$EXPECTED" ]]; then - echo "$TOPIC topic not configured correctly: '$RESULT'" - return 1 - fi - done - - return 0 -} - -# mock the netstat call as made by the create-topics.sh script -function netstat() { echo "1 2 3 :$KAFKA_PORT"; } -export -f netstat - -testCreateTopics diff --git a/tests/kafka_container/test/0.9/test.snappy.kafkacat.sh b/tests/kafka_container/test/0.9/test.snappy.kafkacat.sh deleted file mode 100755 index 1be89101..00000000 --- a/tests/kafka_container/test/0.9/test.snappy.kafkacat.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e - -source version.functions - -testSnappy() { - echo 'foo,bar' | eval "kafkacat -X compression.codec=snappy -b $BROKER_LIST $KAFKACAT_OPTS -P -D, -t snappy" - eval "kafkacat -X compression.codec=snappy -b $BROKER_LIST $KAFKACAT_OPTS -C -e -t snappy" -} - -testSnappy diff --git a/tests/kafka_container/test/0.9/test.start-kafka-advertised-listeners.kafka.sh b/tests/kafka_container/test/0.9/test.start-kafka-advertised-listeners.kafka.sh deleted file mode 100755 index 92ed26f9..00000000 --- a/tests/kafka_container/test/0.9/test.start-kafka-advertised-listeners.kafka.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testAdvertisedListeners() { - # Given a hostname is provided - export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://my.domain.com:9040" - export KAFKA_LISTENERS="PLAINTEXT://:9092" - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration file is correct - assertExpectedConfig 'advertised.listeners=PLAINTEXT://my.domain.com:9040' - assertExpectedConfig 'listeners=PLAINTEXT://:9092' -} - -testAdvertisedListeners diff --git a/tests/kafka_container/test/0.9/test.start-kafka-listeners.kafka.sh b/tests/kafka_container/test/0.9/test.start-kafka-listeners.kafka.sh deleted file mode 100755 index ee5abb10..00000000 --- a/tests/kafka_container/test/0.9/test.start-kafka-listeners.kafka.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e - -source test.functions - -testListeners() { - # Given a hostname is provided - export KAFKA_LISTENERS="PLAINTEXT://internal.domain.com:9040" - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration file is correct - assertAbsent 'advertised.host.name' - assertAbsent 'advertised.port' - assertAbsent 'advertised.listeners' - assertExpectedConfig 'listeners=PLAINTEXT://internal.domain.com:9040' -} - -testListeners diff --git a/tests/kafka_container/test/0.9/test.start-kafka-multiple-listeners.kafka.sh b/tests/kafka_container/test/0.9/test.start-kafka-multiple-listeners.kafka.sh deleted file mode 100755 index 0ca08f4e..00000000 --- a/tests/kafka_container/test/0.9/test.start-kafka-multiple-listeners.kafka.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -e -source test.functions - -testMultipleAdvertisedListeners() { - # Given multiple advertised listeners - export HOSTNAME_COMMAND="f() { echo 'monkey.domain'; }; f" - export KAFKA_ADVERTISED_LISTENERS="INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094" - export KAFKA_LISTENERS="INSIDE://:9092,OUTSIDE://:9094" - export KAFKA_LISTENER_SECURITY_PROTOCOL_MAP="INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" - export KAFKA_INTER_BROKER_LISTENER_NAME="INSIDE" - - # When the script is invoked - source "$START_KAFKA" - - # Then the configuration file is correct - assertAbsent "advertised.host.name" - assertAbsent "advertised.port" - - assertExpectedConfig "advertised.listeners=INSIDE://:9092,OUTSIDE://monkey.domain:9094" - assertExpectedConfig "listeners=INSIDE://:9092,OUTSIDE://:9094" - assertExpectedConfig "listener.security.protocol.map=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" - assertExpectedConfig "inter.broker.listener.name=INSIDE" -} - -testMultipleAdvertisedListeners diff --git a/tests/kafka_container/test/Readme.md b/tests/kafka_container/test/Readme.md deleted file mode 100644 index cdc72c19..00000000 --- a/tests/kafka_container/test/Readme.md +++ /dev/null @@ -1,31 +0,0 @@ -Tests -===== - -This directory contains some basic tests to validate functionality after building. - -To execute ----------- - -``` -cd test -docker-compose up -d zookeeper kafka_1 kafka_2 -./runAllTests.sh -``` - -Run selected tests ------------------- - -### Kafka - -``` -docker-compose run --rm kafkatest -``` - -### Kafkacat - -``` -BROKER_LIST=$(./internal-broker-list.sh) [KAFKA_VERSION=] docker-compose run --rm kafkacattest -``` - -- `` is the kafka version that the tests are targeting. Normally this environment variable should not need to be specified. The default should be the latest image version. Added for CI support. -- `` can be an individual filename, or a pattern such as `'0.0/test.start-kafka*.kafka.sh'` diff --git a/tests/kafka_container/test/docker-compose.yml b/tests/kafka_container/test/docker-compose.yml deleted file mode 100644 index 42a7ca59..00000000 --- a/tests/kafka_container/test/docker-compose.yml +++ /dev/null @@ -1,59 +0,0 @@ -version: '2.1' - -x-kafka-defaults: &kafka-defaults - image: wurstmeister/kafka - ports: - - "9092" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - -x-kafka-environment-defaults: &kafka-environment-defaults - HOSTNAME_COMMAND: "echo $$(hostname)" - KAFKA_ADVERTISED_PORT: 9092 - KAFKA_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - -services: - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181" - kafka_1: - <<: *kafka-defaults - container_name: test_kafka_1 - environment: - <<: *kafka-environment-defaults - KAFKA_BROKER_ID: 1 - kafka_2: - <<: *kafka-defaults - container_name: test_kafka_2 - environment: - <<: *kafka-environment-defaults - KAFKA_BROKER_ID: 2 - - kafkatest: - image: wurstmeister/kafka - environment: - KAFKA_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - volumes: - - .:/tests - - /var/run/docker.sock:/var/run/docker.sock - working_dir: /tests - entrypoint: - - ./runTestPattern.sh - command: - - "*/*.kafka.sh" - - kafkacattest: - image: confluentinc/cp-kafkacat:5.0.0 - environment: - - BROKER_LIST - - KAFKA_VERSION=${KAFKA_VERSION-2.7.0} - volumes: - - .:/tests - working_dir: /tests - entrypoint: - - ./runTestPattern.sh - command: - - "*/*.kafkacat.sh" diff --git a/tests/kafka_container/test/internal-broker-list.sh b/tests/kafka_container/test/internal-broker-list.sh deleted file mode 100755 index dac67a83..00000000 --- a/tests/kafka_container/test/internal-broker-list.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -CONTAINERS=$(docker inspect -f '{{ .NetworkSettings.Networks.test_default.IPAddress }}' test_kafka_1 test_kafka_2 | awk '{printf "%s:9092\n", $1}' | tr '\n' ',') -echo "${CONTAINERS%,}" diff --git a/tests/kafka_container/test/runAllTests.sh b/tests/kafka_container/test/runAllTests.sh deleted file mode 100755 index a575b21f..00000000 --- a/tests/kafka_container/test/runAllTests.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -e - -BROKER_LIST=$(./internal-broker-list.sh) -export BROKER_LIST - -echo "BROKER_LIST=$BROKER_LIST" - -runAll() { - # Tests that require kafka - docker-compose run --rm kafkatest - - RESULT=$? - if [[ $RESULT -eq 0 ]]; then - # Tests that require kafkacat - docker-compose run --rm kafkacattest - RESULT=$? - fi - - return $RESULT -} - -runAll -result=$? -echo "exit status $result" -exit $result diff --git a/tests/kafka_container/test/runTestPattern.sh b/tests/kafka_container/test/runTestPattern.sh deleted file mode 100755 index 16220eeb..00000000 --- a/tests/kafka_container/test/runTestPattern.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -e - -source version.functions - -PATTERN=$1 -VERSION=$KAFKA_VERSION - -# Allow version to be overridden by -v/--version flag -while [[ "$#" -gt 0 ]]; do - case $1 in - -v|--version) - VERSION="$2"; - shift - ;; - *) - PATTERN="$1" - ;; - esac - shift -done - -echo "" -echo "" -echo "Running tests for Kafka $VERSION with pattern $PATTERN" - -runPattern() { - for t in $PATTERN; do - echo - echo "====================================" - - # only run tests compatible with this version of Kafka - TARGET=$(echo "$t" | cut -d/ -f1) - RESULT=$(compareVersion "$VERSION" "$TARGET") - echo "Kafka $VERSION is '$RESULT' target $TARGET of test $t" - if [[ "$RESULT" != "<" ]]; then - echo " testing '$t'" - ( source "$t" ) - status=$? - if [[ -z "$status" || ! "$status" -eq 0 ]]; then - return $status - fi - fi - done - - return $? -} - -runPattern - -exit $? diff --git a/tests/kafka_container/test/scenarios/Readme.md b/tests/kafka_container/test/scenarios/Readme.md deleted file mode 100644 index 11c968f0..00000000 --- a/tests/kafka_container/test/scenarios/Readme.md +++ /dev/null @@ -1,27 +0,0 @@ -Scenarios (end-to-end tests) -============================ - -These tests are supposed to test the configuration of indivdiual features - -TODO: ------ - -- SSL (Client + Broker) -- Security - -Done: ------ - -- JMX - -Executing tests ---------------- - -These tests should spin up required containers for full end-to-end testing and exercise required code paths, returing zero exit code for success and non-zero exit code for failure. - -### JMX - -``` -cd test/scenarios -./runJmxScenario.sh -``` diff --git a/tests/kafka_container/test/scenarios/jmx/docker-compose.yml b/tests/kafka_container/test/scenarios/jmx/docker-compose.yml deleted file mode 100644 index aba78bf2..00000000 --- a/tests/kafka_container/test/scenarios/jmx/docker-compose.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: '2' -services: - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181" - kafka: - image: wurstmeister/kafka - ports: - - "9092" - - "1099" - environment: - KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_ADVERTISED_PORT: 9092 - KAFKA_PORT: 9092 - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka -Dcom.sun.management.jmxremote.rmi.port=1099" - JMX_PORT: 1099 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - jmxexporter: - image: sscaling/jmx-prometheus-exporter - ports: - - "5556:5556" - environment: - SERVICE_PORT: 5556 - volumes: - - $PWD/jmxexporter.yml:/opt/jmx_exporter/config.yml - - test: - image: wurstmeister/kafka - environment: - KAFKA_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - volumes: - - .:/scenario - working_dir: /scenario - entrypoint: - - /bin/bash - - -c - command: - - /scenario/test.sh - diff --git a/tests/kafka_container/test/scenarios/jmx/jmxexporter.yml b/tests/kafka_container/test/scenarios/jmx/jmxexporter.yml deleted file mode 100644 index f365449c..00000000 --- a/tests/kafka_container/test/scenarios/jmx/jmxexporter.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -startDelaySeconds: 3 -hostPort: kafka:1099 -username: -password: - -whitelistObjectNames: ["kafka.server:type=BrokerTopicMetrics,*"] -rules: - - pattern: ".*" diff --git a/tests/kafka_container/test/scenarios/jmx/test.sh b/tests/kafka_container/test/scenarios/jmx/test.sh deleted file mode 100755 index 286e9a0f..00000000 --- a/tests/kafka_container/test/scenarios/jmx/test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -e -o pipefail - -echo "Sleeping 5 seconds until Kafka is started" -sleep 5 - -echo "Checking to see if Kafka is alive" -echo "dump" | nc -w 20 zookeeper 2181 | fgrep "/brokers/ids/" - -echo "Check JMX" -curl -s jmxexporter:5556/metrics | grep 'kafka_server_BrokerTopicMetrics_MeanRate{name="MessagesInPerSec",' diff --git a/tests/kafka_container/test/scenarios/runJmxScenario.sh b/tests/kafka_container/test/scenarios/runJmxScenario.sh deleted file mode 100755 index e99b8851..00000000 --- a/tests/kafka_container/test/scenarios/runJmxScenario.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e -o pipefail - -pushd jmx -docker-compose up -d zookeeper kafka jmxexporter -docker-compose run --rm test -docker-compose stop -popd diff --git a/tests/kafka_container/test/test.functions b/tests/kafka_container/test/test.functions deleted file mode 100644 index 2894cadd..00000000 --- a/tests/kafka_container/test/test.functions +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -e - -# Sourcing this script will replace any of the external calls in the start-kafka.sh script and mock -# any outbound calls (i.e. to docker) - -export START_KAFKA=/usr/bin/start-kafka.sh -export BROKER_CONFIG=/opt/kafka/config/server.properties -export ORIG_LOG4J_CONFIG=/opt/kafka/config/log4j.properties - -enforceOriginalFile() { - SOURCE="$1" - BACKUP="$1.bak" - - if [[ ! -f "$BACKUP" ]]; then - echo "Backing up $SOURCE to $BACKUP" - cp "$SOURCE" "$BACKUP" - else - echo "Restoring $SOURCE from $BACKUP" - cp "$BACKUP" "$SOURCE" - fi -} - -setupStartKafkaScript() { - echo "Preparing $START_KAFKA script for test" - - enforceOriginalFile "$START_KAFKA" - enforceOriginalFile "$BROKER_CONFIG" - enforceOriginalFile "$ORIG_LOG4J_CONFIG" - - # We need to remove all executable commands from the start-kafka.sh script, so it can be sourced to evaluate - # the environment variables - sed -i -E -e '/^create-topics.sh/d' -e '/^exec "\$KAFKA_HOME\/bin\/kafka-server-start.sh"/d' "$START_KAFKA" - - # Mock the call to docker port to return valid result - function docker() { echo "0.0.0.0:9092"; } - export -f docker -} - -setupStartKafkaScript - - -# Will look in the server.properties file, and check if there is an exact match for the provided input -# i.e. `assertExpectedConfig 'broker.id=123'` -# This will only succeed if there is exactly one line matching `broker.id=123` -assertExpectedConfig() { - EXPECTED=$1 - - COUNT=$(grep -E '^'"$EXPECTED"'$' "$BROKER_CONFIG" | wc -l) - RESULT=$(grep -E '^'"$EXPECTED"'$' "$BROKER_CONFIG") - echo " > $COUNT matches of $RESULT" - - [[ "$RESULT" == "$EXPECTED" && "$COUNT" == "1" ]] -} - -assertExpectedLog4jConfig() { - EXPECTED=$1 - - RESULT=$(grep -E '^'"$EXPECTED"'$' "$ORIG_LOG4J_CONFIG") - echo " > $RESULT" - - [[ "$RESULT" == "$EXPECTED" ]] -} - - -assertAbsent() { - EXPECTED_ABSENT=$1 - - RESULT=$(grep -E '^'"$EXPECTED_ABSENT" "$BROKER_CONFIG" | wc -l) - echo " > $RESULT matches for ^$EXPECTED_ABSENT" - - [[ "$RESULT" == "0" ]] -} - -printBrokerConfig() { - echo "----[ $BROKER_CONFIG ]----" - cat "$BROKER_CONFIG" | sed -nE '/^[^#]/p' - echo "--------------------------" -} diff --git a/tests/kafka_container/test/verifyImageLabels.sh b/tests/kafka_container/test/verifyImageLabels.sh deleted file mode 100755 index f59c9985..00000000 --- a/tests/kafka_container/test/verifyImageLabels.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e - -VCS_REF=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.vcs-ref"}}' wurstmeister/kafka) -echo "VCS_REF=$VCS_REF" -if [ -z "$VCS_REF" ] || [ "$VCS_REF" = "unspecified" ]; then - echo "org.label-schema.vcs-ref is empty or unspecified" - exit 1 -fi -if ! git cat-file -e "$VCS_REF^{commit}"; then - echo "$VCS_REF Not a valid git commit" - exit 1 -fi - -BUILD_DATE=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.build-date"}}' wurstmeister/kafka) -echo "BUILD_DATE=$BUILD_DATE" -if ! date -d "$BUILD_DATE"; then - echo "$BUILD_DATE Not a valid date" - exit 1 -fi -exit 0 diff --git a/tests/kafka_container/test/version.functions b/tests/kafka_container/test/version.functions deleted file mode 100644 index 65c00f60..00000000 --- a/tests/kafka_container/test/version.functions +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -e - -# Modified from https://stackoverflow.com/a/4025065 -compareVersion() { - # Only care about major / minor - LEFT=$(echo "$1" | cut -d. -f1-2) - RIGHT=$(echo "$2" | cut -d. -f1-2) - if [[ "$LEFT" != "$RIGHT" ]] - then - local IFS=. - local i ver1=($LEFT) ver2=($RIGHT) - for ((i=0; i<${#ver1[@]}; i++)) - do - if (( "${ver1[i]}" > "${ver2[i]}" )) - then - echo ">" - return - fi - if (( "${ver1[i]}" < "${ver2[i]}" )) - then - echo "<" - return - fi - done - fi - echo "=" -} - -# https://github.com/edenhill/librdkafka/wiki/Broker-version-compatibility -# To support different broker versions, we need to configure kafkacat differently -VERSION_8=$(compareVersion "$KAFKA_VERSION" "0.8") -VERSION_9=$(compareVersion "$KAFKA_VERSION" "0.9") - -if [[ "$VERSION_8" == "=" || "$VERSION_9" == "=" ]]; then - export KAFKACAT_OPTS="-Xapi.version.request=false -Xbroker.version.fallback=$KAFKA_VERSION" - echo "[INFO] Using kafkacat opts on older version '$KAFKACAT_OPTS'" -fi diff --git a/tests/kafka_container/versions.sh b/tests/kafka_container/versions.sh deleted file mode 100755 index d790d1a4..00000000 --- a/tests/kafka_container/versions.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -e - -MAJOR_VERSION=$(echo "$KAFKA_VERSION" | cut -d. -f1) -export MAJOR_VERSION - -MINOR_VERSION=$(echo "$KAFKA_VERSION" | cut -d. -f2) -export MINOR_VERSION diff --git a/tests/local_db.lmdb/data.mdb b/tests/local_db.lmdb/data.mdb deleted file mode 100644 index 16266c96..00000000 Binary files a/tests/local_db.lmdb/data.mdb and /dev/null differ diff --git a/tests/local_db.lmdb/lock.mdb b/tests/local_db.lmdb/lock.mdb deleted file mode 100644 index 8aa90b29..00000000 Binary files a/tests/local_db.lmdb/lock.mdb and /dev/null differ diff --git a/tests/services/CommandTestService.py b/tests/services/CommandTestService.py new file mode 100644 index 00000000..be12d04c --- /dev/null +++ b/tests/services/CommandTestService.py @@ -0,0 +1,17 @@ +from minos.common import ( + Command, +) + + +class CommandService(object): + async def get_order(self, topic: str, command: Command): + return "get_order" + + async def add_order(self, topic: str, command: Command): + return "add_order" + + async def delete_order(self, topic: str, command: Command): + return "delete_order" + + async def update_order(self, topic: str, command: Command): + return "update_order" diff --git a/tests/services/CqrsTestService.py b/tests/services/CqrsTestService.py index 512422eb..cfaf0693 100644 --- a/tests/services/CqrsTestService.py +++ b/tests/services/CqrsTestService.py @@ -1,3 +1,11 @@ +from minos.common import ( + Event, +) + + class CqrsService(object): - def ticket_added(self, request): + async def ticket_added(self, topic: str, event: Event): return "request_added" + + async def ticket_deleted(self, topic: str, event: Event): + return "ticket_deleted" diff --git a/tests/services/SagaTestService.py b/tests/services/SagaTestService.py new file mode 100644 index 00000000..eb1e37ee --- /dev/null +++ b/tests/services/SagaTestService.py @@ -0,0 +1,11 @@ +from minos.common import ( + CommandReply, +) + + +class SagaService(object): + async def add_order(self, topic: str, command: CommandReply): + return "add_order_saga" + + async def delete_order(self, topic: str, command: CommandReply): + return "delete_order_saga" diff --git a/tests/services/TestRestService.py b/tests/services/TestRestService.py new file mode 100644 index 00000000..4b214303 --- /dev/null +++ b/tests/services/TestRestService.py @@ -0,0 +1,11 @@ +from aiohttp import ( + web, +) + + +class RestService(object): + async def add_order(self, request): + return web.Response(text="Order added") + + async def get_order(self, request): + return web.Response(text="Order get") diff --git a/tests/test_config.yml b/tests/test_config.yml new file mode 100644 index 00000000..37f46669 --- /dev/null +++ b/tests/test_config.yml @@ -0,0 +1,86 @@ +service: + name: Order +rest: + host: localhost + port: 8080 + endpoints: + - name: AddOrder + route: /order + method: POST + controller: tests.services.TestRestService.RestService + action: add_order + - name: GetOrder + route: /order + method: GET + controller: tests.services.TestRestService.RestService + action: get_order +repository: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 +snapshot: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 +events: + broker: localhost + port: 9092 + items: + - name: TicketAdded + controller: tests.services.CqrsTestService.CqrsService + action: ticket_added + - name: TicketDeleted + controller: tests.services.CqrsTestService.CqrsService + action: ticket_deleted + queue: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 + records: 10 + retry: 2 +commands: + broker: localhost + port: 9092 + items: + - name: AddOrder + controller: tests.services.CommandTestService.CommandService + action: add_order + - name: DeleteOrder + controller: tests.services.CommandTestService.CommandService + action: delete_order + - name: UpdateOrder + controller: tests.services.CommandTestService.CommandService + action: update_order + - name: GetOrder + controller: tests.service.CommandTestService.CommandService + action: get_order + queue: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 + records: 10 + retry: 2 +saga: + items: + - name: AddOrder + controller: tests.services.SagaTestService.SagaService + action: add_order + - name: DeleteOrder + controller: tests.services.SagaTestService.SagaService + action: delete_order + queue: + database: order_db + user: minos + password: min0s + host: postgres + port: 5432 + records: 10 + retry: 2 diff --git a/tests/test_event_manager.py b/tests/test_event_manager.py deleted file mode 100644 index 58ca9b8d..00000000 --- a/tests/test_event_manager.py +++ /dev/null @@ -1,44 +0,0 @@ -import asyncio -import logging - -import pytest -import string -from aiokafka import AIOKafkaProducer -import random - -from aiomisc.log import basic_config -from minos.common.configuration.config import MinosConfig - -from minos.networks.event import MinosEventServer - - -@pytest.fixture() -def config(): - return MinosConfig(path='./tests/test_config.yaml') - - -@pytest.fixture() -def services(config): - return [MinosEventServer(conf=config)] - - -async def test_producer_kafka(loop): - basic_config( - level=logging.INFO, - buffered=True, - log_format='color', - flush_interval=2 - ) - - producer = AIOKafkaProducer(loop=loop, bootstrap_servers='localhost:9092') - # Get cluster layout and topic/partition allocation - await producer.start() - # Produce messages - string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=20)) - await producer.send_and_wait("TicketAdded", string_to_send.encode()) - await asyncio.sleep(1) - - other_string_to_send = ''.join(random.choices(string.ascii_uppercase + string.digits, k=40)) - await producer.send_and_wait("TicketDeleted", other_string_to_send.encode()) - await asyncio.sleep(1) - await producer.stop() diff --git a/tests/test_networks/__init__.py b/tests/test_networks/__init__.py new file mode 100644 index 00000000..fed3716d --- /dev/null +++ b/tests/test_networks/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" diff --git a/tests/services/__init__.py b/tests/test_networks/rest_interface/__init__.py similarity index 100% rename from tests/services/__init__.py rename to tests/test_networks/rest_interface/__init__.py diff --git a/tests/test_networks/rest_interface/test_interface.py b/tests/test_networks/rest_interface/test_interface.py new file mode 100644 index 00000000..e1a46844 --- /dev/null +++ b/tests/test_networks/rest_interface/test_interface.py @@ -0,0 +1,36 @@ +from aiohttp.test_utils import ( + AioHTTPTestCase, + unittest_run_loop, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.networks.rest_interface import ( + RestInterfaceHandler, +) +from tests.utils import ( + BASE_PATH, +) + +""" +class TestInterface(AioHTTPTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def get_application(self): + rest_interface = RestInterfaceHandler(config=MinosConfig(self.CONFIG_FILE_PATH)) + + return rest_interface.get_app() + + @unittest_run_loop + async def test_methods(self): + url = "/order" + resp = await self.client.request("GET", url) + assert resp.status == 200 + text = await resp.text() + assert "Order get" in text + + resp = await self.client.request("POST", url) + assert resp.status == 200 + text = await resp.text() + assert "Order added" in text +""" diff --git a/tests/test_networks/rest_interface/test_service.py b/tests/test_networks/rest_interface/test_service.py new file mode 100644 index 00000000..ecb13f2c --- /dev/null +++ b/tests/test_networks/rest_interface/test_service.py @@ -0,0 +1,42 @@ +from aiohttp.test_utils import ( + AioHTTPTestCase, + unittest_run_loop, +) +from minos.common.configuration.config import ( + MinosConfig, +) +from minos.networks import ( + REST, +) +from tests.utils import ( + BASE_PATH, +) + + +class TestRestInterfaceService(AioHTTPTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def get_application(self): + """ + Override the get_app method to return your application. + """ + config = MinosConfig(self.CONFIG_FILE_PATH) + rest_interface = REST(config=config) + + return await rest_interface.create_application() + + @unittest_run_loop + async def test_methods(self): + url = "/order" + resp = await self.client.request("GET", url) + assert resp.status == 200 + text = await resp.text() + assert "Order get" in text + + resp = await self.client.request("POST", url) + assert resp.status == 200 + text = await resp.text() + assert "Order added" in text + + resp = await self.client.request("GET", "/system/health") + assert resp.status == 200 diff --git a/tests/test_networks/test_broker/__init__.py b/tests/test_networks/test_broker/__init__.py new file mode 100644 index 00000000..fed3716d --- /dev/null +++ b/tests/test_networks/test_broker/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" diff --git a/tests/test_networks/test_broker/test_command.py b/tests/test_networks/test_broker/test_command.py new file mode 100644 index 00000000..3093c086 --- /dev/null +++ b/tests/test_networks/test_broker/test_command.py @@ -0,0 +1,112 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +import unittest + +import aiopg +from minos.common import ( + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandBroker, + MinosQueueDispatcher, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) + + +class TestMinosCommandBroker(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_commands_broker_insertion(self): + broker = MinosCommandBroker.from_config( + "CommandBroker", + config=self.config, + saga_id="9347839473kfslf", + task_id="92839283hjijh232", + reply_on="test_reply_on", + ) + await broker.setup() + + item = NaiveAggregate(test_id=1, test=2, id=1, version=1) + + queue_id = await broker.send_one(item) + assert queue_id > 0 + + async def test_if_commands_was_deleted(self): + broker = MinosCommandBroker.from_config( + "CommandBroker-Delete", + config=self.config, + saga_id="9347839473kfslf", + task_id="92839283hjijh232", + reply_on="test_reply_on", + ) + await broker.setup() + + item = NaiveAggregate(test_id=1, test=2, id=1, version=1) + + queue_id_1 = await broker.send_one(item) + queue_id_2 = await broker.send_one(item) + + await MinosQueueDispatcher.from_config(config=self.config).dispatch() + + async with aiopg.connect(**self.events_queue_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "CommandBroker-Delete") + records = await cursor.fetchone() + + assert queue_id_1 > 0 + assert queue_id_2 > 0 + assert records[0] == 0 + + async def test_if_commands_retry_was_incremented(self): + broker = MinosCommandBroker.from_config( + "CommandBroker-Delete", + config=self.config, + saga_id="9347839473kfslf", + task_id="92839283hjijh232", + reply_on="test_reply_on", + ) + await broker.setup() + + item = NaiveAggregate(test_id=1, test=2, id=1, version=1) + + queue_id_1 = await broker.send_one(item) + queue_id_2 = await broker.send_one(item) + + config = MinosConfig( + path=BASE_PATH / "wrong_test_config.yml", + events_queue_database=self.config.events.queue.database, + events_queue_user=self.config.events.queue.user, + ) + await MinosQueueDispatcher.from_config(config=config).dispatch() + + async with aiopg.connect(**self.events_queue_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "CommandBroker-Delete") + records = await cursor.fetchone() + + await cursor.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_1) + retry_1 = await cursor.fetchone() + + await cursor.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_2) + retry_2 = await cursor.fetchone() + + assert queue_id_1 > 0 + assert queue_id_2 > 0 + assert records[0] == 2 + assert retry_1[0] > 0 + assert retry_2[0] > 0 + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_networks/test_broker/test_dispatcher.py b/tests/test_networks/test_broker/test_dispatcher.py new file mode 100644 index 00000000..be51f22d --- /dev/null +++ b/tests/test_networks/test_broker/test_dispatcher.py @@ -0,0 +1,40 @@ +import unittest + +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosQueueDispatcher, +) +from tests.utils import ( + BASE_PATH, +) + + +class TestQueueDispatcher(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosQueueDispatcher.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosQueueDispatcher) + + def test_from_config_raises(self): + with self.assertRaises(MinosConfigException): + MinosQueueDispatcher.from_config() + + async def test_select(self): + dispatcher = MinosQueueDispatcher.from_config(config=self.config) + await dispatcher.setup() + self.assertEqual([], [v async for v in dispatcher.select()]) + + async def test_send_to_kafka_ok(self): + dispatcher = MinosQueueDispatcher.from_config(config=self.config) + response = await dispatcher.publish(topic="TestKafkaSend", message=bytes()) + assert response is True + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_networks/test_broker/test_events.py b/tests/test_networks/test_broker/test_events.py new file mode 100644 index 00000000..5cbcd721 --- /dev/null +++ b/tests/test_networks/test_broker/test_events.py @@ -0,0 +1,102 @@ +import unittest + +import aiopg +from minos.common import ( + MinosConfig, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosEventBroker, + MinosQueueDispatcher, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) + + +class TestMinosEventBroker(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_if_queue_table_exists(self): + broker = MinosEventBroker.from_config("EventBroker", config=self.config) + await broker.setup() + + async with aiopg.connect(**self.events_queue_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'producer_queue';" + ) + ret = [] + async for row in cursor: + ret.append(row) + + assert ret == [(1,)] + + async def test_events_broker_insertion(self): + broker = MinosEventBroker.from_config("EventBroker", config=self.config) + await broker.setup() + + item = NaiveAggregate(test_id=1, test=2, id=1, version=1) + queue_id = await broker.send_one(item) + + assert queue_id > 0 + + async def test_if_events_was_deleted(self): + broker = MinosEventBroker.from_config("EventBroker-Delete", config=self.config) + await broker.setup() + + item = NaiveAggregate(test_id=1, test=2, id=1, version=1) + queue_id_1 = await broker.send_one(item) + queue_id_2 = await broker.send_one(item) + + await MinosQueueDispatcher.from_config(config=self.config).dispatch() + + async with aiopg.connect(**self.events_queue_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "EventBroker-Delete") + records = await cursor.fetchone() + + assert queue_id_1 > 0 + assert queue_id_2 > 0 + assert records[0] == 0 + + async def test_if_events_retry_was_incremented(self): + broker = MinosEventBroker.from_config("EventBroker-Delete", config=self.config) + await broker.setup() + + item = NaiveAggregate(test_id=1, test=2, id=1, version=1) + + queue_id_1 = await broker.send_one(item) + queue_id_2 = await broker.send_one(item) + + config = MinosConfig( + path=BASE_PATH / "wrong_test_config.yml", + events_queue_database=self.config.commands.queue.database, + events_queue_user=self.config.commands.queue.user, + ) + + await MinosQueueDispatcher.from_config(config=config).dispatch() + + async with aiopg.connect(**self.events_queue_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute("SELECT COUNT(*) FROM producer_queue WHERE topic = '%s'" % "EventBroker-Delete") + records = await cursor.fetchone() + + await cursor.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_1) + retry_1 = await cursor.fetchone() + + await cursor.execute("SELECT retry FROM producer_queue WHERE id=%d;" % queue_id_2) + retry_2 = await cursor.fetchone() + + assert queue_id_1 > 0 + assert queue_id_2 > 0 + assert records[0] == 2 + assert retry_1[0] > 0 + assert retry_2[0] > 0 + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_networks/test_broker/test_service.py b/tests/test_networks/test_broker/test_service.py new file mode 100644 index 00000000..86d47af2 --- /dev/null +++ b/tests/test_networks/test_broker/test_service.py @@ -0,0 +1,71 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +import unittest +from unittest.mock import ( + MagicMock, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosQueueDispatcher, + MinosQueueService, +) +from tests.utils import ( + BASE_PATH, +) + + +class TestMinosQueueService(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_is_instance(self): + with self.config: + service = MinosQueueService(interval=0.1) + self.assertIsInstance(service, PeriodicService) + + def test_dispatcher_empty(self): + with self.assertRaises(MinosConfigException): + MinosQueueService(interval=0.1) + + def test_dispatcher_config(self): + service = MinosQueueService(interval=0.1, config=self.config) + dispatcher = service.dispatcher + self.assertIsInstance(dispatcher, MinosQueueDispatcher) + self.assertFalse(dispatcher.already_setup) + + def test_dispatcher_config_context(self): + with self.config: + service = MinosQueueService(interval=0.1) + self.assertIsInstance(service.dispatcher, MinosQueueDispatcher) + + async def test_start(self): + service = MinosQueueService(interval=0.1, loop=None, config=self.config) + service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) + await service.start() + self.assertTrue(1, service.dispatcher.setup.call_count) + await service.stop() + + async def test_callback(self): + service = MinosQueueService(interval=0.1, loop=None, config=self.config) + await service.dispatcher.setup() + service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) + await service.callback() + self.assertEqual(1, service.dispatcher.dispatch.call_count) + await service.dispatcher.destroy() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_networks/test_exceptions.py b/tests/test_networks/test_exceptions.py new file mode 100644 index 00000000..590d28f3 --- /dev/null +++ b/tests/test_networks/test_exceptions.py @@ -0,0 +1,45 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +import unittest + +from minos.common import ( + MinosException, +) +from minos.networks import ( + MinosNetworkException, + MinosPreviousVersionSnapshotException, + MinosSnapshotException, +) +from tests.aggregate_classes import ( + Car, +) + + +class TestExceptions(unittest.TestCase): + def test_type(self): + self.assertTrue(issubclass(MinosNetworkException, MinosException)) + + def test_snapshot(self): + self.assertTrue(issubclass(MinosSnapshotException, MinosNetworkException)) + + def test_snapshot_previous_version(self): + self.assertTrue(issubclass(MinosPreviousVersionSnapshotException, MinosSnapshotException)) + + def test_snapshot_previous_version_repr(self): + previous = Car(1, 2, 3, "blue") + new = Car(1, 1, 5, "blue") + exception = MinosPreviousVersionSnapshotException(previous, new) + expected = ( + "MinosPreviousVersionSnapshotException(message=\"Version for 'tests.aggregate_classes.Car' " + 'aggregate must be greater than 2. Obtained: 1")' + ) + self.assertEqual(expected, repr(exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_networks/test_handler/__init__.py b/tests/test_networks/test_handler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_networks/test_handler/test_command/__init__.py b/tests/test_networks/test_handler/test_command/__init__.py new file mode 100644 index 00000000..fed3716d --- /dev/null +++ b/tests/test_networks/test_handler/test_command/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" diff --git a/tests/test_networks/test_handler/test_command/test_command_server.py b/tests/test_networks/test_handler/test_command/test_command_server.py new file mode 100644 index 00000000..de29345d --- /dev/null +++ b/tests/test_networks/test_handler/test_command/test_command_server.py @@ -0,0 +1,86 @@ +from collections import ( + namedtuple, +) + +from minos.common import ( + Command, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandHandlerServer, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) + + +class TestCommandServer(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosCommandHandlerServer.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosCommandHandlerServer) + + async def test_none_config(self): + event_server = MinosCommandHandlerServer.from_config(config=None) + + self.assertIsNone(event_server) + + async def test_queue_add(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Command( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) + bin_data = event_instance.avro_bytes + Command.from_avro_bytes(bin_data) + + event_server = MinosCommandHandlerServer.from_config(config=self.config) + await event_server.setup() + + affected_rows, id = await event_server.queue_add(topic=event_instance.topic, partition=0, binary=bin_data) + + assert affected_rows == 1 + assert id > 0 + + async def test_handle_message(self): + event_server = MinosCommandHandlerServer.from_config(config=self.config) + await event_server.setup() + + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Command( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) + bin_data = event_instance.avro_bytes + + Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) + + async def consumer(): + yield Mensaje(topic="TicketAdded", partition=0, value=bin_data) + + await event_server.handle_message(consumer()) + + async def test_handle_message_ko(self): + event_server = MinosCommandHandlerServer.from_config(config=self.config) + await event_server.setup() + + bin_data = bytes(b"test") + + Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) + + async def consumer(): + yield Mensaje(topic="TicketAdded", partition=0, value=bin_data) + + await event_server.handle_message(consumer()) diff --git a/tests/test_networks/test_handler/test_command/test_command_services.py b/tests/test_networks/test_handler/test_command/test_command_services.py new file mode 100644 index 00000000..c8eaea79 --- /dev/null +++ b/tests/test_networks/test_handler/test_command/test_command_services.py @@ -0,0 +1,52 @@ +import unittest +from unittest.mock import ( + MagicMock, +) + +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandPeriodicService, + MinosCommandServerService, +) +from tests.utils import ( + BASE_PATH, +) + + +class TestMinosCommandServices(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_start(self): + service = MinosCommandServerService(loop=None, config=self.config) + + async def _fn(consumer): + self.assertEqual(service.consumer, consumer) + + mock = MagicMock(side_effect=_fn) + service.dispatcher.handle_message = mock + await service.start() + self.assertTrue(1, mock.call_count) + await service.stop() + + +class TestMinosQueueService(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_start(self): + service = MinosCommandPeriodicService(interval=1, loop=None, config=self.config) + mock = MagicMock(side_effect=service.dispatcher.setup) + service.dispatcher.setup = mock + await service.start() + self.assertTrue(1, mock.call_count) + await service.stop() + + async def test_callback(self): + service = MinosCommandPeriodicService(interval=1, loop=None, config=self.config) + await service.dispatcher.setup() + mock = MagicMock(side_effect=service.dispatcher.queue_checker) + service.dispatcher.queue_checker = mock + await service.callback() + self.assertEqual(1, mock.call_count) + await service.dispatcher.destroy() diff --git a/tests/test_networks/test_handler/test_command/test_dispatcher.py b/tests/test_networks/test_handler/test_command/test_dispatcher.py new file mode 100644 index 00000000..b448a507 --- /dev/null +++ b/tests/test_networks/test_handler/test_command/test_dispatcher.py @@ -0,0 +1,152 @@ +import datetime + +import aiopg +from minos.common import ( + Command, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandHandlerDispatcher, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) + + +class TestCommandDispatcher(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosCommandHandlerDispatcher.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosCommandHandlerDispatcher) + + async def test_if_queue_table_exists(self): + handler = MinosCommandHandlerDispatcher.from_config(config=self.config) + await handler.setup() + + async with aiopg.connect(**self.commands_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'command_queue';" + ) + ret = [] + async for row in cur: + ret.append(row) + + assert ret == [(1,)] + + async def test_get_event_handler(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Command( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) + m = MinosCommandHandlerDispatcher.from_config(config=self.config) + + cls = m.get_event_handler(topic=event_instance.topic) + result = await cls(topic=event_instance.topic, command=event_instance) + + assert result == "add_order" + + async def test_non_implemented_action(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + instance = Command( + topic="NotExisting", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) + m = MinosCommandHandlerDispatcher.from_config(config=self.config) + + with self.assertRaises(MinosNetworkException) as context: + cls = m.get_event_handler(topic=instance.topic) + await cls(topic=instance.topic, command=instance) + + self.assertTrue( + "topic NotExisting have no controller/action configured, please review th configuration file" + in str(context.exception) + ) + + async def test_none_config(self): + handler = MinosCommandHandlerDispatcher.from_config(config=None) + + self.assertIsNone(handler) + + async def test_event_queue_checker(self): + handler = MinosCommandHandlerDispatcher.from_config(config=self.config) + await handler.setup() + + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + instance = Command( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) + bin_data = instance.avro_bytes + Command.from_avro_bytes(bin_data) + + async with aiopg.connect(**self.commands_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO command_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + (instance.topic, 0, bin_data, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + assert affected_rows == 1 + assert queue_id[0] > 0 + + # Must get the record, call on_reply function and delete the record from DB + await handler.queue_checker() + + async with aiopg.connect(**self.commands_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM command_queue WHERE id=%d" % (queue_id)) + records = await cur.fetchone() + + assert records[0] == 0 + + async def test_event_queue_checker_wrong_event(self): + handler = MinosCommandHandlerDispatcher.from_config(config=self.config) + await handler.setup() + bin_data = bytes(b"Test") + + async with aiopg.connect(**self.commands_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO command_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + ("AddOrder", 0, bin_data, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + assert affected_rows == 1 + assert queue_id[0] > 0 + + # Must get the record, call on_reply function and delete the record from DB + await handler.queue_checker() + + async with aiopg.connect(**self.commands_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM command_queue WHERE id=%d" % (queue_id)) + records = await cur.fetchone() + + assert records[0] == 1 diff --git a/tests/test_networks/test_handler/test_command_reply/__init__.py b/tests/test_networks/test_handler/test_command_reply/__init__.py new file mode 100644 index 00000000..fed3716d --- /dev/null +++ b/tests/test_networks/test_handler/test_command_reply/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" diff --git a/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py b/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py new file mode 100644 index 00000000..288564f3 --- /dev/null +++ b/tests/test_networks/test_handler/test_command_reply/test_command__reply_server.py @@ -0,0 +1,86 @@ +from collections import ( + namedtuple, +) + +from minos.common import ( + CommandReply, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandReplyHandlerServer, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) + + +class TestCommandReplyServer(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosCommandReplyHandlerServer.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosCommandReplyHandlerServer) + + async def test_none_config(self): + event_server = MinosCommandReplyHandlerServer.from_config(config=None) + + self.assertIsNone(event_server) + + async def test_queue_add(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = CommandReply( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) + bin_data = event_instance.avro_bytes + CommandReply.from_avro_bytes(bin_data) + + event_server = MinosCommandReplyHandlerServer.from_config(config=self.config) + await event_server.setup() + + affected_rows, id = await event_server.queue_add(topic=event_instance.topic, partition=0, binary=bin_data) + + assert affected_rows == 1 + assert id > 0 + + async def test_handle_message(self): + event_server = MinosCommandReplyHandlerServer.from_config(config=self.config) + await event_server.setup() + + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = CommandReply( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) + bin_data = event_instance.avro_bytes + + Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) + + async def consumer(): + yield Mensaje(topic="AddOrder", partition=0, value=bin_data) + + await event_server.handle_message(consumer()) + + async def test_handle_message_ko(self): + event_server = MinosCommandReplyHandlerServer.from_config(config=self.config) + await event_server.setup() + + bin_data = bytes(b"test") + + Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) + + async def consumer(): + yield Mensaje(topic="AddOrder", partition=0, value=bin_data) + + await event_server.handle_message(consumer()) diff --git a/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py b/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py new file mode 100644 index 00000000..a193743d --- /dev/null +++ b/tests/test_networks/test_handler/test_command_reply/test_command_reply_services.py @@ -0,0 +1,51 @@ +from unittest.mock import ( + MagicMock, +) + +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandReplyPeriodicService, + MinosCommandReplyServerService, +) +from tests.utils import ( + BASE_PATH, +) + + +class TestMinosCommandReplyServices(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_start(self): + service = MinosCommandReplyServerService(loop=None, config=self.config) + + async def _fn(consumer): + self.assertEqual(service.consumer, consumer) + + mock = MagicMock(side_effect=_fn) + service.dispatcher.handle_message = mock + await service.start() + self.assertTrue(1, mock.call_count) + await service.stop() + + +class TestMinosCommandReplyQueueService(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_start(self): + service = MinosCommandReplyPeriodicService(interval=1, loop=None, config=self.config) + mock = MagicMock(side_effect=service.dispatcher.setup) + service.dispatcher.setup = mock + await service.start() + self.assertTrue(1, mock.call_count) + await service.stop() + + async def test_callback(self): + service = MinosCommandReplyPeriodicService(interval=1, loop=None, config=self.config) + await service.dispatcher.setup() + mock = MagicMock(side_effect=service.dispatcher.queue_checker) + service.dispatcher.queue_checker = mock + await service.callback() + self.assertEqual(1, mock.call_count) + await service.dispatcher.destroy() diff --git a/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py b/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py new file mode 100644 index 00000000..2e989184 --- /dev/null +++ b/tests/test_networks/test_handler/test_command_reply/test_dispatcher.py @@ -0,0 +1,171 @@ +import datetime + +import aiopg +from minos.common import ( + CommandReply, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosCommandReplyHandlerDispatcher, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) + + +class TestCommandReplyDispatcher(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosCommandReplyHandlerDispatcher) + + async def test_if_queue_table_exists(self): + handler = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) + await handler.setup() + self._meta_saga_queue_db = self._config.saga.queue._asdict() + self._meta_saga_queue_db.pop("records") + self._meta_saga_queue_db.pop("retry") + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'command_reply_queue';" + ) + ret = [] + async for row in cur: + ret.append(row) + + assert ret == [(1,)] + + async def test_get_event_handler(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = CommandReply( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) + m = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) + + cls = m.get_event_handler(topic=event_instance.topic) + result = await cls(topic=event_instance.topic, command=event_instance) + + assert result == "add_order_saga" + + async def test_non_implemented_action(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + instance = CommandReply( + topic="NotExisting", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) + m = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) + + with self.assertRaises(MinosNetworkException) as context: + cls = m.get_event_handler(topic=instance.topic) + await cls(topic=instance.topic, command=instance) + + self.assertTrue( + "topic NotExisting have no controller/action configured, please review th configuration file" + in str(context.exception) + ) + + async def test_none_config(self): + handler = MinosCommandReplyHandlerDispatcher.from_config(config=None) + + self.assertIsNone(handler) + + async def test_event_queue_checker(self): + self._meta_saga_queue_db = self._config.saga.queue._asdict() + self._meta_saga_queue_db.pop("records") + self._meta_saga_queue_db.pop("retry") + + handler = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) + await handler.setup() + + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("DELETE FROM {table};".format(table=MinosCommandReplyHandlerDispatcher.TABLE)) + + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + instance = CommandReply( + topic="AddOrder", + model=model.classname, + items=[], + saga_id="43434jhij", + task_id="juhjh34", + reply_on="mkk2334", + ) + bin_data = instance.avro_bytes + CommandReply.from_avro_bytes(bin_data) + + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO command_reply_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + (instance.topic, 0, bin_data, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + assert affected_rows == 1 + assert queue_id[0] > 0 + + # Must get the record, call on_reply function and delete the record from DB + await handler.queue_checker() + + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM command_reply_queue WHERE id=%d" % (queue_id)) + records = await cur.fetchone() + + assert records[0] == 0 + + async def test_command_reply_queue_checker_wrong_event(self): + self._meta_saga_queue_db = self._config.saga.queue._asdict() + self._meta_saga_queue_db.pop("records") + self._meta_saga_queue_db.pop("retry") + + handler = MinosCommandReplyHandlerDispatcher.from_config(config=self.config) + await handler.setup() + + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("DELETE FROM {table};".format(table=MinosCommandReplyHandlerDispatcher.TABLE)) + + bin_data = bytes(b"Test") + + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO command_reply_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + ("AddOrder", 0, bin_data, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + assert affected_rows == 1 + assert queue_id[0] > 0 + + # Must get the record, call on_reply function and delete the record from DB + await handler.queue_checker() + + async with aiopg.connect(**self._meta_saga_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM command_reply_queue WHERE id=%d" % (queue_id)) + records = await cur.fetchone() + + assert records[0] == 1 diff --git a/tests/test_networks/test_handler/test_event/__init__.py b/tests/test_networks/test_handler/test_event/__init__.py new file mode 100644 index 00000000..fed3716d --- /dev/null +++ b/tests/test_networks/test_handler/test_event/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" diff --git a/tests/test_networks/test_handler/test_event/test_dispatcher.py b/tests/test_networks/test_handler/test_event/test_dispatcher.py new file mode 100644 index 00000000..f8ce87ce --- /dev/null +++ b/tests/test_networks/test_handler/test_event/test_dispatcher.py @@ -0,0 +1,131 @@ +import datetime + +import aiopg +from minos.common import ( + Event, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosEventHandlerDispatcher, +) +from minos.networks.exceptions import ( + MinosNetworkException, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) + + +class TestEventDispatcher(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosEventHandlerDispatcher.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosEventHandlerDispatcher) + + async def test_if_queue_table_exists(self): + event_handler = MinosEventHandlerDispatcher.from_config(config=self.config) + await event_handler.setup() + + async with aiopg.connect(**self.events_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'event_queue';" + ) + ret = [] + async for row in cur: + ret.append(row) + + assert ret == [(1,)] + + async def test_get_event_handler(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) + m = MinosEventHandlerDispatcher.from_config(config=self.config) + + cls = m.get_event_handler(topic="TicketAdded") + result = await cls(topic="TicketAdded", event=event_instance) + + assert result == "request_added" + + async def test_non_implemented_action(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="NotExisting", model=model.classname, items=[]) + m = MinosEventHandlerDispatcher.from_config(config=self.config) + + with self.assertRaises(MinosNetworkException) as context: + cls = m.get_event_handler(topic=event_instance.topic) + await cls(topic=event_instance.topic, event=event_instance) + + self.assertTrue( + "topic NotExisting have no controller/action configured, please review th configuration file" + in str(context.exception) + ) + + async def test_none_config(self): + event_handler = MinosEventHandlerDispatcher.from_config(config=None) + + self.assertIsNone(event_handler) + + async def test_event_queue_checker(self): + event_handler = MinosEventHandlerDispatcher.from_config(config=self.config) + await event_handler.setup() + + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TicketAdded", model=model.classname, items=[]) + bin_data = event_instance.avro_bytes + Event.from_avro_bytes(bin_data) + + async with aiopg.connect(**self.events_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO event_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + (event_instance.topic, 0, bin_data, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + assert affected_rows == 1 + assert queue_id[0] > 0 + + # Must get the record, call on_reply function and delete the record from DB + await event_handler.queue_checker() + + async with aiopg.connect(**self.events_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM event_queue WHERE id=%d" % (queue_id)) + records = await cur.fetchone() + + assert records[0] == 0 + + async def test_event_queue_checker_wrong_event(self): + handler = MinosEventHandlerDispatcher.from_config(config=self.config) + await handler.setup() + bin_data = bytes(b"Test") + + async with aiopg.connect(**self.events_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute( + "INSERT INTO event_queue (topic, partition_id, binary_data, creation_date) VALUES (%s, %s, %s, %s) RETURNING id;", + ("TicketAdded", 0, bin_data, datetime.datetime.now(),), + ) + + queue_id = await cur.fetchone() + affected_rows = cur.rowcount + + assert affected_rows == 1 + assert queue_id[0] > 0 + + # Must get the record, call on_reply function and delete the record from DB + await handler.queue_checker() + + async with aiopg.connect(**self.events_queue_db) as connect: + async with connect.cursor() as cur: + await cur.execute("SELECT COUNT(*) FROM event_queue WHERE id=%d" % (queue_id)) + records = await cur.fetchone() + + assert records[0] == 1 diff --git a/tests/test_networks/test_handler/test_event/test_event_server.py b/tests/test_networks/test_handler/test_event/test_event_server.py new file mode 100644 index 00000000..4bd14e60 --- /dev/null +++ b/tests/test_networks/test_handler/test_event/test_event_server.py @@ -0,0 +1,72 @@ +from collections import ( + namedtuple, +) + +from minos.common import ( + Event, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosEventHandlerServer, +) +from tests.utils import ( + BASE_PATH, + NaiveAggregate, +) + + +class TestEventServer(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_from_config(self): + dispatcher = MinosEventHandlerServer.from_config(config=self.config) + self.assertIsInstance(dispatcher, MinosEventHandlerServer) + + async def test_none_config(self): + event_server = MinosEventHandlerServer.from_config(config=None) + + self.assertIsNone(event_server) + + async def test_queue_add(self): + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TestEventQueueAdd", model=model.classname, items=[]) + bin_data = event_instance.avro_bytes + Event.from_avro_bytes(bin_data) + + event_server = MinosEventHandlerServer.from_config(config=self.config) + await event_server.setup() + + affected_rows, id = await event_server.queue_add(topic=event_instance.topic, partition=0, binary=bin_data) + + assert affected_rows == 1 + assert id > 0 + + async def test_handle_message(self): + event_server = MinosEventHandlerServer.from_config(config=self.config) + await event_server.setup() + + model = NaiveAggregate(test_id=1, test=2, id=1, version=1) + event_instance = Event(topic="TicketAdded", model=model.classname, items=[model]) + bin_data = event_instance.avro_bytes + + Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) + + async def consumer(): + yield Mensaje(topic="TicketAdded", partition=0, value=bin_data) + + await event_server.handle_message(consumer()) + + async def test_handle_message_ko(self): + event_server = MinosEventHandlerServer.from_config(config=self.config) + await event_server.setup() + + bin_data = bytes(b"test") + + Mensaje = namedtuple("Mensaje", ["topic", "partition", "value"]) + + async def consumer(): + yield Mensaje(topic="TicketAdded", partition=0, value=bin_data) + + await event_server.handle_message(consumer()) diff --git a/tests/test_networks/test_handler/test_event/test_event_services.py b/tests/test_networks/test_handler/test_event/test_event_services.py new file mode 100644 index 00000000..54651f1e --- /dev/null +++ b/tests/test_networks/test_handler/test_event/test_event_services.py @@ -0,0 +1,52 @@ +import unittest +from unittest.mock import ( + MagicMock, +) + +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosEventPeriodicService, + MinosEventServerService, +) +from tests.utils import ( + BASE_PATH, +) + + +class TestMinosEventServices(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_start(self): + service = MinosEventServerService(loop=None, config=self.config) + + async def _fn(consumer): + self.assertEqual(service.consumer, consumer) + + mock = MagicMock(side_effect=_fn) + service.dispatcher.handle_message = mock + await service.start() + self.assertTrue(1, mock.call_count) + await service.stop() + + +class TestMinosQueueService(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + async def test_start(self): + service = MinosEventPeriodicService(interval=1, loop=None, config=self.config) + mock = MagicMock(side_effect=service.dispatcher.setup) + service.dispatcher.setup = mock + await service.start() + self.assertTrue(1, mock.call_count) + await service.stop() + + async def test_callback(self): + service = MinosEventPeriodicService(interval=1, loop=None, config=self.config) + await service.dispatcher.setup() + mock = MagicMock(side_effect=service.dispatcher.queue_checker) + service.dispatcher.queue_checker = mock + await service.callback() + self.assertEqual(1, mock.call_count) + await service.dispatcher.destroy() diff --git a/tests/test_networks/test_snapshots/__init__.py b/tests/test_networks/test_snapshots/__init__.py new file mode 100644 index 00000000..fed3716d --- /dev/null +++ b/tests/test_networks/test_snapshots/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" diff --git a/tests/test_networks/test_snapshots/test_dispatchers.py b/tests/test_networks/test_snapshots/test_dispatchers.py new file mode 100644 index 00000000..5d254ce1 --- /dev/null +++ b/tests/test_networks/test_snapshots/test_dispatchers.py @@ -0,0 +1,166 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +import unittest +from datetime import ( + datetime, +) +from unittest.mock import ( + MagicMock, + call, + patch, +) + +import aiopg +from minos.common import ( + MinosConfigException, + MinosRepositoryEntry, + PostgreSqlMinosRepository, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosSnapshotDispatcher, + MinosSnapshotEntry, +) +from tests.aggregate_classes import ( + Car, +) +from tests.utils import ( + BASE_PATH, +) + + +class TestMinosSnapshotDispatcher(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_type(self): + self.assertTrue(issubclass(MinosSnapshotDispatcher, object)) + + def test_from_config(self): + dispatcher = MinosSnapshotDispatcher.from_config(config=self.config) + self.assertEqual(self.config.snapshot.host, dispatcher.host) + self.assertEqual(self.config.snapshot.port, dispatcher.port) + self.assertEqual(self.config.snapshot.database, dispatcher.database) + self.assertEqual(self.config.snapshot.user, dispatcher.user) + self.assertEqual(self.config.snapshot.password, dispatcher.password) + + def test_from_config_raises(self): + with self.assertRaises(MinosConfigException): + MinosSnapshotDispatcher.from_config() + + async def test_setup_snapshot_table(self): + async with MinosSnapshotDispatcher.from_config(config=self.config): + async with aiopg.connect(**self.snapshot_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute( + "SELECT EXISTS (SELECT FROM pg_tables " + "WHERE schemaname = 'public' AND tablename = 'snapshot');" + ) + observed = (await cursor.fetchone())[0] + self.assertEqual(True, observed) + + async def test_setup_snapshot_aux_offset_table(self): + async with MinosSnapshotDispatcher.from_config(config=self.config): + async with aiopg.connect(**self.snapshot_db) as connection: + async with connection.cursor() as cursor: + await cursor.execute( + "SELECT EXISTS (SELECT FROM pg_tables WHERE " + "schemaname = 'public' AND tablename = 'snapshot_aux_offset');" + ) + observed = (await cursor.fetchone())[0] + self.assertEqual(True, observed) + + async def test_dispatch_select(self): + await self._populate() + with self.config: + async with MinosSnapshotDispatcher.from_config() as dispatcher: + await dispatcher.dispatch() + observed = [v async for v in dispatcher.select()] + + expected = [ + MinosSnapshotEntry.from_aggregate(Car(2, 2, 3, "blue")), + MinosSnapshotEntry.from_aggregate(Car(3, 1, 3, "blue")), + ] + self._assert_equal_snapshot_entries(expected, observed) + + async def test_dispatch_ignore_previous_version(self): + with self.config: + dispatcher = MinosSnapshotDispatcher.from_config() + await dispatcher.setup() + + car = Car(1, 1, 3, "blue") + # noinspection PyTypeChecker + aggregate_name: str = car.classname + + async def _fn(*args, **kwargs): + yield MinosRepositoryEntry(1, aggregate_name, 1, car.avro_bytes) + yield MinosRepositoryEntry(1, aggregate_name, 3, car.avro_bytes) + yield MinosRepositoryEntry(1, aggregate_name, 2, car.avro_bytes) + + with patch("minos.common.PostgreSqlMinosRepository.select", _fn): + await dispatcher.dispatch() + observed = [v async for v in dispatcher.select()] + + expected = [MinosSnapshotEntry(1, aggregate_name, 3, car.avro_bytes)] + self._assert_equal_snapshot_entries(expected, observed) + + def _assert_equal_snapshot_entries(self, expected: list[MinosSnapshotEntry], observed: list[MinosSnapshotEntry]): + self.assertEqual(len(expected), len(observed)) + for exp, obs in zip(expected, observed): + self.assertEqual(exp.aggregate, obs.aggregate) + self.assertIsInstance(obs.created_at, datetime) + self.assertIsInstance(obs.updated_at, datetime) + + async def test_dispatch_with_offset(self): + async with await self._populate() as repository: + async with MinosSnapshotDispatcher.from_config(config=self.config) as dispatcher: + mock = MagicMock(side_effect=dispatcher.repository.select) + dispatcher.repository.select = mock + + await dispatcher.dispatch() + self.assertEqual(1, mock.call_count) + self.assertEqual(call(id_ge=0), mock.call_args) + mock.reset_mock() + + # noinspection PyTypeChecker + await repository.insert(MinosRepositoryEntry(3, Car.classname, 1, Car(1, 1, 3, "blue").avro_bytes)) + + await dispatcher.dispatch() + self.assertEqual(1, mock.call_count) + self.assertEqual(call(id_ge=7), mock.call_args) + mock.reset_mock() + + await dispatcher.dispatch() + self.assertEqual(1, mock.call_count) + self.assertEqual(call(id_ge=8), mock.call_args) + mock.reset_mock() + + await dispatcher.dispatch() + self.assertEqual(1, mock.call_count) + self.assertEqual(call(id_ge=8), mock.call_args) + mock.reset_mock() + + async def _populate(self): + car = Car(1, 1, 3, "blue") + # noinspection PyTypeChecker + aggregate_name: str = car.classname + async with PostgreSqlMinosRepository.from_config(config=self.config) as repository: + await repository.insert(MinosRepositoryEntry(1, aggregate_name, 1, car.avro_bytes)) + await repository.update(MinosRepositoryEntry(1, aggregate_name, 2, car.avro_bytes)) + await repository.insert(MinosRepositoryEntry(2, aggregate_name, 1, car.avro_bytes)) + await repository.update(MinosRepositoryEntry(1, aggregate_name, 3, car.avro_bytes)) + await repository.delete(MinosRepositoryEntry(1, aggregate_name, 4)) + await repository.update(MinosRepositoryEntry(2, aggregate_name, 2, car.avro_bytes)) + await repository.insert(MinosRepositoryEntry(3, aggregate_name, 1, car.avro_bytes)) + return repository + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_networks/test_snapshots/test_entries.py b/tests/test_networks/test_snapshots/test_entries.py new file mode 100644 index 00000000..7c752783 --- /dev/null +++ b/tests/test_networks/test_snapshots/test_entries.py @@ -0,0 +1,83 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +import unittest +from datetime import ( + datetime, +) + +from minos.networks import ( + MinosSnapshotEntry, +) +from tests.aggregate_classes import ( + Car, +) + + +class TestMinosSnapshotEntry(unittest.TestCase): + def test_constructor(self): + entry = MinosSnapshotEntry(1234, "example.Car", 0, bytes("car", "utf-8")) + self.assertEqual(1234, entry.aggregate_id) + self.assertEqual("example.Car", entry.aggregate_name) + self.assertEqual(0, entry.version) + self.assertEqual(bytes("car", "utf-8"), entry.data) + self.assertEqual(None, entry.created_at) + self.assertEqual(None, entry.updated_at) + + def test_constructor_extended(self): + entry = MinosSnapshotEntry( + 1234, "example.Car", 0, bytes("car", "utf-8"), datetime(2020, 1, 10, 4, 23), datetime(2020, 1, 10, 4, 25) + ) + self.assertEqual(1234, entry.aggregate_id) + self.assertEqual("example.Car", entry.aggregate_name) + self.assertEqual(0, entry.version) + self.assertEqual(bytes("car", "utf-8"), entry.data) + self.assertEqual(datetime(2020, 1, 10, 4, 23), entry.created_at) + self.assertEqual(datetime(2020, 1, 10, 4, 25), entry.updated_at) + + def test_from_aggregate(self): + car = Car(1, 1, 3, "blue") + entry = MinosSnapshotEntry.from_aggregate(car) + self.assertEqual(car.id, entry.aggregate_id) + self.assertEqual(car.classname, entry.aggregate_name) + self.assertEqual(car.version, entry.version) + self.assertIsInstance(entry.data, bytes) + self.assertEqual(None, entry.created_at) + self.assertEqual(None, entry.updated_at) + + def test_equals(self): + a = MinosSnapshotEntry(1234, "example.Car", 0, bytes("car", "utf-8")) + b = MinosSnapshotEntry(1234, "example.Car", 0, bytes("car", "utf-8")) + self.assertEqual(a, b) + + def test_hash(self): + entry = MinosSnapshotEntry(1234, "example.Car", 0, bytes("car", "utf-8")) + self.assertIsInstance(hash(entry), int) + + def test_aggregate_cls(self): + car = Car(1, 1, 3, "blue") + entry = MinosSnapshotEntry.from_aggregate(car) + self.assertEqual(Car, entry.aggregate_cls) + + def test_aggregate(self): + car = Car(1, 1, 3, "blue") + entry = MinosSnapshotEntry.from_aggregate(car) + self.assertEqual(car, entry.aggregate) + + def test_repr(self): + entry = MinosSnapshotEntry( + 1234, "example.Car", 0, bytes("car", "utf-8"), datetime(2020, 1, 10, 4, 23), datetime(2020, 1, 10, 4, 25), + ) + expected = ( + "MinosSnapshotEntry(aggregate_id=1234, aggregate_name='example.Car', version=0, data=b'car', " + "created_at=datetime.datetime(2020, 1, 10, 4, 23), updated_at=datetime.datetime(2020, 1, 10, 4, 25))" + ) + self.assertEqual(expected, repr(entry)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_networks/test_snapshots/test_services.py b/tests/test_networks/test_snapshots/test_services.py new file mode 100644 index 00000000..2e599778 --- /dev/null +++ b/tests/test_networks/test_snapshots/test_services.py @@ -0,0 +1,72 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" + +import unittest +from unittest.mock import ( + MagicMock, +) + +from aiomisc.service.periodic import ( + PeriodicService, +) +from minos.common import ( + MinosConfigException, +) +from minos.common.testing import ( + PostgresAsyncTestCase, +) +from minos.networks import ( + MinosSnapshotDispatcher, + MinosSnapshotService, +) +from tests.utils import ( + BASE_PATH, +) + + +class TestMinosSnapshotService(PostgresAsyncTestCase): + CONFIG_FILE_PATH = BASE_PATH / "test_config.yml" + + def test_is_instance(self): + with self.config: + service = MinosSnapshotService(interval=0.1) + self.assertIsInstance(service, PeriodicService) + + def test_dispatcher_config_raises(self): + with self.assertRaises(MinosConfigException): + MinosSnapshotService(interval=0.1) + + def test_dispatcher_config(self): + service = MinosSnapshotService(interval=0.1, config=self.config) + dispatcher = service.dispatcher + self.assertIsInstance(dispatcher, MinosSnapshotDispatcher) + self.assertFalse(dispatcher.already_setup) + + def test_dispatcher_config_context(self): + with self.config: + service = MinosSnapshotService(interval=0.1) + self.assertIsInstance(service.dispatcher, MinosSnapshotDispatcher) + + async def test_start(self): + service = MinosSnapshotService(interval=0.1, loop=None, config=self.config) + service.dispatcher.setup = MagicMock(side_effect=service.dispatcher.setup) + await service.start() + self.assertTrue(1, service.dispatcher.setup.call_count) + await service.stop() + + async def test_callback(self): + service = MinosSnapshotService(interval=0.1, loop=None, config=self.config) + await service.dispatcher.setup() + service.dispatcher.dispatch = MagicMock(side_effect=service.dispatcher.dispatch) + await service.callback() + self.assertEqual(1, service.dispatcher.dispatch.call_count) + await service.dispatcher.destroy() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 00000000..60075770 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,22 @@ +""" +Copyright (C) 2021 Clariteia SL + +This file is part of minos framework. + +Minos framework can not be copied and/or distributed without the express permission of Clariteia SL. +""" +from pathlib import ( + Path, +) + +from minos.common import ( + Aggregate, +) + +BASE_PATH = Path(__file__).parent + + +class NaiveAggregate(Aggregate): + """Naive aggregate class to be used for testing purposes.""" + + test: int diff --git a/tests/test_config.yaml b/tests/wrong_test_config.yml similarity index 65% rename from tests/test_config.yaml rename to tests/wrong_test_config.yml index 01e79614..9792f28a 100644 --- a/tests/test_config.yaml +++ b/tests/wrong_test_config.yml @@ -9,9 +9,21 @@ rest: method: POST controller: minos.services.OrderService action: add_order +repository: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 +snapshot: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 events: broker: localhost - port: 9092 + port: 4092 database: path: ./tests/local_db.lmdb name: database_events_test @@ -22,9 +34,17 @@ events: - name: TicketDeleted controller: tests.services.CqrsTestService.CqrsService action: ticket_deleted + queue: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 + records: 10 + retry: 2 commands: broker: localhost - port: 9092 + port: 4092 items: - name: AddOrder controller: minos.services.OrderService @@ -38,3 +58,11 @@ commands: - name: GetOrder controller: minos.service.OrderService action: get_order + queue: + database: order_db + user: minos + password: min0s + host: localhost + port: 5432 + records: 10 + retry: 2