diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ac782f0..d834a69 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,47 +1,35 @@ -# This is a basic workflow to help you get started with Actions +name: CI -name: Test Suite - -# Controls when the action will run. on: - # Triggers the workflow on push or pull request events but only for the master branch push: - branches: [ master ] + branches: + - master pull_request: - branches: [ master ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: jobs: - test: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] - python-version: [ '3.6', '3.7', '3.8', '3.9', '3.10'] + build: + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Test dependencies - run: | - python -m pip install --upgrade pip - pip install pytest mock - - name: Test with pytest - run: | - pytest - -# # to be done: https://github.com/marketplace/actions/coveralls-python#coveragepy-configuration -# coveralls_finish: -# needs: test -# runs-on: ubuntu-latest -# steps: -# - name: Coveralls Finished -# uses: AndreMiras/coveralls-python-action@develop -# with: -# parallel-finished: true + + - name: Build broker + run: docker-compose build app + + - name: Install python packages + run: docker-compose run --rm app poetry install + + - name: Check your installed dependencies for security vulnerabilities + run: docker-compose run --rm app poetry check + + - name: Test + run: docker-compose run --rm app poetry run pytest + + - name: Flake8 + run: docker-compose run --rm app poetry run flake8 arend + + - name: MyPy + run: docker-compose run --rm app poetry run mypy arend --no-strict-optional --ignore-missing-imports + + - name: Remove Services + run: docker-compose down diff --git a/.gitignore b/.gitignore index 286c016..f0b5fbb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,53 +1,98 @@ -# From https://github.com/github/gitignore/blob/master/Python.gitignore +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class -# env vars -.env +# C extensions +*.so -# Pyc/pyo files. -*.py[co] - -# Packages -*.egg -*.egg-info -dist -build -eggs -parts -bin -var -sdist -develop-eggs +# Distribution / packaging +.Python +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ .installed.cfg -.venv -venv +*.egg +static_build_files/ + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec # Installer logs pip-log.txt +pip-delete-this-directory.txt + +# eventual secrets file used in dynaconf (these usually contain passwords which should not be stored in the version control system) +.secrets.* # Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache nosetests.xml coverage.xml -.coverage -htmlcov +*.cover +.hypothesis/ +.pytest_cache/ + +# Flask: +instance/ +.webassets-cache -#Translations -# *.mo -# ^^^ No, django needs the .mo files as it doesn't compile them on the fly. +# Jupyter Notebook +.ipynb_checkpoints -#Mr Developer -.mr.developer.cfg -src +# IPython +profile_default/ +ipython_config.py -# Ignore generated config files -etc/*.nginx.conf -etc/*.logrotate +# pyenv +.python-version + +# Environments +# files created with the virtual environment (these are dependent on the developer environment so they should not be stored in the version control system) +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ -# Ignore ide's -.idea/ -.vscode +# files related with IDE VSC or IntelliJ +.idea/* +.vscode/* +**/.password +### VisualStudioCode Patch ### +# Ignore all local history of files +.history -# ignore docs -docs/_* -notebooks -*.ipynb +**/__pycache__ +# db config file +odi_db_config.yml +config/db_config.yml +design/.DS_Store +.DS_Store +flake8.txt +.dockerignore +/source/ +/docs/_build/ +/.devcontainer/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 015e5d0..dfed934 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,22 +1,25 @@ default_language_version: python: python3 + repos: -- repo: https://github.com/pre-commit/mirrors-isort - rev: 'v4.3.21' - hooks: - - id: isort - exclude: 'settings' - repo: https://github.com/ambv/black - rev: stable + rev: 22.6.0 hooks: - id: black - exclude: 'setup.py' args: [--line-length=79] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.3.0 hooks: - id: check-merge-conflict - id: check-yaml + - id: check-json + - id: check-toml + - id: end-of-file-fixer + - id: mixed-line-ending - id: flake8 - # NB The "exclude" setting in setup.cfg is ignored by pre-commit -# exclude: '|' +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.971 + hooks: + - id: mypy + files: ^arend/ + args: [--no-strict-optional, --ignore-missing-imports] diff --git a/.readthedocs.yml b/.readthedocs.yml index f367dfc..53e8212 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,23 +5,20 @@ # Required version: 2 +build: + os: ubuntu-20.04 + tools: + python: '3.10' + jobs: + post_install: + - pip install poetry==1.2.0b1 + - poetry config virtualenvs.create false + - poetry install --with doc + # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/conf.py - fail_on_warning: false - -# Build documentation with MkDocs -#mkdocs: -# configuration: mkdocs.yml # Optionally build your docs in additional formats such as PDF and ePub formats: - pdf - -python: - version: 3.7 - # use system_packages for numpy; the rest is mocked by autodoc_mock_imports - system_packages: true - install: - - method: setuptools - path: . # setup.py includes a hack that emulates --no-deps for RTD diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d556d54 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +Changelog +=================================== + +0.0.1 (2022-XX-XX) +------------------- + +- Initial project structure. + +- Test suite. diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index 7c30b52..0000000 --- a/CHANGES.rst +++ /dev/null @@ -1,10 +0,0 @@ -Changelog -======================================================== - - -0.0.1 (2021-01-20) -------------------- - -- Initial project structure. - -- Test suite. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a27c535 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.10.2 + +LABEL maintainer="Jose-Maria Vazquez-Jimenez" +RUN apt-get update + +# Set timezone, not really necessary... +ENV TZ=Europe/Amsterdam +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# install basic python packages: setuptools, pip3, virtualenv +RUN pip3 install --upgrade setuptools && pip3 install pip virtualenv + +# Let's go with poetry... +ENV POETRY_VERSION=1.1.13 VENV_PATH="/code/.venv" POETRY_HOME="/opt/poetry" +RUN curl -sSL https://install.python-poetry.org | python3 - --version $POETRY_VERSION +ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH" + +VOLUME /code +WORKDIR /code diff --git a/LICENSE b/LICENSE index d9ac644..ae2ce06 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2021 +Copyright (c) 2021, Python Retry. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -14,8 +14,8 @@ copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +FITNESS FOR A PARTICULAR PURPOSE AND NON INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 38817e0..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -# Include docs in the root. -include *.rst -include LICENSE diff --git a/README.md b/README.md new file mode 100644 index 0000000..6f8a313 --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +Arend +======== + +A simple producer-consumer library for the Beanstalkd queue. + +Installation +-------------- +Hit the command: +```shell +pip install arend +``` + +Basic Usage +-------------- + +In your code: + ```python +from arend import arend_task +from arend.backends.mongo import MongoSettings +from arend.brokers import BeanstalkdSettings +from arend.settings import ArendSettings +from arend.worker import consumer + +settings = ArendSettings( + beanstalkd=BeanstalkdSettings(host="beanstalkd", port=11300), + backend=MongoSettings( + mongo_connection="mongodb://user:pass@mongo:27017", + mongo_db="db", + mongo_collection="Tasks" + ), +) + +@arend_task(queue="my_queue", settings=settings) +def double(num: int): + return 2 * num + +double(2) # returns 4 +task = double.apply_async(args=(4,)) # It is sent to the queue + +consumer(queue="my_queue", settings=settings) # consume tasks from the queue + +Task = settings.get_backend() # you can check your backend for the result +task = Task.get(uuid=task.uuid) +assert task.result == 4 +``` + +Backends +------------------- +The available backends to store logs are **Mongo** and **Redis**. +Please read the [docs](https://arend.readthedocs.io/en/latest/) +for further information. + +Setting your backend with environment variables +-------------------------------------------------- +You can set your backend by defining env vars. +The `AREND__` prefix indicates that it belongs to the Arend. +```shell +# Redis +AREND__REDIS_HOST='redis' +AREND__REDIS_DB='1' +AREND__REDIS_PASSWORD='pass' +... + +# Mongo +AREND__MONGO_CONNECTION='mongodb://user:pass@mongo:27017' +AREND__MONGO_DB='db' +AREND__MONGO_COLLECTION='logs' +... +``` + +In your code: + ```python +from arend import arend_task +from arend.worker import consumer + + +@arend_task(queue="my_queue") +def double(num: int): + return 2 * num + +double.apply_async(args=(4,)) # It is sent to the queue + +consumer(queue="my_queue") +``` + +Documentation +-------------- + +Please visit this [link](https://arend.readthedocs.io/en/latest/) for documentation. diff --git a/README_USERS.md b/README_USERS.md new file mode 100644 index 0000000..5ce6d92 --- /dev/null +++ b/README_USERS.md @@ -0,0 +1,46 @@ + arend +================== + +Clone the project and move inside: +```shell +git clone https://github.com/pyprogrammerblog/arend.git +cd arend +``` + +Install the virtualenv on the root project folder: +```shell +docker-compose run --rm --no-deps app poetry install +``` + +Check your installed dependencies for security vulnerabilities +```shell +docker-compose run --rm app poetry check +``` + +Run the tests: +```shell +docker-compose run --rm app poetry run pytest +``` + +Shut down all services +```shell +docker-compose down +``` + +### Jupyter Notebook + +Hit the command + +```shell +docker-compose run --rm -p 8888:8888 app poetry shell +``` + +Then inside the docker: + +```shell +docker notebook --ip 0.0.0.0 --no-browser --allow-root +``` + +That is all! + +Happy Coding! diff --git a/arend/__init__.py b/arend/__init__.py index e69de29..0299fc9 100644 --- a/arend/__init__.py +++ b/arend/__init__.py @@ -0,0 +1,6 @@ +from arend.arend import ArendTask, arend_task +from arend.worker.consumer import consumer +from arend.worker.pool_consumers import pool_consumers + + +__all__ = ["ArendTask", "arend_task", "consumer", "pool_consumers"] diff --git a/arend/arend.py b/arend/arend.py new file mode 100644 index 0000000..1e8b564 --- /dev/null +++ b/arend/arend.py @@ -0,0 +1,165 @@ +from arend.settings import ArendSettings, Settings +from arend.backends import MongoTask, RedisTask +from typing import Callable, Union +from pydantic import BaseModel +from datetime import timedelta +from arend.settings.tasks import ( + TASK_DELAY, + TASK_PRIORITY, + TASK_RETRY_BACKOFF_FACTOR, + TASK_MAX_RETRIES, +) + +import functools +import logging + +__all__ = ["arend_task", "ArendTask"] + + +logger = logging.getLogger(__name__) + + +class ArendTask(BaseModel): + """ + Defines the ArendTask. This class handles writing + to the Backend and the Queue the Arend tasks + """ + + queue: str + name: str + location: str + processor: Callable + priority: int = TASK_PRIORITY + max_retries: int = TASK_MAX_RETRIES + delay: Union[timedelta, int] = TASK_DELAY + retry_backoff_factor: int = TASK_RETRY_BACKOFF_FACTOR + settings: ArendSettings = None + + def __call__(self, *args, **kwargs): + return self.run(*args, **kwargs) + + def __repr__(self): + return f"<{self.__class__.__name__} at {self.__class__.__module__}>" + + def run(self, *args, **kwargs): + """ + Run the task immediately. + """ + return self.processor(*args, **kwargs) + + def apply_async( + self, + queue: str = None, + args: tuple = None, + kwargs: dict = None, + priority: int = TASK_PRIORITY, + max_retries: int = TASK_MAX_RETRIES, + delay: Union[timedelta, int] = TASK_DELAY, + retry_backoff_factor: int = TASK_RETRY_BACKOFF_FACTOR, + settings: ArendSettings = settings, + ) -> Union[MongoTask, RedisTask]: + """ + Run task asynchronously. + """ + settings = settings or Settings().arend + Task = settings.get_backend() + + # create and save a task in your backend + task = Task( + name=self.name, + location=self.location, + queue=self.queue or queue, + args=args or (), + kwargs=kwargs or {}, + priority=priority + or self.priority + or settings.task_priority + or TASK_PRIORITY, + delay=delay or self.delay or settings.task_delay or TASK_DELAY, + max_retries=max_retries + or self.max_retries + or settings.task_max_retries + or TASK_MAX_RETRIES, + retry_backoff_factor=retry_backoff_factor + or self.retry_backoff_factor + or settings.task_retry_backoff_factor + or TASK_RETRY_BACKOFF_FACTOR, + ).save() # default to SCHEDULED + + task.send_to_queue() # put into the queue and set task as PENDING + + return task # return a PENDING task + + +def arend_task( + queue: str = None, + priority: int = TASK_PRIORITY, + delay: Union[timedelta, int] = TASK_DELAY, + max_retries: int = TASK_MAX_RETRIES, + retry_backoff_factor: int = TASK_RETRY_BACKOFF_FACTOR, + settings: ArendSettings = None, +): + """ + Register functions as arend task + + Usage: + >>> from arend import arend_task + >>> from arend.backends.mongo import MongoSettings + >>> from arend.brokers import BeanstalkdSettings + >>> from arend.settings import ArendSettings + >>> from arend.worker import consumer + >>> + >>> settings = ArendSettings( + >>> beanstalkd=BeanstalkdSettings(host="beanstalkd", port=11300), + >>> backend=MongoSettings( + >>> mongo_connection="mongodb://user:pass@mongo:27017", + >>> mongo_db="db", + >>> mongo_collection="Tasks" + >>> ), + >>> ) + >>> + >>> @arend_task(queue="my_queue", settings=settings) + >>> def double(num: int): + >>> return 2 * num + >>> + >>> double(2) # returns 4 + >>> task = double.apply_async(args=(4,)) # It is sent to the queue + >>> + >>> consumer(queue="my_queue", settings=settings) # consume tasks + >>> + >>> Task = settings.get_backend() # you can check your result backend + >>> task = Task.get(uuid=task.uuid) + >>> assert task.result == 4 + + By defining the backend in the environment variables: + >>> from arend import arend_task + >>> from arend.worker import consumer + >>> + >>> + >>> @arend_task(queue="my_queue") + >>> def double(num: int): + >>> return 2 * num + >>> + >>> double.apply_async(args=(4,)) # It is sent to the queue + >>> + >>> consumer(queue="my_queue") # consume tasks from the queue + """ + + def decorator(func): + @functools.wraps(func) + def wrapper_register(): + return ArendTask( + name=func.__name__, + location=func.__module__, + processor=func, + queue=queue, + priority=priority, + delay=delay, + max_retries=max_retries, + retry_backoff_factor=retry_backoff_factor, + settings=settings, + ) + + return wrapper_register() + + return decorator diff --git a/arend/backends/__init__.py b/arend/backends/__init__.py index 567f990..d3d50c1 100644 --- a/arend/backends/__init__.py +++ b/arend/backends/__init__.py @@ -1,18 +1,9 @@ -from .mongo import MongoBackend -from .redis import RedisBackend -from .sql import SqlBackend -from arend.settings import settings -from functools import lru_cache - - -@lru_cache -def get_queue_backend(): - backends = { - "redis": RedisBackend, - "mongo": MongoBackend, - "sql": SqlBackend, - } - return backends.get(settings.broker) - - -QueueBroker = get_queue_backend() +from arend.backends.redis import RedisSettings, RedisTask +from arend.backends.mongo import MongoSettings, MongoTask + +__all__ = [ + "MongoSettings", + "RedisSettings", + "RedisTask", + "MongoTask", +] diff --git a/arend/backends/base.py b/arend/backends/base.py index d3cb816..443650e 100644 --- a/arend/backends/base.py +++ b/arend/backends/base.py @@ -1,25 +1,148 @@ -# from arend.settings import settings -# from pymongo import MongoClient -# from pymongo.collection import Collection +from datetime import datetime +from pydantic import BaseModel +from pydantic import Field +from typing import List +from typing import Optional +from uuid import uuid4, UUID +from typing import TYPE_CHECKING +from arend.brokers.beanstalkd import BeanstalkdConnection +from arend.settings.tasks import ( + TASK_DELAY, + TASK_PRIORITY, + TASK_RETRY_BACKOFF_FACTOR, + TASK_MAX_RETRIES, +) +import importlib import logging +import traceback + + +if TYPE_CHECKING: + from arend.arend import ArendTask + from arend.settings.arend import ArendSettings logger = logging.getLogger(__name__) -class TasksBackend: - def __init__(self): - pass +class Status: + SCHEDULED: str = "SCHEDULED" + PENDING: str = "PENDING" + STARTED: str = "STARTED" + RETRY: str = "RETRY" + FINISHED: str = "FINISHED" + FAIL: str = "FAIL" + REVOKED: str = "REVOKED" + + +class BaseTask(BaseModel): + """ + Base Task + """ + + uuid: UUID = Field(default_factory=uuid4, description="UUID") + name: str = Field(..., description="Full path task name") + description: str = Field(default=None, description="Description") + location: str = Field(default="tasks", description="Module location") + + status: str = Field(default=Status.SCHEDULED, description="Status") + result: str = Field(default=None, description="Task result") + detail: Optional[str] = Field(description="Task details") + start_time: Optional[datetime] = Field(default=None) + end_time: Optional[datetime] = Field(default=None) + + args: tuple = Field(default_factory=tuple, description="Task args") + kwargs: dict = Field(default_factory=dict, description="Task kwargs") + + queue: str = Field(..., description="Queue name") + delay: int = Field(default=TASK_DELAY, description="Queue delay") + priority: int = Field(default=TASK_PRIORITY, description="Queue priority") + count_retries: int = Field(default=0, description="Number of retries") + max_retries: int = Field(default=TASK_MAX_RETRIES) + retry_backoff_factor: int = Field(default=TASK_RETRY_BACKOFF_FACTOR) + + created: datetime = Field(default_factory=datetime.utcnow) + updated: datetime = Field(default=None) + + class Meta: + settings: "ArendSettings" + + def save(self): + return NotImplementedError + + def delete(self): + return NotImplementedError + + @classmethod + def get(cls, uuid: UUID): + return NotImplementedError + + def send_to_queue(self): + with BeanstalkdConnection( + queue=self.queue, settings=self.Meta.settings.beanstalkd + ) as conn: + conn.put( + body=str(self.uuid), + priority=self.priority, + delay=self.delay + + self.count_retries * self.retry_backoff_factor, + ) + self.status = Status.PENDING + self.save() def __enter__(self): + if self.status == [Status.REVOKED, Status.FAIL, Status.FINISHED]: + return + + self.start_time = datetime.utcnow() + self.status = Status.STARTED + self.save() return self def __exit__(self, exc_type, exc_val, exc_tb): - self.close() + if exc_type: + last_trace = "".join(traceback.format_tb(exc_tb)).strip() + self.detail = f"Failure: {last_trace}\n" + if self.count_retries < self.max_retries: + self.count_retries += 1 + self.status = Status.RETRY + self.send_to_queue() # put it in the tube again + else: + self.status = Status.FAIL + else: + self.status = Status.FINISHED + + self.end_time = datetime.utcnow() + self.save() + return True + + def __call__(self): + return self.run() + + def run(self): + """ + Run task. + - Get signature + - Run signature with args and kwargs + - Update result + """ + with self: + task = self.get_task_signature() + result = task.run(*self.args, **self.kwargs) + self.result = result + + def get_task_signature(self) -> "ArendTask": + """ + Get task signature. + + Returns: ArendTask. + """ + module = importlib.import_module(self.location) + task = getattr(module, self.name) + return task - def close(self): - pass - def get_(self): - pass +class BaseTasks(BaseModel): + tasks: List[BaseTask] = Field(default_factory=list) + count: int = Field(default=0) diff --git a/arend/backends/mongo.py b/arend/backends/mongo.py index 98e37ef..29a3775 100644 --- a/arend/backends/mongo.py +++ b/arend/backends/mongo.py @@ -1,27 +1,140 @@ -from arend.settings import settings -from pymongo import MongoClient +import logging +from datetime import datetime +from uuid import UUID +from typing import Type, List, Union, TYPE_CHECKING +from pydantic import BaseModel, Field from pymongo.collection import Collection +from contextlib import contextmanager +from arend.backends.base import BaseTask +from pymongo.mongo_client import MongoClient -import logging + +if TYPE_CHECKING: + from arend.settings import ArendSettings + + +__all__ = ["MongoTask", "MongoTasks", "MongoSettings"] logger = logging.getLogger(__name__) -class MongoBackend: - def __init__(self): - self.db: MongoClient = MongoClient(settings.mongodb_string) - collection = settings.mongodb_notifier_task_results - self.tasks_collection: Collection = self.db[collection] +class MongoTask(BaseTask): + """ + MongoTask class. Defines the Task Model for Mongo Backend + + Usage: + + >>> from arend.backends.mongo import MongoSettings + >>> + >>> settings = MongoSettings( + >>> mongo_connection="mongodb://user:pass@mongo:27017", + >>> mongo_db="db", + >>> mongo_collection="Tasks" + >>> ) + >>> MongoTask = MongoSettings.get_backend() # type: Type[MongoTask] + >>> task = MongoTask(name="My task", ...) + >>> task.save() + >>> + >>> assert task.dict() == {"name": "My task", ...} + >>> assert task.json() == '{"name": "My task", ...}' + >>> + >>> task = MongoTask.get(uuid=UUID("")) + >>> assert task.description == "A cool task" + >>> + >>> assert task.delete() == 1 + >>> assert task.delete() == 0 + """ + + class Meta: + settings: "ArendSettings" + + @classmethod + @contextmanager + def mongo_collection(cls): + """ + Yield a MongoTask Collection + """ + mongo_conn = cls.Meta.settings.backend.mongo_connection + mongo_db = cls.Meta.settings.backend.mongo_db + mongo_collection = cls.Meta.settings.backend.mongo_collection + + with MongoClient( + mongo_conn, UuidRepresentation="standard" + ) as client: # type: MongoClient + db = client.get_database(mongo_db) + collection = db.get_collection(mongo_collection) + yield collection + + @classmethod + def get(cls, uuid: UUID) -> Union["MongoTask", None]: + """ + Get object from DataBase - def __enter__(self): + Usage: + + >>> task = MongoTask.get(uuid=UUID("")) + >>> assert task.uuid == UUID("") + """ + with cls.mongo_collection() as collection: # type: Collection + if task := collection.find_one({"uuid": uuid}): + return cls(**task) + return None + + def save(self) -> "MongoTask": + """ + Updates/Creates object in DataBase + + Usage: + + >>> task = MongoTask(name="My Task") + >>> task.save() + >>> task.description = "A new description" + >>> task.save() + """ + self.updated = datetime.utcnow() + with self.mongo_collection() as collection: # type: Collection + collection.update_one( + filter={"uuid": self.uuid}, + update={"$set": self.dict()}, + upsert=True, + ) return self - def __exit__(self, exc_type, exc_val, exc_tb): - self.db.close() + def delete(self) -> int: + """ + Deletes object in DataBase + + Usage: + + >>> assert task.delete() == 1 # count deleted 1 + >>> assert task.delete() == 0 # count deleted 0 + """ + with self.mongo_collection() as collection: # type: Collection + deleted = collection.delete_one({"uuid": self.uuid}) + return deleted.deleted_count + + +class MongoTasks(BaseModel): + """ + Defines the MongoTasks collection + """ + + tasks: List[MongoTask] = Field(default_factory=list, description="Tasks") + count: int = Field(default=0, description="Count") + + +class MongoSettings(BaseModel): + """ + Mongo Settings. Defines settings for Mongo Backend + """ - def find_one(self): - pass + mongo_connection: str = Field(..., description="Connection string") + mongo_db: str = Field(..., description="Database name") + mongo_collection: str = Field(..., description="Collection name") - def update_one(self): - pass + def get_backend(self) -> Type[MongoTask]: + """ + Returns a MongoTask class with settings already set + """ + return MongoTask diff --git a/arend/backends/redis.py b/arend/backends/redis.py index c0d8e55..a787579 100644 --- a/arend/backends/redis.py +++ b/arend/backends/redis.py @@ -1,27 +1,134 @@ -from arend.settings import settings -from pymongo import MongoClient -from pymongo.collection import Collection - import logging +from redis import Redis # type: ignore +from typing import Dict, Type, List, Union, TYPE_CHECKING +from datetime import datetime +from uuid import UUID +from pydantic import BaseModel, Field +from arend.backends.base import BaseTask +from contextlib import contextmanager + +if TYPE_CHECKING: + from arend.settings import ArendSettings + + +__all__ = ["RedisTask", "RedisTasks", "RedisSettings"] logger = logging.getLogger(__name__) -class RedisBackend: - def __init__(self): - self.db: MongoClient = MongoClient(settings.mongodb_string) - collection = settings.mongodb_notifier_task_results - self.tasks_collection: Collection = self.db[collection] +class RedisTask(BaseTask): + """ + RedisLog class. Defines the Log for Redis Backend + + Usage: + + >>> from arend.backends.redis import RedisSettings + >>> + >>> settings = RedisSettings( + >>> redis_host="redis", + >>> redis_port=6379, + >>> redis_db="logs", + >>> redis_password="pass" + >>> ) + >>> RedisTask = RedisSettings.get_backend() # type: Type[RedisLog] + >>> task = RedisTask(name="My task", description="A cool task") + >>> task.save() + >>> + >>> assert task.dict() == {"name": "My task", ...} + >>> assert task.json() == '{"name": "My task", ...}' + >>> + >>> task = RedisTask.get(uuid=UUID("")) + >>> assert task.description == "A cool task" + >>> + >>> assert task.delete() == 1 + """ + + class Meta: + settings: "ArendSettings" + + @classmethod + @contextmanager + def redis_connection(cls): + """ + Yield a Redis connection to a specific database + """ + host = cls.Meta.settings.backend.redis_host + port = cls.Meta.settings.backend.redis_port + db = cls.Meta.settings.backend.redis_db + password = cls.Meta.settings.backend.redis_password + + with Redis(host=host, port=port, db=db, password=password) as r: + yield r + + @classmethod + def get(cls, uuid: UUID) -> Union["RedisTask", None]: + """ + Get object from DataBase + + Usage: - def __enter__(self): + >>> log = RedisTask.get(uuid=UUID("")) + >>> assert log.uuid == UUID("") + """ + with cls.redis_connection() as r: # type: Redis + if task := r.get(str(uuid)): + return cls.parse_raw(task) + return None + + def save(self) -> "RedisTask": + """ + Updates/Creates object in DataBase + + Usage: + + >>> task = RedisTask(name="My Task") + >>> task.save() + >>> task.description = "A new description" + >>> task.save() + """ + self.updated = datetime.utcnow() + with self.redis_connection() as r: # type: Redis + r.set(str(self.uuid), self.json()) return self - def __exit__(self, exc_type, exc_val, exc_tb): - self.db.close() + def delete(self) -> int: + """ + Deletes object in DataBase + + Usage: + + >>> assert task.delete() == 1 # count deleted 1 + >>> assert task.delete() == 0 # count deleted 0 + """ + with self.redis_connection() as r: # type: Redis + return r.delete(str(self.uuid)) + + +class RedisTasks(BaseModel): + """ + Defines the RedisTasks collection + """ + + tasks: List[RedisTask] = Field(default_factory=list, description="Logs") + count: int = Field(default=0, description="Count") + + +class RedisSettings(BaseModel): + """ + Defines settings for Redis Backend + """ - def find_one(self): - pass + redis_host: str = Field(..., description="Redis Host") + redis_port: int = Field(default=6379, description="Redis Host") + redis_db: int = Field(default=1, description="Redis DB") + redis_password: str = Field(..., description="Redis Password") + redis_extras: Dict = Field( + default_factory=dict, description="Redis extras" + ) - def update_one(self): - pass + def get_backend(self) -> Type[RedisTask]: + """ + Returns a RedisTask class with settings already set + """ + return RedisTask diff --git a/arend/backends/sql.py b/arend/backends/sql.py deleted file mode 100644 index c6e1241..0000000 --- a/arend/backends/sql.py +++ /dev/null @@ -1,27 +0,0 @@ -from arend.settings import settings -from pymongo import MongoClient -from pymongo.collection import Collection - -import logging - - -logger = logging.getLogger(__name__) - - -class SqlBackend: - def __init__(self): - self.db: MongoClient = MongoClient(settings.mongodb_string) - collection = settings.mongodb_notifier_task_results - self.tasks_collection: Collection = self.db[collection] - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.db.close() - - def find_one(self): - pass - - def update_one(self): - pass diff --git a/arend/brokers/__init__.py b/arend/brokers/__init__.py index fefcca8..9a63ca0 100644 --- a/arend/brokers/__init__.py +++ b/arend/brokers/__init__.py @@ -1,18 +1,7 @@ -from .beanstalkd import BeanstalkdBroker -from .redis import RedisBroker -from .sqs import SQSBroker -from arend.settings import settings -from functools import lru_cache +from arend.brokers.beanstalkd import ( + BeanstalkdConnection, + BeanstalkdSettings, +) -@lru_cache -def get_queue_broker(): - brokers = { - "beanstalk": BeanstalkdBroker, - "redis": RedisBroker, - "sqs": SQSBroker, - } - return brokers.get(settings.broker) - - -QueueBroker = get_queue_broker() +__all__ = ["BeanstalkdConnection", "BeanstalkdSettings"] diff --git a/arend/brokers/base.py b/arend/brokers/base.py deleted file mode 100644 index 30fb0e2..0000000 --- a/arend/brokers/base.py +++ /dev/null @@ -1,23 +0,0 @@ -import logging -import pystalkd -import uuid - - -logger = logging.getLogger(__name__) - - -class BaseBroker: - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - raise NotImplementedError - - def add_to_queue(self, task_uuid: uuid.UUID): - raise NotImplementedError - - def reserve(self, timeout: int = None) -> pystalkd.Job: - raise NotImplementedError - - def delete(self, job): - raise NotImplementedError diff --git a/arend/brokers/beanstalkd.py b/arend/brokers/beanstalkd.py index 6b79de0..462f7e3 100644 --- a/arend/brokers/beanstalkd.py +++ b/arend/brokers/beanstalkd.py @@ -1,36 +1,55 @@ -from arend.brokers.base import BaseBroker -from arend.settings import settings from pystalkd.Beanstalkd import Connection +from pydantic import BaseModel +from pydantic import Field import logging -import pystalkd -import uuid + +__all__ = ["BeanstalkdConnection", "BeanstalkdSettings"] logger = logging.getLogger(__name__) -class BeanstalkdBroker(BaseBroker): - def __init__(self, queue_name: str): - self.queue_name = queue_name +class BeanstalkdSettings(BaseModel): + """ + Defines settings for the Beanstalkd Queue + """ + + host: str = Field(description="Beanstalkd Host") + port: int = Field(description="Beanstalkd Port") + + +class BeanstalkdConnection: + """ + Beanstalkd Connection. + + Defines a context manager to open and close connection + when interacting with our queue. + + Usage: + >>> from arend.brokers import BeanstalkdConnection, BeanstalkdSettings + >>> from arend.backends.mongo import MongoSettings + >>> + >>> settings = BeanstalkdSettings(host="beanstalkd", port=11300) + >>> with BeanstalkdConnection( + >>> queue="my_queue", + >>> settings=settings + >>> ) as conn: + >>> conn.put(body="my message") + >>> ... + >>> + """ + + def __init__(self, queue: str, settings: BeanstalkdSettings): + self.settings = settings self.connection = Connection( - host=settings.beanstalkd_host, port=settings.beanstalkd_port + host=self.settings.host, port=self.settings.port ) - self.connection.watch(name=queue_name) - self.connection.use(name=queue_name) + self.connection.watch(queue) + self.connection.use(queue) - def __enter__(self): - return self + def __enter__(self: "BeanstalkdConnection") -> Connection: + return self.connection def __exit__(self, exc_type, exc_val, exc_tb): self.connection.close() - - def add_to_queue(self, task_uuid: uuid.UUID): - self.connection.put(body=str(task_uuid)) - - def reserve(self, timeout: int = None) -> pystalkd.Job: - job = self.connection.reserve(timeout=timeout) - return job - - def delete(self, job: pystalkd.Job): - job.delete() diff --git a/arend/brokers/redis.py b/arend/brokers/redis.py deleted file mode 100644 index eb78037..0000000 --- a/arend/brokers/redis.py +++ /dev/null @@ -1,28 +0,0 @@ -from arend.brokers.base import BaseBroker - -import logging -import pystalkd -import uuid - - -logger = logging.getLogger(__name__) - - -class RedisBroker(BaseBroker): - def __init__(self, queue_name: str): - self.queue_name = queue_name - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def add_to_queue(self, task_uuid: uuid.UUID): - pass - - def reserve(self, timeout: int = None): - pass - - def delete(self, job: pystalkd.Job): - pass diff --git a/arend/brokers/sqs.py b/arend/brokers/sqs.py deleted file mode 100644 index 03c0c96..0000000 --- a/arend/brokers/sqs.py +++ /dev/null @@ -1,28 +0,0 @@ -from arend.brokers.base import BaseBroker - -import logging -import pystalkd -import uuid - - -logger = logging.getLogger(__name__) - - -class SQSBroker(BaseBroker): - def __init__(self, queue_name: str): - self.queue_name = queue_name - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def add_to_queue(self, task_uuid: uuid.UUID): - pass - - def reserve(self, timeout: int = None): - pass - - def delete(self, job: pystalkd.Job): - pass diff --git a/arend/settings/__init__.py b/arend/settings/__init__.py index 9b70b7c..c83089f 100644 --- a/arend/settings/__init__.py +++ b/arend/settings/__init__.py @@ -1,10 +1,3 @@ -from functools import lru_cache -from arend.settings.settings import Settings +from arend.settings.arend import ArendSettings, Settings, BeanstalkdSettings - -@lru_cache -def get_settings(): - return Settings() - - -settings = get_settings() +__all__ = ["ArendSettings", "BeanstalkdSettings", "Settings"] diff --git a/arend/settings/arend.py b/arend/settings/arend.py new file mode 100644 index 0000000..6e476be --- /dev/null +++ b/arend/settings/arend.py @@ -0,0 +1,77 @@ +from pydantic import BaseSettings, BaseModel +from typing import Union, Type +from arend.brokers.beanstalkd import BeanstalkdSettings +from arend.backends.redis import RedisSettings, RedisTask +from arend.backends.mongo import MongoSettings, MongoTask +from arend.settings.tasks import ( + TASK_DELAY, + TASK_PRIORITY, + TASK_RETRY_BACKOFF_FACTOR, + TASK_MAX_RETRIES, +) + +import logging + + +logger = logging.getLogger(__name__) + + +__all__ = [ + "Settings", + "ArendSettings", + "MongoSettings", + "RedisSettings", + "RedisTask", + "MongoTask", +] + + +class ArendSettings(BaseModel): + """ + Defines settings for the Arend + + Usage: + >>> from arend import arend_task + >>> from arend.backends.mongo import MongoSettings + >>> from arend.brokers import BeanstalkdSettings + >>> from arend.settings import ArendSettings + >>> from arend.worker import consumer + >>> + >>> settings = ArendSettings( + >>> beanstalkd=BeanstalkdSettings(host="beanstalkd", port=11300), + >>> backend=MongoSettings( + >>> mongo_connection="mongodb://user:pass@mongo:27017", + >>> mongo_db="db", + >>> mongo_collection="Tasks" + >>> ), + >>> task_max_retries = 3 + >>> task_retry_backoff_factor = 1 + >>> task_priority = 0 + >>> task_delay = 1 + >>> ) + """ + + beanstalkd: BeanstalkdSettings + backend: Union[MongoSettings, RedisSettings] + task_max_retries: int = TASK_MAX_RETRIES + task_retry_backoff_factor: int = TASK_RETRY_BACKOFF_FACTOR + task_priority: int = TASK_PRIORITY + task_delay: int = TASK_DELAY + + def get_backend(self) -> Type[Union[MongoTask, RedisTask]]: + """ + Return a Task Backend with configuration already set + """ + Task = self.backend.get_backend() + Task.Meta.settings = self + return Task + + +class Settings(BaseSettings): + + arend: ArendSettings + + class Config: + env_file = ".env" + env_file_encoding = "utf-8" + env_nested_delimiter = "__" diff --git a/arend/settings/settings.py b/arend/settings/settings.py deleted file mode 100644 index 4a2b047..0000000 --- a/arend/settings/settings.py +++ /dev/null @@ -1,37 +0,0 @@ -from typing import Literal - -from pydantic import ( - BaseModel, - BaseSettings, - RedisDsn, - PostgresDsn, - AmqpDsn, -) - - -class Settings(BaseSettings): - - # brokers - broker: Literal["redis", "beanstalk", "sqs"] - broker_uri: str - - # backends - backend: Literal["redis", "postgres", "ampqn", "beanstalk", "sqs"] - backend_uri: str - - # general settings - max_retries: int = 10 - backoff_factor: int = 1 - connect_timeout: int = 10 # seconds - priority: int = None - delay: int = None - delay_factor: int = 10 - sleep_time_consumer: int = 1 - - # redis settings - redis_socket_timeout: int = 2 * 60 - redis_socket_connect_timeout: int = 2 * 60 - - # mongo backend settings - mongodb_max_pool_size: int = 10 - mongodb_min_pool_size: int = 0 diff --git a/arend/settings/status.py b/arend/settings/status.py deleted file mode 100644 index 228a1fc..0000000 --- a/arend/settings/status.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Union -import logging - -logger = logging.getLogger(__name__) - - -SCHEDULED: str = "SCHEDULED" -PENDING: str = "PENDING" -STARTED: str = "STARTED" -RETRY: str = "RETRY" -FINISHED: str = "FINISHED" -FAIL: str = "FAIL" -REVOKED: str = "REVOKED" - -STATUSES: Union[ - PENDING, STARTED, RETRY, FINISHED, FAIL, REVOKED, SCHEDULED -] = PENDING diff --git a/arend/settings/tasks.py b/arend/settings/tasks.py new file mode 100644 index 0000000..839520a --- /dev/null +++ b/arend/settings/tasks.py @@ -0,0 +1,20 @@ +import logging + + +logger = logging.getLogger(__name__) + + +__all__ = [ + "TASK_MAX_RETRIES", + "TASK_DELAY", + "TASK_PRIORITY", + "TASK_DELAY_FACTOR", + "TASK_RETRY_BACKOFF_FACTOR", +] + +# settings +TASK_MAX_RETRIES = 3 +TASK_RETRY_BACKOFF_FACTOR = 1 +TASK_PRIORITY = 1 +TASK_DELAY_FACTOR: int = 1 +TASK_DELAY: int = 0 diff --git a/arend/tasks/async_task.py b/arend/tasks/async_task.py deleted file mode 100644 index 3f5fb09..0000000 --- a/arend/tasks/async_task.py +++ /dev/null @@ -1,79 +0,0 @@ -from arend.settings import settings -from arend.tasks.locking import Lock -from arend.tube.task import Task -from datetime import timedelta -from pydantic import BaseModel -from pystalkd.Beanstalkd import DEFAULT_PRIORITY -from typing import Callable -from typing import Union - -import datetime -import logging - - -logger = logging.getLogger(__name__) - - -class AsyncTask(BaseModel): - task_name: str - task_location: str - processor: Callable - queue_name: str = None - task_priority: int = None - task_delay: Union[timedelta, int] = None - exclusive: bool = False - - def __call__(self, *args, **kwargs): - return self.run(*args, **kwargs) - - def __repr__(self): - return f"<{self.__class__.__name__} at {self.task_location}>" - - def run(self, *args, **kwargs): - """ - Run the task immediately. - """ - if self.exclusive: - with Lock(name=f"{settings.env}.{self.task_location}"): - return self.processor(*args, **kwargs) - else: - return self.processor(*args, **kwargs) - - def apply_async( - self, - queue_name: str = None, - task_priority: str = None, - task_delay: Union[timedelta, int] = 0, - args: tuple = None, - kwargs: dict = None, - ) -> Task: - """ - Run task asynchronously. - """ - # settings - queue_name = self.queue_name or queue_name or settings.queue_name - task_delay = task_delay or self.task_delay or settings.task_delay or 0 - task_priority = ( - task_priority - or self.task_priority - or settings.task_priority - or DEFAULT_PRIORITY - ) - - # insert in backend - task = Task( - task_name=self.task_name, - task_location=self.task_location, - queue_name=queue_name, - task_priority=task_priority, - task_delay=task_delay, - args=args or (), - kwargs=kwargs or {}, - exclusive=self.exclusive, - created=datetime.datetime.utcnow(), - ).save() # set as SCHEDULED (default) - - # broker send to queue - task.send_to_queue() # put into the queue and set as PENDING - - return task # return a PENDING task diff --git a/arend/tasks/decorator.py b/arend/tasks/decorator.py deleted file mode 100644 index 0e27ed1..0000000 --- a/arend/tasks/decorator.py +++ /dev/null @@ -1,42 +0,0 @@ -from arend.tasks.async_task import AsyncTask -from datetime import timedelta -from typing import Union - -import functools -import logging - - -logger = logging.getLogger(__name__) - - -def arend_task( - queue_name: str = None, - queue_priority: int = None, - queue_delay: Union[timedelta, int] = None, - exclusive: bool = False, -): - """ - Register functions as async functions - Examples: - - @arend_task() - def task(kwargs): - return "a result" - """ - - def decorator(func): - @functools.wraps(func) - def wrapper_register(): - return AsyncTask( - task_name=func.__name__, - task_location=f"{func.__module__}.{func.__name__}", - processor=func, - queue_name=queue_name, - queue_priority=queue_priority, - queue_delay=queue_delay, - exclusive=exclusive, - ) - - return wrapper_register() - - return decorator diff --git a/arend/tasks/locking.py b/arend/tasks/locking.py deleted file mode 100644 index 5c95b49..0000000 --- a/arend/tasks/locking.py +++ /dev/null @@ -1,63 +0,0 @@ -from arend.settings import settings - -import logging -import redis - - -logger = logging.getLogger(__name__) - - -client = redis.Redis( - host=settings.redis_host, - db=settings.redis_db, - password=settings.redis_password, - socket_timeout=settings.socket_timeout, - socket_connect_timeout=settings.socket_connect_timeout, -) - - -class LockingException(Exception): - pass - - -class Lock: - def __init__(self, name: str, timeout: int = None): - """Acquire a lock for an object - :param name: a string that uniquely determines the locked object - :param timeout: the amount of seconds after a lock will expire - """ - self.name = name - self.timeout = timeout or 5 * 60 # 5 min - self.lock = None - - def flush(self): - locks = list(client.scan_iter(self.name)) - if len(locks) > 0: - client.delete(*locks) - - def acquire(self): - lock = client.lock(self.name, timeout=self.timeout, sleep=0) - if lock.acquire(blocking=False): - self.lock = lock - return lock - else: - raise LockingException( - f"Could not lock '{self.name}' as it already has lock" - ) - - def release(self): - if self.lock is not None: - try: - self.lock.release() - except redis.exceptions.LockError: - logger.warning( - f"Could not release lock '{self.lock.name}', " - f"maybe it has expired?" - ) - self.lock = None - - def __enter__(self): - return self.acquire() - - def __exit__(self, *args, **kwargs): - self.release() diff --git a/arend/tasks/registered_tasks.py b/arend/tasks/registered_tasks.py deleted file mode 100644 index de76e6f..0000000 --- a/arend/tasks/registered_tasks.py +++ /dev/null @@ -1,31 +0,0 @@ -from inspect import getmembers -from notifier.arend.tasks.async_task import AsyncTask -from typing import Dict -from typing import List - -import importlib -import logging - - -logger = logging.getLogger(__name__) - - -def is_async_task(object: object) -> bool: - """Return true if the object is a AsyncTask.""" - - return isinstance(object, AsyncTask) - - -def registered_tasks( - locations: List[str], file_name: str = "tasks" -) -> Dict[str, AsyncTask]: - """""" - tasks = {} - - for location in locations: - full_location = f"{location}.{file_name}" - module = importlib.import_module(full_location) - members = dict(getmembers(module, is_async_task)) - tasks.update({v.task_location: v for k, v in members.items()}) - - return tasks diff --git a/arend/tube/task.py b/arend/tube/task.py deleted file mode 100644 index b4056c0..0000000 --- a/arend/tube/task.py +++ /dev/null @@ -1,125 +0,0 @@ -from arend.backends import Backend -from arend.brokers import Broker -from arend.settings import settings -from arend.settings.status import FAIL -from arend.settings.status import FINISHED -from arend.settings.status import PENDING -from arend.settings.status import RETRY -from arend.settings.status import REVOKED -from arend.settings.status import SCHEDULED -from arend.settings.status import STARTED -from datetime import datetime -from pydantic import BaseModel -from pydantic import Field -from typing import Any -from typing import Optional -from uuid import uuid4 - -import logging -import traceback - - -logger = logging.getLogger(__name__) - - -DEFAULT_TTR = 30 * 60 # 30 min - - -class Task(BaseModel): - uuid: str = Field(default_factory=lambda: str(uuid4()), description="ID") - task_name: str = Field(description="Full path task name.") - task_location: str = Field(description="Full path task location.") - status: str = Field(default=SCHEDULED, description="Current status.") - result: Optional[Any] = Field(default=None, description="Task result.") - detail: str = Field(default="", description="Task details.") - start_time: Optional[datetime] = Field( - default=None, description="Datetime when task is STARTED." - ) - end_time: Optional[datetime] = Field( - default=None, - description="Datetime when task is FINISHED, FAIL, or REVOKED.", - ) - args: tuple = Field(default_factory=tuple, description="Task args.") - kwargs: dict = Field(default_factory=dict, description="Task arguments.") - - queue_name: str = Field(description="Queue name.") - task_priority: int = Field(description="Queue priority.") - task_delay: int = Field(description="Queue delay.") - - created: datetime = Field(default_factory=datetime.utcnow) - updated: datetime = None - exclusive: bool = False - count_retries: int = 0 - - @classmethod - def get(cls, uuid: str): - with Backend() as backend: - queue_task = backend.find_one(uuid=uuid) - - if queue_task: - return Task(**queue_task) - - def save(self): - with Backend() as backend: - self.updated = datetime.utcnow() - backend.update_one(task_uuid=self.uuid, to_update=self.dict()) - return self - - def send_to_queue(self): - with Broker(queue_name=self.queue_name) as broker: - broker.add_to_queue( - body=self.uuid, - priority=self.task_priority, - delay=self.task_delay - + self.count_retries * settings.delay_factor, - ttr=DEFAULT_TTR, - ) - self.status = PENDING - self.save() - - def notify(self, message: str): - self.detail += f"- {message}\n" - - def __enter__(self): - self.start_time = datetime.utcnow() - self.status = STARTED - self.save() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if exc_type: - last_trace = "".join(traceback.format_tb(exc_tb)).strip() - self.detail = f"Failure: {last_trace}\n" - if self.count_retries < settings.task_max_retries: - self.count_retries += 1 - self.status = RETRY - self.save() - # put in the tube again - self.send_to_queue() - else: - self.end_time = datetime.utcnow() - self.status = FAIL - self.save() - return True - - def __call__(self): - return self.run() - - def run(self): - - with self: - if self.status == [REVOKED, FAIL, FINISHED]: - return - - from arend.tasks.registered_tasks import registered_tasks - - registered = registered_tasks(locations=["notifier"]) - task = registered[self.task_location] - - result = task.run(*self.args, **self.kwargs) - - # update finished task - self.status = FINISHED - self.end_time = datetime.utcnow() - self.result = result - self.save() diff --git a/arend/worker/__init__.py b/arend/worker/__init__.py index 5e71931..9cbc152 100644 --- a/arend/worker/__init__.py +++ b/arend/worker/__init__.py @@ -1,10 +1,4 @@ -from arend.worker.worker import pool_processor +from arend.worker.consumer import consumer +from arend.worker.pool_consumers import pool_consumers -import logging - - -logger = logging.getLogger(__name__) - - -if __name__ == "__main__": - pool_processor() +__all__ = ["consumer", "pool_consumers"] diff --git a/arend/worker/consumer.py b/arend/worker/consumer.py index 711189d..bf908c2 100644 --- a/arend/worker/consumer.py +++ b/arend/worker/consumer.py @@ -1,40 +1,54 @@ -from arend.brokers import QueueBroker -from arend.settings import settings -from arend.tube.task import Task - +from arend.brokers import BeanstalkdConnection +from arend.settings import Settings, ArendSettings +from uuid import UUID import logging import time +__all__ = ["consumer"] + logger = logging.getLogger(__name__) -def consumer(queue_name: str, timeout: int = 20, testing: bool = False): +def consumer( + queue: str, + timeout: int = 20, + sleep_time: float = 0.1, + long_polling: bool = False, + settings: ArendSettings = None, +): """ - Single consumer. - - :param queue_name: - :param timeout: - :param testing: - :return: + Consumer. Consume tasks from the queue. + + Args: + queue: str. Queue name. + timeout: int. Polling timeout. + sleep_time: float. Sleeping time between polling cycles. + long_polling: bool. Break the loop if no more messages. + settings: ArendSettings, None. If no settings are passed, + the consumer will try to get them from env variables. + + Usage: + >>> from arend.worker.consumer import consumer + >>> + >>> consumer(queue="my_queue", timeout=0) """ - run = True + settings = settings or Settings().arend + Task = settings.get_backend() - while run: + while True: - with QueueBroker(queue_name=queue_name) as broker: + with BeanstalkdConnection(queue, settings=settings.beanstalkd) as conn: - message = broker.reserve(timeout=timeout) - if message is None and testing: # for testing purposes - run = False - continue + message = conn.reserve(timeout=timeout) + if message is None and not long_polling: + break # if not long_polling, consume all messages and break if message: - queue_task = Task.get(uuid=message.body) - if queue_task: - queue_task.run() # run sync inside worker + if task := Task.get(uuid=UUID(message.body)): + task.run() # run task here - broker.delete(message) + message.delete() - time.sleep(settings.sleep_time_consumer) + time.sleep(sleep_time) diff --git a/arend/worker/worker.py b/arend/worker/pool_consumers.py similarity index 73% rename from arend/worker/worker.py rename to arend/worker/pool_consumers.py index 2284d94..5fd1ab2 100644 --- a/arend/worker/worker.py +++ b/arend/worker/pool_consumers.py @@ -5,19 +5,18 @@ import logging +__all__ = ["pool_consumers"] + + logger = logging.getLogger(__name__) -@click.command(name="Start Processor") +@click.command(name="Start Pool of Consumers") @click.argument("args", nargs=-1) -def pool_processor(args): +def pool_consumers(args): """ - Pool processor. - - :param args: str. - Example: - ```python3 /path/to/pool_processor --queue_1=2 --queue_2=3``` + ```python3 pool_processor --queue_1=4 --queue_2=2``` """ # parse arguments diff --git a/beanstalkd/Dockerfile b/beanstalkd/Dockerfile new file mode 100644 index 0000000..2b035f1 --- /dev/null +++ b/beanstalkd/Dockerfile @@ -0,0 +1,6 @@ +FROM alpine:latest +LABEL maintainer="Jose Maria Vazquez Jimenez" + +RUN apk add --no-cache beanstalkd + +EXPOSE 11300 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..30dec6f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,43 @@ +version: '3.3' +services: + + beanstalkd: + build: + context: beanstalkd + dockerfile: Dockerfile + command: /usr/bin/beanstalkd -b /var/lib/beanstalkd + ports: + - "11300:11300" + + redis: + image: redis:4.0.9-alpine + command: redis-server --requirepass 'pass' + ports: + - "6379:6379" + + mongo: + image: mongo:latest + environment: + - MONGO_INITDB_ROOT_USERNAME=user + - MONGO_INITDB_ROOT_PASSWORD=pass + ports: + - "27017:27017" + + app: + build: + context: . + dockerfile: Dockerfile + depends_on: + - beanstalkd + - mongo + - redis + environment: + - DOCKER=True + - PYTHONUNBUFFERED=1 + - PYTHONPATH=$PYTHONPATH:/code + - POETRY_VIRTUALENVS_IN_PROJECT=true + - SHELL=/bin/bash + ports: + - "8888:8888" # for jupyter examples + volumes: + - ./:/code diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/arend.rst b/docs/arend.rst new file mode 100644 index 0000000..236af95 --- /dev/null +++ b/docs/arend.rst @@ -0,0 +1,8 @@ +.. _arend: + + +Arend +============== + +.. automodule:: arend + :members: arend_task diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..b46d4dc --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,56 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + +import os +import sys +from typing import List + +sys.path.insert(0, os.path.abspath("..")) + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "arend" +copyright = "2022, Jose-Maria Vazquez-Jimenez" +author = "Jose-Maria Vazquez-Jimenez" + +# -- General configuration --------------------------------------------------- + +master_doc = "index" + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon"] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path: List[str] = [] + +# mock imports so that we don't need all the dependencies to build the docs +autodoc_mock_imports = ["requests"] diff --git a/docs/help.rst b/docs/help.rst new file mode 100644 index 0000000..d1353b9 --- /dev/null +++ b/docs/help.rst @@ -0,0 +1,7 @@ +.. _help: + +Any help +======== + +Everyone is encouraged to file bug reports, feature requests, and pull requests +through `GitHub `_. diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..51c19b6 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,107 @@ +.. arend documentation master file, created by + sphinx-quickstart on Tue Aug 9 09:38:22 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. +.. _index: + +Welcome to arend' documentation! +============================================= + +A simple producer-consumer library for the Beanstalkd queue! + +Installation +--------------- + +Install it using `pip`:: + + pip install arend + +Basic usage +--------------- + +Make sure you have the `arend` installed:: + + from arend import arend_task + from arend.backends.mongo import MongoSettings + from arend.brokers import BeanstalkdSettings + from arend.settings import ArendSettings + from arend.worker import consumer + + settings = ArendSettings( + beanstalkd=BeanstalkdSettings(host="beanstalkd", port=11300), + backend=MongoSettings( + mongo_connection="mongodb://user:pass@mongo:27017", + mongo_db="db", + mongo_collection="Tasks" + ), + ) + + @arend_task(queue="my_queue", settings=settings) + def double(num: int): + return 2 * num + + double(2) # returns 4 + task = double.apply_async(args=(4,)) # It is sent to the queue + + consumer(queue="my_queue", settings=settings) # consume tasks from the queue + + Task = settings.get_backend() # you can check your backend for the result + task = Task.get(uuid=task.uuid) + assert task.result == 4 + + +Backends +------------------- +The available backends to store logs are **Mongo** and **Redis**. + +Setting your backend with environment variables +--------------------------------------------------- +You can set your backend by defining env vars. +The `AREND__` prefix indicates that it belongs to the Arend:: + + # Redis + AREND__REDIS_HOST='redis' + AREND__REDIS_DB='1' + AREND__REDIS_PASSWORD='pass' + ... + + # Mongo + AREND__MONGO_CONNECTION='mongodb://user:pass@mongo:27017' + AREND__MONGO_DB='db' + AREND__MONGO_COLLECTION='logs' + ... + +Now in your code:: + + from arend import arend_task + from arend.worker import consumer + + + @arend_task(queue="my_queue") + def double(num: int): + return 2 * num + + double.apply_async(args=(4,)) # It is sent to the queue + + consumer(queue="my_queue") + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + installation + arend + settings + mongo + redis + license + help + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..57e576e --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,52 @@ +.. _installation: + + +Installation +============= + +Install arend:: + + pip install arend + + +Advanced: local setup for development (Ubuntu) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These instructions assume that ``git``, ``docker``, and ``docker-compose`` are +installed on your host machine. + +First, clone this repo and make some required directories.:: + + git clone https://github.com/pyprogrammerblog/arend.git + cd arend + +Then build the docker image:: + + docker-compose run --rm app poetry install + +Then install dependencies:: + + docker-compose run --rm app poetry install + +Run the tests:: + + docker-compose run app poetry run pytest + +Then run the app and access inside the docker with the env activated:: + + docker-compose run --rm app poetry shell + +Finally you can down the services:: + + docker-compose down + +Advanced: Jupyter Notebook +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Hit the command:: + + docker-compose run --rm -p 8888:8888 app poetry shell + +Then inside the docker:: + + jupyter notebook --ip 0.0.0.0 --no-browser --allow-root diff --git a/docs/license.rst b/docs/license.rst new file mode 100644 index 0000000..696cb3d --- /dev/null +++ b/docs/license.rst @@ -0,0 +1,27 @@ +.. _license: + + +License +======= + +The MIT License (MIT) + +Copyright (c) 2022, JM Vazquez-Jimenez + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NON INFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..8d79f1e --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,37 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/mongo.rst b/docs/mongo.rst new file mode 100644 index 0000000..3f3eee2 --- /dev/null +++ b/docs/mongo.rst @@ -0,0 +1,9 @@ +.. _mongo_backends: + + +Mongo Backend +============== + +.. automodule:: arend.backends.mongo + :members: MongoSettings, MongoTask, MongoTasks + :exclude-members: mongo_collection diff --git a/docs/redis.rst b/docs/redis.rst new file mode 100644 index 0000000..f719dcf --- /dev/null +++ b/docs/redis.rst @@ -0,0 +1,9 @@ +.. _redis_backends: + + +Redis Backend +============== + +.. automodule:: arend.backends.redis + :members: RedisSettings, RedisTask, RedisTasks + :exclude-members: redis_connection diff --git a/docs/settings.rst b/docs/settings.rst new file mode 100644 index 0000000..8ea0fdf --- /dev/null +++ b/docs/settings.rst @@ -0,0 +1,8 @@ +.. _settings: + + +Settings +============== + +.. automodule:: arend.settings + :members: ArendSettings, BeanstalkdSettings diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..a4963a6 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,1464 @@ +[[package]] +name = "alabaster" +version = "0.7.12" +description = "A configurable sidebar-enabled Sphinx theme" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "appnope" +version = "0.1.3" +description = "Disable App Nap on macOS >= 10.9" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "argon2-cffi" +version = "21.3.0" +description = "The secure Argon2 password hashing algorithm." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +argon2-cffi-bindings = "*" + +[package.extras] +dev = ["pre-commit", "cogapp", "tomli", "coverage[toml] (>=5.0.2)", "hypothesis", "pytest", "sphinx", "sphinx-notfound-page", "furo"] +docs = ["sphinx", "sphinx-notfound-page", "furo"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pytest"] + +[[package]] +name = "argon2-cffi-bindings" +version = "21.2.0" +description = "Low-level CFFI bindings for Argon2" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = ">=1.0.1" + +[package.extras] +dev = ["pytest", "cogapp", "pre-commit", "wheel"] +tests = ["pytest"] + +[[package]] +name = "asttokens" +version = "2.0.8" +description = "Annotate AST trees with source code positions" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +six = "*" + +[package.extras] +test = ["astroid (<=2.5.3)", "pytest"] + +[[package]] +name = "async-timeout" +version = "4.0.2" +description = "Timeout context manager for asyncio programs" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "attrs" +version = "22.1.0" +description = "Classes Without Boilerplate" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] + +[[package]] +name = "babel" +version = "2.10.3" +description = "Internationalization utilities" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pytz = ">=2015.7" + +[[package]] +name = "backcall" +version = "0.2.0" +description = "Specifications for callback functions passed in to an API" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "beautifulsoup4" +version = "4.11.1" +description = "Screen-scraping library" +category = "dev" +optional = false +python-versions = ">=3.6.0" + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "black" +version = "22.8.0" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""} +typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "bleach" +version = "5.0.1" +description = "An easy safelist-based HTML-sanitizing tool." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +six = ">=1.9.0" +webencodings = "*" + +[package.extras] +css = ["tinycss2 (>=1.1.0,<1.2)"] +dev = ["build (==0.8.0)", "flake8 (==4.0.1)", "hashin (==0.17.0)", "pip-tools (==6.6.2)", "pytest (==7.1.2)", "Sphinx (==4.3.2)", "tox (==3.25.0)", "twine (==4.0.1)", "wheel (==0.37.1)", "black (==22.3.0)", "mypy (==0.961)"] + +[[package]] +name = "certifi" +version = "2022.9.24" +description = "Python package for providing Mozilla's CA Bundle." +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "cffi" +version = "1.15.1" +description = "Foreign Function Interface for Python calling C code." +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "2.1.1" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "dev" +optional = false +python-versions = ">=3.6.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.5" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "debugpy" +version = "1.6.3" +description = "An implementation of the Debug Adapter Protocol for Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "decorator" +version = "5.1.1" +description = "Decorators for Humans" +category = "dev" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "defusedxml" +version = "0.7.1" +description = "XML bomb protection for Python stdlib modules" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "deprecated" +version = "1.2.13" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["tox", "bump2version (<1)", "sphinx (<2)", "importlib-metadata (<3)", "importlib-resources (<4)", "configparser (<5)", "sphinxcontrib-websupport (<2)", "zipp (<2)", "PyTest (<5)", "PyTest-Cov (<2.6)", "pytest", "pytest-cov"] + +[[package]] +name = "docutils" +version = "0.17.1" +description = "Docutils -- Python Documentation Utilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "entrypoints" +version = "0.4" +description = "Discover and load entry points from installed packages." +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "executing" +version = "1.1.0" +description = "Get the currently executing AST node of a frame, and other information" +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +tests = ["rich", "littleutils", "pytest", "asttokens"] + +[[package]] +name = "fastjsonschema" +version = "2.16.2" +description = "Fastest Python implementation of JSON schema" +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +devel = ["colorama", "jsonschema", "json-spec", "pylint", "pytest", "pytest-benchmark", "pytest-cache", "validictory"] + +[[package]] +name = "flake8" +version = "4.0.1" +description = "the modular source code checker: pep8 pyflakes and co" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +mccabe = ">=0.6.0,<0.7.0" +pycodestyle = ">=2.8.0,<2.9.0" +pyflakes = ">=2.4.0,<2.5.0" + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "dev" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "imagesize" +version = "1.4.1" +description = "Getting image size from png/jpeg/jpeg2000/gif file" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "importlib-metadata" +version = "5.0.0" +description = "Read metadata from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] +perf = ["ipython"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "ipykernel" +version = "6.16.0" +description = "IPython Kernel for Jupyter" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +appnope = {version = "*", markers = "platform_system == \"Darwin\""} +debugpy = ">=1.0" +ipython = ">=7.23.1" +jupyter-client = ">=6.1.12" +matplotlib-inline = ">=0.1" +nest-asyncio = "*" +packaging = "*" +psutil = "*" +pyzmq = ">=17" +tornado = ">=6.1" +traitlets = ">=5.1.0" + +[package.extras] +test = ["flaky", "ipyparallel", "pre-commit", "pytest-cov", "pytest-timeout", "pytest (>=6.0)"] + +[[package]] +name = "ipython" +version = "8.5.0" +description = "IPython: Productive Interactive Computing" +category = "dev" +optional = false +python-versions = ">=3.8" + +[package.dependencies] +appnope = {version = "*", markers = "sys_platform == \"darwin\""} +backcall = "*" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +decorator = "*" +jedi = ">=0.16" +matplotlib-inline = "*" +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} +pickleshare = "*" +prompt-toolkit = ">3.0.1,<3.1.0" +pygments = ">=2.4.0" +stack-data = "*" +traitlets = ">=5" + +[package.extras] +all = ["black", "Sphinx (>=1.3)", "ipykernel", "nbconvert", "nbformat", "ipywidgets", "notebook", "ipyparallel", "qtconsole", "pytest (<7.1)", "pytest-asyncio", "testpath", "curio", "matplotlib (!=3.2.0)", "numpy (>=1.19)", "pandas", "trio"] +black = ["black"] +doc = ["Sphinx (>=1.3)"] +kernel = ["ipykernel"] +nbconvert = ["nbconvert"] +nbformat = ["nbformat"] +notebook = ["ipywidgets", "notebook"] +parallel = ["ipyparallel"] +qtconsole = ["qtconsole"] +test = ["pytest (<7.1)", "pytest-asyncio", "testpath"] +test_extra = ["pytest (<7.1)", "pytest-asyncio", "testpath", "curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.19)", "pandas", "trio"] + +[[package]] +name = "ipython-genutils" +version = "0.2.0" +description = "Vestigial utilities from IPython" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "jedi" +version = "0.18.1" +description = "An autocompletion tool for Python that can be used for text editors." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +parso = ">=0.8.0,<0.9.0" + +[package.extras] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["Django (<3.1)", "colorama", "docopt", "pytest (<7.0.0)"] + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonschema" +version = "4.16.0" +description = "An implementation of JSON Schema validation for Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] + +[[package]] +name = "jupyter-client" +version = "7.3.5" +description = "Jupyter protocol implementation and client libraries" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +entrypoints = "*" +jupyter-core = ">=4.9.2" +nest-asyncio = ">=1.5.4" +python-dateutil = ">=2.8.2" +pyzmq = ">=23.0" +tornado = ">=6.2" +traitlets = "*" + +[package.extras] +doc = ["ipykernel", "myst-parser", "sphinx-rtd-theme", "sphinx (>=1.3.6)", "sphinxcontrib-github-alt"] +test = ["codecov", "coverage", "ipykernel (>=6.5)", "ipython", "mypy", "pre-commit", "pytest", "pytest-asyncio (>=0.18)", "pytest-cov", "pytest-timeout"] + +[[package]] +name = "jupyter-core" +version = "4.11.1" +description = "Jupyter core package. A base package on which Jupyter projects rely." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +pywin32 = {version = ">=1.0", markers = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\""} +traitlets = "*" + +[package.extras] +test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] + +[[package]] +name = "jupyterlab-pygments" +version = "0.2.2" +description = "Pygments theme using JupyterLab CSS variables" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "lxml" +version = "4.9.1" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html5 = ["html5lib"] +htmlsoup = ["beautifulsoup4"] +source = ["Cython (>=0.29.7)"] + +[[package]] +name = "markupsafe" +version = "2.1.1" +description = "Safely add untrusted strings to HTML/XML markup." +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "matplotlib-inline" +version = "0.1.6" +description = "Inline Matplotlib backend for Jupyter" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +traitlets = "*" + +[[package]] +name = "mccabe" +version = "0.6.1" +description = "McCabe checker, plugin for flake8" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "mistune" +version = "2.0.4" +description = "A sane Markdown parser with useful plugins and renderers" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "mock" +version = "4.0.3" +description = "Rolling backport of unittest.mock for all Pythons" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +build = ["twine", "wheel", "blurb"] +docs = ["sphinx"] +test = ["pytest (<5.4)", "pytest-cov"] + +[[package]] +name = "mypy" +version = "0.971" +description = "Optional static typing for Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +mypy-extensions = ">=0.4.3" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=3.10" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "nbclient" +version = "0.6.8" +description = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor." +category = "dev" +optional = false +python-versions = ">=3.7.0" + +[package.dependencies] +jupyter-client = ">=6.1.5" +nbformat = ">=5.0" +nest-asyncio = "*" +traitlets = ">=5.2.2" + +[package.extras] +sphinx = ["autodoc-traits", "mock", "moto", "myst-parser", "Sphinx (>=1.7)", "sphinx-book-theme"] +test = ["black", "check-manifest", "flake8", "ipykernel", "ipython", "ipywidgets", "mypy", "nbconvert", "pip (>=18.1)", "pre-commit", "pytest (>=4.1)", "pytest-asyncio", "pytest-cov (>=2.6.1)", "setuptools (>=60.0)", "testpath", "twine (>=1.11.0)", "xmltodict"] + +[[package]] +name = "nbconvert" +version = "7.0.0" +description = "Converting Jupyter Notebooks" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +beautifulsoup4 = "*" +bleach = "*" +defusedxml = "*" +importlib-metadata = {version = ">=3.6", markers = "python_version < \"3.10\""} +jinja2 = ">=3.0" +jupyter-core = ">=4.7" +jupyterlab-pygments = "*" +lxml = "*" +markupsafe = ">=2.0" +mistune = ">=2.0.3,<3" +nbclient = ">=0.5.0" +nbformat = ">=5.1" +packaging = "*" +pandocfilters = ">=1.4.1" +pygments = ">=2.4.1" +tinycss2 = "*" +traitlets = ">=5.0" + +[package.extras] +all = ["ipykernel", "ipython", "ipywidgets (>=7)", "nbsphinx (>=0.2.12)", "pre-commit", "pyppeteer (>=1,<1.1)", "pyqtwebengine (>=5.15)", "pytest", "pytest-cov", "pytest-dependency", "sphinx-rtd-theme", "sphinx (==5.0.2)", "tornado (>=6.1)"] +docs = ["ipython", "nbsphinx (>=0.2.12)", "sphinx-rtd-theme", "sphinx (==5.0.2)"] +qtpdf = ["pyqtwebengine (>=5.15)"] +qtpng = ["pyqtwebengine (>=5.15)"] +serve = ["tornado (>=6.1)"] +test = ["ipykernel", "ipywidgets (>=7)", "pre-commit", "pyppeteer (>=1,<1.1)", "pytest", "pytest-cov", "pytest-dependency"] +webpdf = ["pyppeteer (>=1,<1.1)"] + +[[package]] +name = "nbformat" +version = "5.6.1" +description = "The Jupyter Notebook format" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +fastjsonschema = "*" +jsonschema = ">=2.6" +jupyter-core = "*" +traitlets = ">=5.1" + +[package.extras] +test = ["check-manifest", "pep440", "pre-commit", "pytest", "testpath"] + +[[package]] +name = "nest-asyncio" +version = "1.5.6" +description = "Patch asyncio to allow nested event loops" +category = "dev" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "notebook" +version = "6.4.12" +description = "A web-based notebook environment for interactive computing" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +argon2-cffi = "*" +ipykernel = "*" +ipython-genutils = "*" +jinja2 = "*" +jupyter-client = ">=5.3.4" +jupyter-core = ">=4.6.1" +nbconvert = ">=5" +nbformat = "*" +nest-asyncio = ">=1.5" +prometheus-client = "*" +pyzmq = ">=17" +Send2Trash = ">=1.8.0" +terminado = ">=0.8.3" +tornado = ">=6.1" +traitlets = ">=4.2.1" + +[package.extras] +docs = ["sphinx", "nbsphinx", "sphinxcontrib-github-alt", "sphinx-rtd-theme", "myst-parser"] +json-logging = ["json-logging"] +test = ["pytest", "coverage", "requests", "testpath", "nbval", "selenium", "pytest-cov", "requests-unixsocket"] + +[[package]] +name = "numpy" +version = "1.23.3" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.8" + +[[package]] +name = "packaging" +version = "21.3" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" + +[[package]] +name = "pandas" +version = "1.5.0" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = false +python-versions = ">=3.8" + +[package.dependencies] +numpy = [ + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, +] +python-dateutil = ">=2.8.1" +pytz = ">=2020.1" + +[package.extras] +test = ["pytest-xdist (>=1.31)", "pytest (>=6.0)", "hypothesis (>=5.5.3)"] + +[[package]] +name = "pandocfilters" +version = "1.5.0" +description = "Utilities for writing pandoc filters in python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "parso" +version = "0.8.3" +description = "A Python Parser" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["docopt", "pytest (<6.0.0)"] + +[[package]] +name = "pathspec" +version = "0.10.1" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "pexpect" +version = "4.8.0" +description = "Pexpect allows easy control of interactive console applications." +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +ptyprocess = ">=0.5" + +[[package]] +name = "pickleshare" +version = "0.7.5" +description = "Tiny 'shelve'-like database with concurrency support" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "platformdirs" +version = "2.5.2" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"] +test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +testing = ["pytest-benchmark", "pytest"] +dev = ["tox", "pre-commit"] + +[[package]] +name = "prometheus-client" +version = "0.14.1" +description = "Python client for the Prometheus monitoring system." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +twisted = ["twisted"] + +[[package]] +name = "prompt-toolkit" +version = "3.0.31" +description = "Library for building powerful interactive command lines in Python" +category = "dev" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +wcwidth = "*" + +[[package]] +name = "psutil" +version = "5.9.2" +description = "Cross-platform lib for process and system monitoring in Python." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +test = ["ipaddress", "mock", "enum34", "pywin32", "wmi"] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +description = "Run a subprocess in a pseudo terminal" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "pure-eval" +version = "0.2.2" +description = "Safely evaluate AST nodes without side effects" +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +tests = ["pytest"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "pycodestyle" +version = "2.8.0" +description = "Python style guide checker" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pydantic" +version = "1.10.2" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +typing-extensions = ">=4.1.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyflakes" +version = "2.4.0" +description = "passive checker of Python programs" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pygments" +version = "2.13.0" +description = "Pygments is a syntax highlighting package written in Python." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +plugins = ["importlib-metadata"] + +[[package]] +name = "pymongo" +version = "4.2.0" +description = "Python driver for MongoDB " +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +aws = ["pymongo-auth-aws (<2.0.0)"] +encryption = ["pymongocrypt (>=1.3.0,<2.0.0)"] +gssapi = ["pykerberos"] +ocsp = ["pyopenssl (>=17.2.0)", "requests (<3.0.0)", "service-identity (>=18.1.0)", "certifi"] +snappy = ["python-snappy"] +srv = ["dnspython (>=1.16.0,<3.0.0)"] +zstd = ["zstandard"] + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" +optional = false +python-versions = ">=3.6.8" + +[package.extras] +diagrams = ["railroad-diagrams", "jinja2"] + +[[package]] +name = "pyrsistent" +version = "0.18.1" +description = "Persistent/Functional/Immutable data structures" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "pystalkd" +version = "1.3.0" +description = "Beanstalkd bindings for python3" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +yaml = ["pyyaml"] + +[[package]] +name = "pytest" +version = "7.1.3" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +tomli = ">=1.0.0" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2022.4" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "pywin32" +version = "304" +description = "Python for Window Extensions" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "pywinpty" +version = "2.0.8" +description = "Pseudo terminal support for Windows from Python." +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "pyzmq" +version = "24.0.1" +description = "Python bindings for 0MQ" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = {version = "*", markers = "implementation_name == \"pypy\""} +py = {version = "*", markers = "implementation_name == \"pypy\""} + +[[package]] +name = "redis" +version = "4.3.4" +description = "Python client for Redis database and key-value store" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +async-timeout = ">=4.0.2" +deprecated = ">=1.2.3" +packaging = ">=20.4" + +[package.extras] +hiredis = ["hiredis (>=1.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] + +[[package]] +name = "requests" +version = "2.28.1" +description = "Python HTTP for Humans." +category = "dev" +optional = false +python-versions = ">=3.7, <4" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "send2trash" +version = "1.8.0" +description = "Send file to trash natively under Mac OS X, Windows and Linux." +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +nativelib = ["pyobjc-framework-cocoa", "pywin32"] +objc = ["pyobjc-framework-cocoa"] +win32 = ["pywin32"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "snowballstemmer" +version = "2.2.0" +description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "soupsieve" +version = "2.3.2.post1" +description = "A modern CSS selector implementation for Beautiful Soup." +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "sphinx" +version = "5.2.3" +description = "Python documentation generator" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +alabaster = ">=0.7,<0.8" +babel = ">=2.9" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +docutils = ">=0.14,<0.20" +imagesize = ">=1.3" +importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""} +Jinja2 = ">=3.0" +packaging = ">=21.0" +Pygments = ">=2.12" +requests = ">=2.5.0" +snowballstemmer = ">=2.0" +sphinxcontrib-applehelp = "*" +sphinxcontrib-devhelp = "*" +sphinxcontrib-htmlhelp = ">=2.0.0" +sphinxcontrib-jsmath = "*" +sphinxcontrib-qthelp = "*" +sphinxcontrib-serializinghtml = ">=1.1.5" + +[package.extras] +docs = ["sphinxcontrib-websupport"] +lint = ["flake8 (>=3.5.0)", "flake8-comprehensions", "flake8-bugbear", "flake8-simplify", "isort", "mypy (>=0.981)", "sphinx-lint", "docutils-stubs", "types-typed-ast", "types-requests"] +test = ["pytest (>=4.6)", "html5lib", "typed-ast", "cython"] + +[[package]] +name = "sphinx-rtd-theme" +version = "1.0.0" +description = "Read the Docs theme for Sphinx" +category = "dev" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" + +[package.dependencies] +docutils = "<0.18" +sphinx = ">=1.6" + +[package.extras] +dev = ["bump2version", "sphinxcontrib-httpdomain", "transifex-client"] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "1.0.2" +description = "sphinxcontrib-applehelp is a sphinx extension which outputs Apple help books" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +test = ["pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "1.0.2" +description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +test = ["pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.0.0" +description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +test = ["html5lib", "pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +description = "A sphinx extension which renders display math in HTML via JavaScript" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +test = ["mypy", "flake8", "pytest"] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "1.0.3" +description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +test = ["pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "1.1.5" +description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +lint = ["flake8", "mypy", "docutils-stubs"] +test = ["pytest"] + +[[package]] +name = "stack-data" +version = "0.5.1" +description = "Extract data from python stack frames and tracebacks for informative displays" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +asttokens = "*" +executing = "*" +pure-eval = "*" + +[package.extras] +tests = ["cython", "littleutils", "pygments", "typeguard", "pytest"] + +[[package]] +name = "terminado" +version = "0.16.0" +description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +ptyprocess = {version = "*", markers = "os_name != \"nt\""} +pywinpty = {version = ">=1.1.0", markers = "os_name == \"nt\""} +tornado = ">=6.1.0" + +[package.extras] +test = ["pre-commit", "pytest-timeout", "pytest (>=6.0)"] + +[[package]] +name = "tinycss2" +version = "1.1.1" +description = "A tiny CSS parser" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +webencodings = ">=0.4" + +[package.extras] +test = ["coverage", "pytest-isort", "pytest-flake8", "pytest-cov", "pytest"] +doc = ["sphinx-rtd-theme", "sphinx"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "tornado" +version = "6.2" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +category = "dev" +optional = false +python-versions = ">= 3.7" + +[[package]] +name = "traitlets" +version = "5.4.0" +description = "" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pre-commit", "pytest"] + +[[package]] +name = "typing-extensions" +version = "4.3.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "urllib3" +version = "1.26.12" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" + +[package.extras] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "wcwidth" +version = "0.2.5" +description = "Measures the displayed width of unicode strings in a terminal" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "webencodings" +version = "0.5.1" +description = "Character encoding aliases for legacy web content" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "wrapt" +version = "1.14.1" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[[package]] +name = "zipp" +version = "3.8.1" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.9" +content-hash = "67eb5b9f875fff74ad5ce003a554817ac4475e3af40dda7cf55d377fe10c5019" + +[metadata.files] +alabaster = [] +appnope = [] +argon2-cffi = [] +argon2-cffi-bindings = [] +asttokens = [] +async-timeout = [] +attrs = [] +babel = [] +backcall = [] +beautifulsoup4 = [] +black = [] +bleach = [] +certifi = [] +cffi = [] +charset-normalizer = [] +click = [] +colorama = [] +debugpy = [] +decorator = [] +defusedxml = [] +deprecated = [] +docutils = [] +entrypoints = [] +executing = [] +fastjsonschema = [] +flake8 = [] +idna = [] +imagesize = [] +importlib-metadata = [] +iniconfig = [] +ipykernel = [] +ipython = [] +ipython-genutils = [] +jedi = [] +jinja2 = [] +jsonschema = [] +jupyter-client = [] +jupyter-core = [] +jupyterlab-pygments = [] +lxml = [] +markupsafe = [] +matplotlib-inline = [] +mccabe = [] +mistune = [] +mock = [] +mypy = [] +mypy-extensions = [] +nbclient = [] +nbconvert = [] +nbformat = [] +nest-asyncio = [] +notebook = [] +numpy = [] +packaging = [] +pandas = [] +pandocfilters = [] +parso = [] +pathspec = [] +pexpect = [] +pickleshare = [] +platformdirs = [] +pluggy = [] +prometheus-client = [] +prompt-toolkit = [] +psutil = [] +ptyprocess = [] +pure-eval = [] +py = [] +pycodestyle = [] +pycparser = [] +pydantic = [] +pyflakes = [] +pygments = [] +pymongo = [] +pyparsing = [] +pyrsistent = [] +pystalkd = [] +pytest = [] +python-dateutil = [] +pytz = [] +pywin32 = [] +pywinpty = [] +pyyaml = [] +pyzmq = [] +redis = [] +requests = [] +send2trash = [] +six = [] +snowballstemmer = [] +soupsieve = [] +sphinx = [] +sphinx-rtd-theme = [] +sphinxcontrib-applehelp = [] +sphinxcontrib-devhelp = [] +sphinxcontrib-htmlhelp = [] +sphinxcontrib-jsmath = [] +sphinxcontrib-qthelp = [] +sphinxcontrib-serializinghtml = [] +stack-data = [] +terminado = [] +tinycss2 = [] +tomli = [] +tornado = [] +traitlets = [] +typing-extensions = [] +urllib3 = [] +wcwidth = [] +webencodings = [] +wrapt = [] +zipp = [] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..19390c0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[tool.poetry] +name = "arend" +version = "0.1.0" +description = "A simple producer consumer library for Beanstalkd." +readme = "README.md" +license = "LICENSE" +include = ["CHANGELOG.md"] +authors = ["Jose Vazquez "] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +homepage = "https://github.com/pyprogrammerblog/arend" +documentation = "https://arend.readthedocs.io/en/latest/" + +[virtualenvs] +create = true # this creates a .venv folder +in-project = true + +[tool.poetry.dependencies] +python = "^3.9" +pydantic = "^1.9.1" +pandas = "^1.4.3" +pystalkd = "^1.3.0" +pymongo = "^4.2.0" +redis = "^4.3.4" +PyYAML = "^6.0" + +[tool.poetry.dev-dependencies] +pytest = "^7.1.2" +flake8 = "^4.0.1" +black = "^22.6.0" +ipython = "^8.4.0" +notebook = "^6.4.12" +mock = "^4.0.3" +mypy = "^0.971" +Sphinx = "^5.1.1" +sphinx-rtd-theme = "^1.0.0" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index f9ad31e..0000000 --- a/setup.cfg +++ /dev/null @@ -1,19 +0,0 @@ -[zest.releaser] -release = no - -[flake8] -exclude = -ignore = E203, E266, E501, W503 - -[isort] -atomic = true -force_alphabetical_sort = true -force_single_line = true -include_trailing_comma = true -lines_after_imports = 2 -multi_line_output = 3 -not_skip = __init__.py -use_parentheses = true - -[tool:pytest] -python_files = test_*.py diff --git a/setup.py b/setup.py deleted file mode 100644 index c5059df..0000000 --- a/setup.py +++ /dev/null @@ -1,55 +0,0 @@ -from setuptools import setup - -import os - - -version = "0.0.1" - -long_description = "\n\n".join([open("README.rst").read(), open("CHANGES.rst").read()]) - -install_requires = [ - "pydantic", - "pymongo", - "pystalkd", -] - -# emulate "--no-deps" on the readthedocs build (there is no way to specify this -# behaviour in the .readthedocs.yml) -if os.environ.get("READTHEDOCS") == "True": - install_requires = [] - - -tests_require = [ - "requests", - "pytest", - "mock", - "pydantic", - "pymongo", - "pystalkd", -] - -setup( - name="Arend", - version=version, - description="A ", - long_description=long_description, - # Get strings from http://www.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - ], - keywords=["arend", "producer-consumer", "process-queue"], - author="Jose Maria Vazquez Jimenez", - author_email="josevazjim88@gmail.com", - url="", - license="MIT License", - packages=["arend"], - include_package_data=True, - zip_safe=False, - install_requires=install_requires, - tests_require=tests_require, - python_requires=">=3.6", - extras_require={"test": tests_require}, - entry_points={"console_scripts": []}, -) diff --git a/arend/tasks/__init__.py b/tests/__init__.py similarity index 100% rename from arend/tasks/__init__.py rename to tests/__init__.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..4045482 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,96 @@ +import redis +import pytest +import os +from arend import arend_task +from pymongo.mongo_client import MongoClient +from arend.settings.arend import BeanstalkdSettings +from arend.brokers.beanstalkd import BeanstalkdConnection + + +def flush_beanstalkd_queue(queue: str, settings: BeanstalkdSettings): + with BeanstalkdConnection(queue=queue, settings=settings) as conn: + while True: + message = conn.reserve(timeout=0) + if not message: + break + message.delete() + + +@pytest.fixture(scope="function") +def mongo_backend(): + with MongoClient( + "mongodb://user:pass@mongo:27017", UuidRepresentation="standard" + ) as client: + client.drop_database("db") + db = client.get_database("db") + collection = db.get_collection("logs") + yield collection + client.drop_database("db") + + +@pytest.fixture(scope="function") +def redis_backend(): + with redis.Redis(host="redis", password="pass", port=6379, db=1) as r: + r.flushdb() + yield r + r.flushdb() + + +@pytest.fixture(scope="function") +def beanstalkd_setting(): + settings = BeanstalkdSettings(host="beanstalkd", port=11300) + flush_beanstalkd_queue(queue="test", settings=settings) + yield settings + flush_beanstalkd_queue(queue="test", settings=settings) + + +@pytest.fixture(scope="function") +def env_vars_redis(): + os.environ["AREND__BACKEND__REDIS_HOST"] = "redis" + os.environ["AREND__BACKEND__REDIS_DB"] = "1" + os.environ["AREND__BACKEND__REDIS_PASSWORD"] = "pass" + os.environ["AREND__BEANSTALKD__HOST"] = "beanstalkd" + os.environ["AREND__BEANSTALKD__PORT"] = "11300" + try: + yield + finally: + del os.environ["AREND__BACKEND__REDIS_HOST"] + del os.environ["AREND__BACKEND__REDIS_DB"] + del os.environ["AREND__BACKEND__REDIS_PASSWORD"] + del os.environ["AREND__BEANSTALKD__HOST"] + del os.environ["AREND__BEANSTALKD__PORT"] + + +@pytest.fixture(scope="function") +def env_vars_mongo(): + os.environ[ + "AREND__BACKEND__MONGO_CONNECTION" + ] = "mongodb://user:pass@mongo:27017" + os.environ["AREND__BACKEND__MONGO_DB"] = "db" + os.environ["AREND__BACKEND__MONGO_COLLECTION"] = "logs" + os.environ["AREND__BEANSTALKD__HOST"] = "beanstalkd" + os.environ["AREND__BEANSTALKD__PORT"] = "11300" + try: + yield + finally: + del os.environ["AREND__BACKEND__MONGO_CONNECTION"] + del os.environ["AREND__BACKEND__MONGO_DB"] + del os.environ["AREND__BACKEND__MONGO_COLLECTION"] + del os.environ["AREND__BEANSTALKD__HOST"] + del os.environ["AREND__BEANSTALKD__PORT"] + + +@arend_task(queue="test") +def task_capitalize(name: str): + """ + Example task for testing + """ + return name.capitalize() + + +@arend_task(queue="test") +def task_capitalize_all(name: str): + """ + Example task for testing + """ + return name.upper() diff --git a/arend/tests/__init__.py b/tests/test_arend/__init__.py similarity index 100% rename from arend/tests/__init__.py rename to tests/test_arend/__init__.py diff --git a/tests/test_arend/test_decorator.py b/tests/test_arend/test_decorator.py new file mode 100644 index 0000000..31b2461 --- /dev/null +++ b/tests/test_arend/test_decorator.py @@ -0,0 +1,125 @@ +from arend.backends.mongo import MongoSettings +from arend.backends.redis import RedisSettings +from arend.settings import ArendSettings, Settings +from tests.conftest import task_capitalize_all, task_capitalize +from arend.brokers.beanstalkd import BeanstalkdSettings + + +def test_arend_task_mongo_backend(beanstalkd_setting, mongo_backend): + + settings = ArendSettings( + beanstalkd=BeanstalkdSettings(host="beanstalkd", port=11300), + backend=MongoSettings( + mongo_connection="mongodb://user:pass@mongo:27017", + mongo_collection="logs", + mongo_db="db", + ), + ) + + assert 0 == mongo_backend.count_documents({}) + assert "Capitalize" == task_capitalize(name="capitalize") + assert 0 == mongo_backend.count_documents({}) + + task = task_capitalize.apply_async( + queue="test", args=("capitalize",), settings=settings + ) + assert 1 == mongo_backend.count_documents({}) + assert "PENDING" == mongo_backend.find_one({})["status"] + + task.run() + + Task = settings.get_backend() + task = Task.get(uuid=task.uuid) + assert task.result == "Capitalize" + assert task.start_time < task.end_time + assert task.get_task_signature() == task_capitalize + + assert 1 == mongo_backend.count_documents({}) + assert 1 == task.delete() + assert 0 == mongo_backend.count_documents({}) + + +def test_arend_task_redis_backend(beanstalkd_setting, redis_backend): + + settings = ArendSettings( + beanstalkd=BeanstalkdSettings(host="beanstalkd", port=11300), + backend=RedisSettings(redis_host="redis", redis_password="pass"), + ) + + assert 0 == redis_backend.dbsize() + assert "CAPITALIZE" == task_capitalize_all(name="capitalize") + assert 0 == redis_backend.dbsize() + + task = task_capitalize_all.apply_async( + queue="test", args=("capitalize",), settings=settings + ) + assert 1 == redis_backend.dbsize() + + task.run() + + Task = settings.get_backend() + task = Task.get(uuid=task.uuid) + assert "FINISHED" == task.status + assert task.result == "CAPITALIZE" + assert task.start_time < task.end_time + assert task.get_task_signature() == task_capitalize_all + + assert 1 == redis_backend.dbsize() + assert 1 == task.delete() + assert 0 == redis_backend.dbsize() + + +def test_arend_task_mongo_env_vars_backend( + beanstalkd_setting, mongo_backend, env_vars_mongo +): + + settings = Settings().arend + + assert 0 == mongo_backend.count_documents({}) + assert "Capitalize" == task_capitalize(name="capitalize") + assert 0 == mongo_backend.count_documents({}) + + task = task_capitalize.apply_async( + queue="test", args=("capitalize",), settings=settings + ) + assert 1 == mongo_backend.count_documents({}) + assert "PENDING" == mongo_backend.find_one({})["status"] + + task.run() + + Task = settings.get_backend() + task = Task.get(uuid=task.uuid) + assert task.result == "Capitalize" + assert task.start_time < task.end_time + assert task.get_task_signature() == task_capitalize + + assert 1 == mongo_backend.count_documents({}) + assert 1 == task.delete() + assert 0 == mongo_backend.count_documents({}) + + +def test_arend_task_redis_env_vars_backend( + beanstalkd_setting, redis_backend, env_vars_redis +): + + settings = Settings().arend + + assert 0 == redis_backend.dbsize() + assert "CAPITALIZE" == task_capitalize_all(name="capitalize") + assert 0 == redis_backend.dbsize() + + task = task_capitalize_all.apply_async(queue="test", args=("capitalize",)) + assert 1 == redis_backend.dbsize() + + task.run() + + Task = settings.get_backend() + task = Task.get(uuid=task.uuid) + assert "FINISHED" == task.status + assert task.result == "CAPITALIZE" + assert task.start_time < task.end_time + assert task.get_task_signature() == task_capitalize_all + + assert 1 == redis_backend.dbsize() + assert 1 == task.delete() + assert 0 == redis_backend.dbsize() diff --git a/arend/tube/__init__.py b/tests/test_backends/__init__.py similarity index 100% rename from arend/tube/__init__.py rename to tests/test_backends/__init__.py diff --git a/tests/test_backends/test_mongo.py b/tests/test_backends/test_mongo.py new file mode 100644 index 0000000..e1ba9d1 --- /dev/null +++ b/tests/test_backends/test_mongo.py @@ -0,0 +1,57 @@ +import uuid +from arend.settings import Settings, ArendSettings +from arend.settings.arend import BeanstalkdSettings +from arend.backends.mongo import MongoSettings + + +def test_create_settings_passing_params_mongo(mongo_backend): + + mongo_settings = MongoSettings( + mongo_connection="mongodb://user:pass@mongo:27017", + mongo_db="db", + mongo_collection="logs", + ) + beanstalkd_settings = BeanstalkdSettings(host="beanstalkd", port=11300) + settings = ArendSettings( + backend=mongo_settings, beanstalkd=beanstalkd_settings + ) + + Task = settings.get_backend() + task = Task(name="My task", queue="test") + + assert 0 == mongo_backend.count_documents(filter={}) + + task.description = "A description" + task = task.save() + task = Task.get(uuid=task.uuid) + + assert task.description == "A description" + assert isinstance(task.uuid, uuid.UUID) + + assert 1 == mongo_backend.count_documents(filter={}) + assert 1 == task.delete() + + assert 0 == mongo_backend.count_documents(filter={}) + assert 0 == task.delete() + + +def test_create_settings_env_vars_mongo(mongo_backend, env_vars_mongo): + + settings = Settings() + Task = settings.arend.get_backend() + task = Task(name="My task", queue="test") + + assert 0 == mongo_backend.count_documents(filter={}) + + task.description = "A description" + task = task.save() + task = Task.get(uuid=task.uuid) + + assert task.description == "A description" + assert isinstance(task.uuid, uuid.UUID) + + assert 1 == mongo_backend.count_documents(filter={}) + assert 1 == task.delete() + + assert 0 == mongo_backend.count_documents(filter={}) + assert 0 == task.delete() diff --git a/tests/test_backends/test_redis.py b/tests/test_backends/test_redis.py new file mode 100644 index 0000000..aa31d59 --- /dev/null +++ b/tests/test_backends/test_redis.py @@ -0,0 +1,49 @@ +import uuid +from arend.settings import Settings, ArendSettings +from arend.settings.arend import BeanstalkdSettings +from arend.backends.redis import RedisTask, RedisSettings + + +def test_create_settings_passing_params_redis(redis_backend): + + redis_settings = RedisSettings(redis_host="redis", redis_password="pass") + beanstalkd_settings = BeanstalkdSettings(host="beanstalkd", port=11300) + settings = ArendSettings( + backend=redis_settings, beanstalkd=beanstalkd_settings + ) + + Task = settings.get_backend() + task: RedisTask = Task(name="My task", queue="test") + + assert not redis_backend.get(str(task.uuid)) + + task.description = "A description" + task = task.save() + task = Task.get(uuid=task.uuid) + + assert task.description == "A description" + assert isinstance(task.uuid, uuid.UUID) + assert redis_backend.get(str(task.uuid)) + assert 1 == task.delete() + assert not redis_backend.get(str(task.uuid)) + assert 0 == task.delete() + + +def test_create_settings_env_vars_redis(redis_backend, env_vars_redis): + + settings = Settings() + Task = settings.arend.get_backend() + task: RedisTask = Task(name="My task", queue="test") + + assert not redis_backend.get(str(task.uuid)) + + task.description = "A description" + task = task.save() + task = Task.get(uuid=task.uuid) + + assert task.description == "A description" + assert isinstance(task.uuid, uuid.UUID) + assert redis_backend.get(str(task.uuid)) + assert 1 == task.delete() + assert not redis_backend.get(str(task.uuid)) + assert 0 == task.delete() diff --git a/arend/tests/test_pool_processors.py b/tests/test_broker/__init__.py similarity index 100% rename from arend/tests/test_pool_processors.py rename to tests/test_broker/__init__.py diff --git a/tests/test_broker/test_beanstalkd.py b/tests/test_broker/test_beanstalkd.py new file mode 100644 index 0000000..017cd69 --- /dev/null +++ b/tests/test_broker/test_beanstalkd.py @@ -0,0 +1,49 @@ +import pytest +from uuid import uuid4 +from arend.brokers.beanstalkd import BeanstalkdConnection +from pystalkd.Beanstalkd import CommandFailed + + +def test_beanstalkd_broker(beanstalkd_setting): + + body = str(uuid4()) + + with BeanstalkdConnection( + queue="test", settings=beanstalkd_setting + ) as conn: + + conn.put(body=body) + + stats = conn.stats() + assert stats["current-jobs-urgent"] == 0 + assert stats["current-jobs-ready"] == 1 + assert stats["current-jobs-reserved"] == 0 + assert stats["current-jobs-delayed"] == 0 + assert stats["current-jobs-buried"] == 0 + + tube_stats = conn.stats_tube(name="test") + assert tube_stats["current-jobs-ready"] == 1 + assert tube_stats["total-jobs"] == 1 + assert tube_stats["current-using"] == 1 + assert tube_stats["current-watching"] == 1 + + job = conn.reserve() + + assert job.body == body + assert conn.stats_job(job_id=job.job_id) + + job.delete() + + with pytest.raises(CommandFailed): + job.delete() + + +def test_beanstalkd_broker_put_messages(beanstalkd_setting): + + with BeanstalkdConnection( + queue="test", settings=beanstalkd_setting + ) as conn: + body = str(uuid4()) + conn.put(body=body) + job = conn.reserve() + assert job.body == body diff --git a/arend/tests/test_settings.py b/tests/test_settings/__init__.py similarity index 100% rename from arend/tests/test_settings.py rename to tests/test_settings/__init__.py diff --git a/tests/test_settings/test_settings.py b/tests/test_settings/test_settings.py new file mode 100644 index 0000000..622f8dc --- /dev/null +++ b/tests/test_settings/test_settings.py @@ -0,0 +1,50 @@ +from arend.settings import Settings, ArendSettings +from arend.settings.arend import BeanstalkdSettings +from arend.backends.mongo import MongoSettings, MongoTask +from arend.backends.redis import RedisSettings, RedisTask + + +# passing params +def test_create_settings_passing_params_redis(): + + redis_settings = RedisSettings(redis_password="pass", redis_host="redis") + beanstalkd_settings = BeanstalkdSettings(host="beanstalkd", port=11300) + settings = ArendSettings( + backend=redis_settings, beanstalkd=beanstalkd_settings + ) + + klass = settings.get_backend() + assert issubclass(klass, RedisTask) + assert klass.Meta.settings == settings + + +def test_create_settings_passing_params_mongo(): + + mongo_settings = MongoSettings( + mongo_connection="mongodb://user:pass@mongo:27017", + mongo_db="db", + mongo_collection="logs", + ) + beanstalkd_settings = BeanstalkdSettings(host="beanstalkd", port=11300) + settings = ArendSettings( + backend=mongo_settings, beanstalkd=beanstalkd_settings + ) + + klass = settings.get_backend() + assert issubclass(klass, MongoTask) + assert klass.Meta.settings == settings + + +# env vars +def test_create_settings_env_vars_redis(env_vars_redis): + settings = Settings() + klass = settings.arend.get_backend() + assert klass.Meta.settings == settings.arend + assert issubclass(klass, RedisTask) + + +def test_create_settings_env_vars_mongo(env_vars_mongo): + settings = Settings() + klass = settings.arend.get_backend() + assert klass.Meta.settings == settings.arend + assert issubclass(klass, MongoTask) diff --git a/tests/test_worker/__init__.py b/tests/test_worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_worker/test_consumer.py b/tests/test_worker/test_consumer.py new file mode 100644 index 0000000..84433cf --- /dev/null +++ b/tests/test_worker/test_consumer.py @@ -0,0 +1,95 @@ +from arend.backends.mongo import MongoSettings +from arend.backends.redis import RedisSettings +from arend.settings import ArendSettings, Settings +from tests.conftest import task_capitalize_all, task_capitalize +from arend.brokers.beanstalkd import BeanstalkdSettings +from arend.worker.consumer import consumer +import json + + +def test_arend_task_mongo_backend(beanstalkd_setting, mongo_backend): + + settings = ArendSettings( + beanstalkd=BeanstalkdSettings(host="beanstalkd", port=11300), + backend=MongoSettings( + mongo_connection="mongodb://user:pass@mongo:27017", + mongo_collection="logs", + mongo_db="db", + ), + ) + + assert 0 == mongo_backend.count_documents({}) + task_capitalize.apply_async( + queue="test", args=("capitalize",), settings=settings + ) + assert 1 == mongo_backend.count_documents({}) + assert "PENDING" == mongo_backend.find_one({})["status"] + + consumer(queue="test", long_polling=False, timeout=0, settings=settings) + + assert 1 == mongo_backend.count_documents({}) + assert "FINISHED" == mongo_backend.find_one({})["status"] + assert "Capitalize" == mongo_backend.find_one({})["result"] + + +def test_arend_task_redis_backend(beanstalkd_setting, redis_backend): + + settings = ArendSettings( + beanstalkd=BeanstalkdSettings(host="beanstalkd", port=11300), + backend=RedisSettings(redis_host="redis", redis_password="pass"), + ) + + assert 0 == redis_backend.dbsize() + task = task_capitalize.apply_async( + queue="test", args=("capitalize",), settings=settings + ) + assert 1 == redis_backend.dbsize() + assert "PENDING" == json.loads(redis_backend.get(str(task.uuid)))["status"] + + consumer(queue="test", long_polling=False, timeout=0, settings=settings) + + assert 1 == redis_backend.dbsize() + result = json.loads(redis_backend.get(str(task.uuid))) + assert "FINISHED" == result["status"] + assert "Capitalize" == result["result"] + + +def test_arend_task_mongo_env_vars_backend( + beanstalkd_setting, mongo_backend, env_vars_mongo +): + + settings = Settings().arend + + assert 0 == mongo_backend.count_documents({}) + task_capitalize_all.apply_async( + queue="test", args=("capitalize",), settings=settings + ) + assert 1 == mongo_backend.count_documents({}) + assert "PENDING" == mongo_backend.find_one({})["status"] + + consumer(queue="test", long_polling=False, timeout=0, settings=settings) + + assert 1 == mongo_backend.count_documents({}) + assert "FINISHED" == mongo_backend.find_one({})["status"] + assert "CAPITALIZE" == mongo_backend.find_one({})["result"] + + +def test_arend_task_redis_env_vars_backend( + beanstalkd_setting, redis_backend, env_vars_redis +): + + settings = Settings().arend + + assert 0 == redis_backend.dbsize() + task = task_capitalize.apply_async( + queue="test", args=("capitalize",), settings=settings + ) + assert 1 == redis_backend.dbsize() + assert "PENDING" == json.loads(redis_backend.get(str(task.uuid)))["status"] + + consumer(queue="test", long_polling=False, timeout=0, settings=settings) + + assert 1 == redis_backend.dbsize() + result = json.loads(redis_backend.get(str(task.uuid))) + assert "FINISHED" == result["status"] + assert "Capitalize" == result["result"]