From 2f8485e8cf7ea5f621915ecad348b7e0ed962a28 Mon Sep 17 00:00:00 2001 From: Dmirty Simonov Date: Wed, 24 Nov 2021 18:11:00 +0300 Subject: [PATCH] cmd cleanup, step context builder removed --- carnival/__init__.py | 2 - carnival/cli.py | 9 +-- carnival/cmd/__init__.py | 4 +- carnival/cmd/apt.py | 120 ------------------------------------- carnival/cmd/cli.py | 20 +------ carnival/cmd/fs.py | 61 ++++++++++++------- carnival/cmd/system.py | 72 ++++------------------ carnival/cmd/systemd.py | 82 ------------------------- carnival/cmd/transfer.py | 17 +++--- carnival/connection.py | 36 ----------- carnival/context.py | 83 ------------------------- carnival/exceptions.py | 14 ++--- carnival/internal_tasks.py | 1 - carnival/step.py | 28 ++++----- carnival/task.py | 46 +++++++------- carnival/tasks_loader.py | 18 +++--- carnival/templates.py | 17 +++--- carnival/utils.py | 27 +++++---- 18 files changed, 141 insertions(+), 516 deletions(-) delete mode 100644 carnival/cmd/apt.py delete mode 100644 carnival/cmd/systemd.py delete mode 100644 carnival/connection.py delete mode 100644 carnival/context.py diff --git a/carnival/__init__.py b/carnival/__init__.py index 2c20aa0..61d8365 100644 --- a/carnival/__init__.py +++ b/carnival/__init__.py @@ -6,7 +6,6 @@ from carnival import cmd from carnival import internal_tasks from carnival.utils import log -from carnival.context import context_ref if not sys.warnoptions: @@ -23,6 +22,5 @@ 'TaskBase', 'StepsTask', 'cmd', 'log', - 'context_ref', 'internal_tasks', ] diff --git a/carnival/cli.py b/carnival/cli.py index ae21c95..2cca542 100644 --- a/carnival/cli.py +++ b/carnival/cli.py @@ -1,6 +1,7 @@ import os import sys -from typing import Any, Dict, Iterable, Type +import typing +import collections import click import dotenv @@ -23,10 +24,10 @@ def is_completion_script(complete_var: str) -> bool: return os.getenv(complete_var, None) is not None -task_types: Dict[str, Type[TaskBase]] = {} +task_types: typing.OrderedDict[str, typing.Type[TaskBase]] = collections.OrderedDict() -def except_hook(type: Type[Any], value: Any, traceback: Any) -> None: +def except_hook(type: typing.Type[typing.Any], value: typing.Any, traceback: typing.Any) -> None: print(f"{type.__name__}: {value} \nYou can use --debug flag to see full traceback.") @@ -49,7 +50,7 @@ def main() -> int: @click.command() @click.option('--debug', is_flag=True, default=False, help="Turn on debug mode") @click.argument('tasks', required=True, type=click.Choice(list(task_types.keys())), nargs=-1) - def cli(debug: bool, tasks: Iterable[str]) -> None: + def cli(debug: bool, tasks: typing.Iterable[str]) -> None: if debug is True: print("Debug mode on.") else: diff --git a/carnival/cmd/__init__.py b/carnival/cmd/__init__.py index 0d2fb66..da91654 100644 --- a/carnival/cmd/__init__.py +++ b/carnival/cmd/__init__.py @@ -7,14 +7,12 @@ Основные шаги доступны в отдельном репозитории: . """ -from carnival.cmd import cli, system, systemd, apt, transfer, fs +from carnival.cmd import cli, system, transfer, fs __all__ = [ 'cli', 'system', - 'systemd', - 'apt', 'transfer', 'fs', ] diff --git a/carnival/cmd/apt.py b/carnival/cmd/apt.py deleted file mode 100644 index e4b9b44..0000000 --- a/carnival/cmd/apt.py +++ /dev/null @@ -1,120 +0,0 @@ -from typing import List, Optional - -from carnival import cmd -from carnival.utils import log - - -def get_pkg_versions(pkgname: str) -> List[str]: - """ - Получить список доступных версий пакета - """ - versions = [] - result = cmd.cli.run(f"DEBIAN_FRONTEND=noninteractive apt-cache madison {pkgname}", hide=True, warn=True) - if result.ok is False: - return [] - - for line in result.stdout.strip().split("\n"): - n, ver, r = line.split("|") - versions.append(ver.strip()) - return versions - - -def get_installed_version(pkgname: str) -> Optional[str]: - """ - Получить установленную версию пакета - - :return: Версия пакета если установлен, `None` если пакет не установлен - """ - result = cmd.cli.run(f"DEBIAN_FRONTEND=noninteractive dpkg -l {pkgname}", hide=True, warn=True) - if result.ok is False: - return None - installed, pkgn, ver, arch, *desc = result.stdout.strip().split("\n")[-1].split() - if installed != 'ii': - return None - - assert isinstance(ver, str) - return ver.strip() - - -def is_pkg_installed(pkgname: str, version: Optional[str] = None) -> bool: - """ - Проверить установлен ли пакет - Если версия не указана - проверяется любая - """ - - pkgver = get_installed_version(pkgname) - if version is None and pkgver is not None: - return True - - if version is not None and pkgver == version: - return True - - return False - - -def force_install(pkgname: str, version: Optional[str] = None, update: bool = False, hide: bool = False) -> None: - """ - Установить пакет без проверки установлен ли он - """ - if version: - pkgname = f"{pkgname}={version}" - - if update: - cmd.cli.run("DEBIAN_FRONTEND=noninteractive sudo apt-get update", pty=True, hide=hide) - - cmd.cli.run(f"DEBIAN_FRONTEND=noninteractive sudo apt-get install -y {pkgname}", pty=True, hide=hide) - - -def install(pkgname: str, version: Optional[str] = None, update: bool = True, hide: bool = False) -> bool: - """ - Установить пакет если он еще не установлен в системе - - :param pkgname: название пакета - :param version: версия - :param update: запустить apt-get update перед установкой - :param hide: скрыть вывод этапов - :return: `True` если пакет был установлен, `False` если пакет уже был установлен ранее - """ - if is_pkg_installed(pkgname, version): - if version: - if not hide: - log(f"{pkgname}={version} already installed") - else: - if not hide: - log(f"{pkgname} already installed") - return False - force_install(pkgname=pkgname, version=version, update=update, hide=hide) - return True - - -def install_multiple(*pkg_names: str, update: bool = True, hide: bool = False) -> bool: - """ - Установить несколько пакетов, если они не установлены - - :param pkg_names: список пакетов которые нужно установить - :param update: запустить apt-get update перед установкой - :param hide: скрыть вывод этапов - :return: `True` если хотя бы один пакет был установлен, `False` если все пакеты уже были установлен ранее - """ - if all([is_pkg_installed(x) for x in pkg_names]): - if not hide: - log(f"{','.join(pkg_names)} already installed") - return False - - if update: - cmd.cli.run("DEBIAN_FRONTEND=noninteractive sudo apt-get update", pty=True, hide=hide) - - for pkg in pkg_names: - install(pkg, update=False, hide=hide) - return True - - -def remove(*pkg_names: str, hide: bool = False) -> None: - """ - Удалить пакет - - :param pkg_names: список пакетов которые нужно удалить - :param hide: скрыть вывод этапов - """ - assert pkg_names, "pkg_names is empty" - cmd.cli.run(f"DEBIAN_FRONTEND=noninteractive sudo apt-get remove --auto-remove -y {' '.join(pkg_names)}", hide=hide) diff --git a/carnival/cmd/cli.py b/carnival/cmd/cli.py index 7698bcd..54b370d 100644 --- a/carnival/cmd/cli.py +++ b/carnival/cmd/cli.py @@ -1,25 +1,11 @@ -from typing import Any - -from carnival import connection +from carnival.host import AnyConnection from invoke import Result # type: ignore -def _run_command(command: str, **kwargs: Any) -> Result: - assert connection.conn is not None, "No connection" - return connection.conn.run(command, **kwargs) - - -def run(command: str, **kwargs: Any) -> Result: +def run(c: AnyConnection, command: str, warn: bool = True, hide: bool = False) -> Result: """ Запустить комманду - """ - return _run_command(command, **kwargs) - - -def pty(command: str, **kwargs: Any) -> Result: - """ - Запустить комманду, используя псевдотерминальную сессию См """ - return run(command, pty=True, **kwargs) + return c.run(command, pty=True, warn=warn, hide=hide) diff --git a/carnival/cmd/fs.py b/carnival/cmd/fs.py index a50774e..617660b 100644 --- a/carnival/cmd/fs.py +++ b/carnival/cmd/fs.py @@ -1,32 +1,45 @@ from typing import List, Optional +import re + +from carnival import cmd +from carnival.host import AnyConnection -from carnival import cmd, connection from invoke import Result # type: ignore -from patchwork import files # type:ignore -def mkdirs(*dirs: str) -> List[Result]: +def _escape_for_regex(text: str) -> str: + """Escape ``text`` to allow literal matching using egrep""" + regex = re.escape(text) + # Seems like double escaping is needed for \ + regex = regex.replace("\\\\", "\\\\\\") + # Triple-escaping seems to be required for $ signs + regex = regex.replace(r"\$", r"\\\$") + # Whereas single quotes should not be escaped + regex = regex.replace(r"\'", "'") + return regex + + +def mkdirs(c: AnyConnection, *dirs: str) -> List[Result]: """ Создать директории :param dirs: пути которые нужно создать """ - return [cmd.cli.run(f"mkdir -p {x}", hide=True) for x in dirs] + return [cmd.cli.run(c, f"mkdir -p {x}", hide=True) for x in dirs] -def is_dir_exists(dir_path: str) -> bool: +def is_dir_exists(c: AnyConnection, dir_path: str) -> bool: """ Узнать существует ли директория :param dir_path: путь до директории """ - return bool(cmd.cli.run(f"test -d {dir_path}", warn=True, hide=True).ok) + return bool(cmd.cli.run(c, f"test -d {dir_path}", warn=True, hide=True).ok) -def is_file_contains(filename: str, text: str, exact: bool = False, escape: bool = True) -> bool: +def is_file_contains(c: AnyConnection, filename: str, text: str, exact: bool = False, escape: bool = True) -> bool: """ Содержит ли файл текст - См :param filename: путь до файла :param text: текст который нужно искать @@ -35,26 +48,27 @@ def is_file_contains(filename: str, text: str, exact: bool = False, escape: bool """ - assert connection.conn is not None, "No connection" - return bool(files.contains( - connection.conn, - runner=connection.conn.run, - filename=filename, text=text, exact=exact, escape=escape - )) + if escape: + text = _escape_for_regex(text) + if exact: + text = "^{}$".format(text) + egrep_cmd = 'egrep "{}" "{}"'.format(text, filename) + return c.run(egrep_cmd, hide=True, warn=True).ok # type: ignore -def is_file_exists(path: str) -> bool: +def is_file_exists(c: AnyConnection, path: str) -> bool: """ Проверить существует ли файл - :param path: путь до файла """ - assert connection.conn is not None, "No connection" - return bool(files.exists(connection.conn, runner=connection.conn.run, path=path)) + + cmd = 'test -e "$(echo {})"'.format(path) + return c.run(cmd, hide=True, warn=True).ok # type: ignore def ensure_dir_exists( + c: AnyConnection, path: str, user: Optional[str] = None, group: Optional[str] = None, @@ -63,12 +77,15 @@ def ensure_dir_exists( """ Проверить что директория существует и параметры соответствуют заданным - - :param path: путь до директории :param user: владелец :param group: группа :param mode: права """ - assert connection.conn is not None, "No connection" - files.directory(connection.conn, runner=connection.conn.run, path=path, user=user, group=group, mode=mode) + + c.run("mkdir -p {}".format(path)) + if user is not None: + group = group or user + c.run("chown {}:{} {}".format(user, group, path)) + if mode is not None: + c.run("chmod {} {}".format(mode, path)) diff --git a/carnival/cmd/system.py b/carnival/cmd/system.py index 8ab26ee..ac40205 100644 --- a/carnival/cmd/system.py +++ b/carnival/cmd/system.py @@ -1,88 +1,36 @@ -import os -from typing import List - from carnival import cmd +from carnival.host import AnyConnection + from invoke import Result # type: ignore -def set_password(username: str, password: str) -> Result: +def set_password(c: AnyConnection, username: str, password: str) -> Result: """ Установить пароль пользователю :param username: Пользователь :param password: Новый пароль """ - return cmd.cli.pty(f"echo '{username}:{password}' | chpasswd", hide=True) - - -def ssh_authorized_keys_add(ssh_key: str, keys_file: str = ".ssh/authorized_keys") -> bool: - """ - Добавить ssh ключ в `authorized_keys` - - :param ssh_key: ключ - :param keys_file: пусть до файла `authorized_keys` - :return: `True` если ключ был добавлен, `False` если ключ уже был в файле - """ - ssh_key = ssh_key.strip() - - cmd.cli.run("mkdir -p ~/.ssh") - cmd.cli.run("chmod 700 ~/.ssh") - cmd.cli.run(f"touch {keys_file}") - - if not cmd.fs.is_file_contains(keys_file, ssh_key, escape=True): - cmd.cli.run(f"echo '{ssh_key}' >> {keys_file}") - return True - return False - - -def ssh_authorized_keys_list() -> List[str]: - """ - Получить список авторизованных ssh-ключей сервера - """ - if cmd.fs.is_file_exists("~/.ssh/authorized_keys") is False: - return [] - - keys_file: str = cmd.cli.run("cat ~/.ssh/authorized_keys", hide=True).stdout.strip() - return keys_file.split("\n") - - -def ssh_authorized_keys_ensure(*ssh_keys: str) -> List[bool]: - """ - Добавить несколько ssh-ключей в авторизованные - - :param ssh_keys: ssh-ключи - :return: Список `True` если ключ был добавлен, `False` если ключ уже был в файле - """ - return [ssh_authorized_keys_add(x) for x in ssh_keys] - - -def ssh_copy_id(pubkey_file: str = "~/.ssh/id_rsa.pub") -> bool: - """ - Добавить публичный ssh-ключ текущего пользователя в авторизованные - - :param pubkey_file: путь до файла с публичным ключем - :return: `True` если ключ был добавлен, `False` если ключ уже был в файле - """ - return ssh_authorized_keys_add(open(os.path.expanduser(pubkey_file)).read().strip()) + return cmd.cli.run(c, f"echo '{username}:{password}' | chpasswd", hide=True) -def get_current_user_name() -> str: +def get_current_user_name(c: AnyConnection) -> str: """ Получить имя текущего пользователя """ - id_res: str = cmd.cli.run("id -u -n", hide=True).stdout + id_res: str = cmd.cli.run(c, "id -u -n", hide=True).stdout return id_res.strip() -def get_current_user_id() -> int: +def get_current_user_id(c: AnyConnection) -> int: """ Получить id текущего пользователя """ - return int(cmd.cli.run("id -u", hide=True).stdout.strip()) + return int(cmd.cli.run(c, "id -u", hide=True).stdout.strip()) -def is_current_user_root() -> bool: +def is_current_user_root(c: AnyConnection) -> bool: """ Проверить что текущий пользователь - `root` """ - return get_current_user_id() == 0 + return get_current_user_id(c) == 0 diff --git a/carnival/cmd/systemd.py b/carnival/cmd/systemd.py deleted file mode 100644 index 3706fd9..0000000 --- a/carnival/cmd/systemd.py +++ /dev/null @@ -1,82 +0,0 @@ -from carnival import cmd -from invoke import Result # type: ignore - - -def daemon_reload() -> Result: - """ - Перегрузить systemd - """ - return cmd.cli.run("sudo systemctl --system daemon-reload") - - -def start(service_name: str, reload_daemon: bool = False) -> Result: - """ - Запустить сервис - - :param service_name: имя сервиса - :param reload_daemon: перегрузить systemd - """ - - if reload_daemon: - daemon_reload() - return cmd.cli.run(f"sudo systemctl start {service_name}") - - -def stop(service_name: str, reload_daemon: bool = False) -> Result: - """ - Остановить сервис - - :param service_name: имя сервиса - :param reload_daemon: перегрузить systemd - """ - if reload_daemon: - daemon_reload() - return cmd.cli.run(f"sudo systemctl stop {service_name}") - - -def restart(service_name: str) -> Result: - """ - Перезапустить сервис - - :param service_name: имя сервиса - """ - return cmd.cli.run(f"sudo systemctl restart {service_name}") - - -def enable(service_name: str, reload_daemon: bool = False, start_now: bool = True) -> Result: - """ - Добавить сервис в автозапуск - - :param service_name: имя сервиса - :param reload_daemon: перегрузить systemd - :param start_now: запустить сервис после добавления - """ - if reload_daemon: - daemon_reload() - - res = cmd.cli.run(f"sudo systemctl enable {service_name}") - - if start_now: - start(service_name) - - return res - - -def disable(service_name: str, reload_daemon: bool = False, stop_now: bool = True) -> Result: - """ - Убрать сервис из автозапуска - - :param service_name: имя сервиса - :param reload_daemon: перегрузить systemd - :param stop_now: Остановить сервис - """ - - if reload_daemon: - daemon_reload() - - res = cmd.cli.run(f"sudo systemctl disable {service_name}") - - if stop_now: - stop(service_name) - - return res diff --git a/carnival/cmd/transfer.py b/carnival/cmd/transfer.py index c9edbaf..6a96ada 100644 --- a/carnival/cmd/transfer.py +++ b/carnival/cmd/transfer.py @@ -1,13 +1,14 @@ from io import BytesIO from typing import Any, Iterable -from carnival import connection +from carnival.host import AnyConnection from carnival.templates import render from fabric.transfer import Result, Transfer # type:ignore from patchwork import transfers # type:ignore def rsync( + c: AnyConnection, source: str, target: str, exclude: Iterable[str] = (), delete: bool = False, strict_host_keys: bool = True, @@ -18,7 +19,7 @@ def rsync( """ return transfers.rsync( - c=connection.conn, + c=c, source=source, target=target, exclude=exclude, @@ -29,7 +30,7 @@ def rsync( ) -def get(remote: str, local: str, preserve_mode: bool = True) -> Result: +def get(c: AnyConnection, remote: str, local: str, preserve_mode: bool = True) -> Result: """ Скачать файл с сервера @@ -38,11 +39,11 @@ def get(remote: str, local: str, preserve_mode: bool = True) -> Result: :param local: путь куда сохранить файл :param preserve_mode: сохранить права """ - t = Transfer(connection.conn) + t = Transfer(c) return t.get(remote=remote, local=local, preserve_mode=preserve_mode) -def put(local: str, remote: str, preserve_mode: bool = True) -> Result: +def put(c: AnyConnection, local: str, remote: str, preserve_mode: bool = True) -> Result: """ Закачать файл на сервер @@ -51,11 +52,11 @@ def put(local: str, remote: str, preserve_mode: bool = True) -> Result: :param remote: путь куда сохранить на сервере :param preserve_mode: сохранить права """ - t = Transfer(connection.conn) + t = Transfer(c) return t.put(local=local, remote=remote, preserve_mode=preserve_mode) -def put_template(template_path: str, remote: str, **context: Any) -> Result: +def put_template(c: AnyConnection, template_path: str, remote: str, **context: Any) -> Result: """ Отрендерить файл с помощью jinja-шаблонов и закачать на сервер См раздел templates. @@ -67,5 +68,5 @@ def put_template(template_path: str, remote: str, **context: Any) -> Result: :param context: контекс для рендеринга jinja2 """ filestr = render(template_path=template_path, **context) - t = Transfer(connection.conn) + t = Transfer(c) return t.put(local=BytesIO(filestr.encode()), remote=remote, preserve_mode=False) diff --git a/carnival/connection.py b/carnival/connection.py deleted file mode 100644 index e392ffd..0000000 --- a/carnival/connection.py +++ /dev/null @@ -1,36 +0,0 @@ -from typing import Any, Optional, Type, Union - -from fabric import Connection # type:ignore -from invoke import Context # type:ignore - -from carnival.host import AnyHost -from carnival.exceptions import GlobalConnectionError - - -# noinspection PyTypeChecker -conn: Union[Connection, Context, None] = None -# noinspection PyTypeChecker -host: Optional[AnyHost] = None - - -class SetConnection: - def __init__(self, h: AnyHost): - self.host = h - - def __enter__(self) -> None: - global conn - global host - - if host is not None: - raise GlobalConnectionError(f"Cannot set context, while other context active: {host}") - if conn is not None: - raise GlobalConnectionError(f"Cannot set context, while other context active: {conn}") - - conn = self.host.connect() - host = self.host - - def __exit__(self, exc_type: Type[Any], exc_val: Any, exc_tb: Any) -> None: - global conn - global host - conn = None - host = None diff --git a/carnival/context.py b/carnival/context.py deleted file mode 100644 index d255fd1..0000000 --- a/carnival/context.py +++ /dev/null @@ -1,83 +0,0 @@ -import inspect -import os -import typing - -from carnival.exceptions import ContextBuilderError, ContextBuilderPassAllArgs - - -def _get_arg_names(fn: typing.Callable[..., typing.Any]) -> typing.Dict[str, bool]: - arg_names: typing.Dict[str, bool] = {} - - for arg_name, arg_parameter in inspect.signature(fn).parameters.items(): - if arg_parameter.kind not in [ - arg_parameter.KEYWORD_ONLY, - arg_parameter.POSITIONAL_OR_KEYWORD, - arg_parameter.VAR_KEYWORD, - ]: - raise ContextBuilderError("only keyword parameters required for autocontext") - - if arg_parameter.kind == arg_parameter.VAR_KEYWORD: - raise ContextBuilderPassAllArgs() - - arg_names[arg_name] = arg_parameter.default == arg_parameter.empty - - if 'self' in arg_names: - arg_names.pop('self') - - return arg_names - - -def build_kwargs( - fn: typing.Callable[..., typing.Any], - context: typing.Dict[str, typing.Any], -) -> typing.Dict[str, typing.Any]: - try: - arg_names: typing.Dict[str, bool] = _get_arg_names(fn) - except ContextBuilderPassAllArgs: - # Pass all context if kwargs var exists - return context.copy() - - kwargs = {} - for arg_name, is_arg_required in arg_names.items(): - if arg_name not in context and is_arg_required: - raise ContextBuilderError(f"Variable '{arg_name}', is not present in context.") - - if arg_name in context: - kwargs[arg_name] = context[arg_name] - return kwargs - - -def build_context(*context_chain: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: - run_context: typing.Dict[str, typing.Any] = {} - - env_prefix = "CARNIVAL_CTX_" - # Build context from environment variables - for env_name, env_val in os.environ.items(): - if env_name.startswith(env_prefix): - run_context[env_name[len(env_prefix):]] = env_val - - for context_item in context_chain: - for var_name, var_val in context_item.items(): - if isinstance(var_val, context_ref): - try: - run_context[var_name] = run_context[var_val.context_var_name] - except KeyError as e: - raise ContextBuilderError(f"There is no '{var_val.context_var_name}' variable in context") from e - else: - run_context[var_name] = var_val - - return run_context - - -class context_ref: - """ - Ссылка на другую переменную контекста - - Например, так можно передать в переменную c именем `name` `Step` другую переменную контекста с именем `dns_domain` - >>> TestStep( - >>> name=context_ref('dns_domain'), - >>> ), - - """ - def __init__(self, context_var_name: str): - self.context_var_name = context_var_name diff --git a/carnival/exceptions.py b/carnival/exceptions.py index 7350216..024c5c1 100644 --- a/carnival/exceptions.py +++ b/carnival/exceptions.py @@ -6,19 +6,13 @@ class CarnivalException(BaseException): """ -class ContextBuilderError(CarnivalException): - """ - Error when build context for step - """ - - -class ContextBuilderPassAllArgs(CarnivalException): +class GlobalConnectionError(CarnivalException): """ - Signal to send full context to step with **kwargs + Global connection switching error """ -class GlobalConnectionError(CarnivalException): +class StepValidationError(CarnivalException): """ - Global connection switching error + Ошибка валидации шага """ diff --git a/carnival/internal_tasks.py b/carnival/internal_tasks.py index d3e5db4..e94131c 100644 --- a/carnival/internal_tasks.py +++ b/carnival/internal_tasks.py @@ -12,7 +12,6 @@ def run(self) -> None: from carnival.cli import task_types task_list = list(task_types.keys()) - task_list.sort() ralign = 4 if task_list: diff --git a/carnival/step.py b/carnival/step.py index 9a08362..d13c93d 100644 --- a/carnival/step.py +++ b/carnival/step.py @@ -1,7 +1,8 @@ import abc import typing -from carnival.context import build_context, build_kwargs +from carnival.host import AnyHost +from carnival.exceptions import StepValidationError class Step: @@ -27,29 +28,26 @@ class Step: >>> ... """ - def __init__(self, **context: typing.Any): - """ - :param context: Переменные контекста, назначенные при вызове Шага - """ - self.context = context + def __init__(self) -> None: + pass - def run_with_context(self, host_ctx: typing.Dict[str, typing.Any]) -> typing.Callable[[], typing.Any]: + def validate(self, host: AnyHost) -> None: """ - Выполнить шаг + Валидатор шага, запускается перед выполнением + Должен выкидывать .StepValidationError в случае ошибки + + :param host: На котором будет выполнен шаг - :param host_ctx: конекст хоста, (`AnyHost.context`) + :raises StepValidationError: в случае ошибок валидации """ - context = build_context(host_ctx, self.context) - kwargs = build_kwargs(self.run, context) - return lambda: self.run(**kwargs) # type: ignore + raise StepValidationError("Step validation is not implemented") @abc.abstractmethod - @typing.no_type_check - def run(self, **kwargs) -> None: + def run(self, host: AnyHost) -> typing.Any: """ Метод который нужно определить для выполнения комманд - :param kwargs: Автоматические подставляемые переменные контекста, поддерживается `**kwargs` + :param host: Хост для выполнения шага """ raise NotImplementedError diff --git a/carnival/task.py b/carnival/task.py index 15793c5..e39517e 100644 --- a/carnival/task.py +++ b/carnival/task.py @@ -1,11 +1,10 @@ import abc import re -import copy import typing -from carnival import Step, connection +from carnival import Step from carnival.host import AnyHost -from carnival.exceptions import ContextBuilderError +from carnival.exceptions import StepValidationError def _underscore(word: str) -> str: @@ -93,34 +92,34 @@ class StepsTask(abc.ABC, TaskBase): hosts: typing.List[AnyHost] """ - Список хостов - """ - steps: typing.List[Step] - """ - Список шагов в порядке выполнения + Список хостов для выполнения шагов """ - def extend_host_context(self, host: AnyHost) -> typing.Dict[str, typing.Any]: + @abc.abstractmethod + def get_steps(self, host: AnyHost) -> typing.List[Step]: """ - Метод для переопределения контекста хоста - - :param host: хост на котором готовится запуск + Список шагов в порядке выполнения """ - return copy.deepcopy(host.context) + raise NotImplementedError def validate(self) -> typing.List[str]: """ Хук для проверки валидности задачи перед запуском, проверяет примеримость контекста хостов на шагах """ + from carnival.cli import carnival_tasks_module + from carnival.tasks_loader import get_task_full_name + errors: typing.List[str] = [] for host in self.hosts: - for step in self.steps: + for step in self.get_steps(host): try: - step.run_with_context(self.extend_host_context(host=host)) - except ContextBuilderError as ex: - errors.append(f"{self.__class__.__qualname__} -> {step.__class__.__qualname__} on {host}: {ex}") + step.validate(host=host) + except StepValidationError as ex: + task_name = get_task_full_name(carnival_tasks_module, self.__class__) + step_name = step.__class__.__name__ + errors.append(f"{task_name} -> {step_name} on {host}: {ex}") return errors @@ -128,16 +127,13 @@ def run(self) -> None: errors = self.validate() if errors: - print("There is context building errors") + print("There is validation errors") for e in errors: print(f" * {e}") return for host in self.hosts: - host_ctx = self.extend_host_context(host=host) - with connection.SetConnection(host): - for step in self.steps: - step_name = _underscore(step.__class__.__name__) - print(f"💃💃💃 Running {self.get_name()}:{step_name} at {host}") - call_step = step.run_with_context(host_ctx) - call_step() + for step in self.get_steps(host): + step_name = _underscore(step.__class__.__name__) + print(f"💃💃💃 Running {self.get_name()}:{step_name} at {host}") + step.run(host=host) diff --git a/carnival/tasks_loader.py b/carnival/tasks_loader.py index b04e8fd..bfa60ad 100644 --- a/carnival/tasks_loader.py +++ b/carnival/tasks_loader.py @@ -1,12 +1,13 @@ import abc import os import sys -from typing import Any, Dict, Set, Type +import typing +import collections from carnival.task import TaskBase -def task_subclasses(cls: Type[Any]) -> Set[Type[Any]]: +def task_subclasses(cls: typing.Type[typing.Any]) -> typing.Set[typing.Type[typing.Any]]: # Get subclasses of task, which not abstract subclasses = set() @@ -20,7 +21,7 @@ def task_subclasses(cls: Type[Any]) -> Set[Type[Any]]: return subclasses -def get_task_full_name(carnival_tasks_module: str, task_class: Type[TaskBase]) -> str: +def get_task_full_name(carnival_tasks_module: str, task_class: typing.Type[TaskBase]) -> str: task_name = task_class.get_name() task_mod = task_class.module_name @@ -46,8 +47,8 @@ def import_tasks_file(carnival_tasks_module: str, silent: bool) -> None: print(f"Cannot import {carnival_tasks_module}: {ex}", file=sys.stderr) -def get_tasks_from_runtime(carnival_tasks_module: str) -> Dict[str, Type[TaskBase]]: - tasks: Dict[str, Type[TaskBase]] = {} +def get_tasks_from_runtime(carnival_tasks_module: str) -> typing.Dict[str, typing.Type[TaskBase]]: + tasks: typing.Dict[str, typing.Type[TaskBase]] = {} for task_class in task_subclasses(TaskBase): task_full_name = get_task_full_name(carnival_tasks_module, task_class) @@ -56,8 +57,11 @@ def get_tasks_from_runtime(carnival_tasks_module: str) -> Dict[str, Type[TaskBas return tasks -def get_tasks(carnival_tasks_module: str, for_completion: bool = False) -> Dict[str, Type[TaskBase]]: +def get_tasks( + carnival_tasks_module: str, + for_completion: bool = False, +) -> typing.OrderedDict[str, typing.Type[TaskBase]]: sys.path.insert(0, os.getcwd()) from carnival import internal_tasks # noqa import_tasks_file(carnival_tasks_module, silent=for_completion) - return get_tasks_from_runtime(carnival_tasks_module) + return collections.OrderedDict(sorted(get_tasks_from_runtime(carnival_tasks_module).items())) diff --git a/carnival/templates.py b/carnival/templates.py index 4b5e69f..9b476f3 100644 --- a/carnival/templates.py +++ b/carnival/templates.py @@ -1,10 +1,14 @@ import os from typing import Any -from jinja2 import (ChoiceLoader, Environment, FileSystemLoader, PackageLoader, - PrefixLoader) +from jinja2 import ( + ChoiceLoader, + Environment, + FileSystemLoader, + PackageLoader, + PrefixLoader, +) -from carnival import connection from carnival.plugins import discover_plugins """ @@ -24,9 +28,4 @@ def render(template_path: str, **context: Any) -> str: :param context: контекст шаблона """ template = j2_env.get_template(template_path) - return template.render( - conn=connection.conn, - connected_host=connection.host, - host_context=connection, - **context, - ) + return template.render(**context) diff --git a/carnival/utils.py b/carnival/utils.py index 839a92d..ea09059 100644 --- a/carnival/utils.py +++ b/carnival/utils.py @@ -1,16 +1,23 @@ -from typing import Any, Optional, Protocol +import typing +import os +from carnival.host import AnyHost -from carnival import connection +class _Writer(typing.Protocol): + def write(self, __s: str) -> typing.Any: ... -class _Writer(Protocol): - def write(self, __s: str) -> Any: ... - -def log(message: str, file: Optional[_Writer] = None) -> None: - if connection.host is None: - host = "NO CONNECTION" +def log(message: str, host: typing.Optional[AnyHost], file: typing.Optional[_Writer] = None) -> None: + if host is None: + host_part = "NO CONNECTION" else: - host = str(connection.host) + host_part = str(host) + + print(f"💃💃💃 {host_part}> {message}", file=file) + + +def envvar(varname: str) -> str: + if varname not in os.environ: + raise ValueError(f"{varname} is not persent in environment") - print(f"💃💃💃 {host}> {message}", file=file) + return os.environ[varname]