step validation rules, no_validate flag

carnival-org · Nov 24, 2021 · 71fe32f · 71fe32f
1 parent 2f8485e
commit 71fe32f
Show file tree

Hide file tree

Showing 27 changed files with 469 additions and 473 deletions.
diff --git a/carnival/__init__.py b/carnival/__init__.py
@@ -1,7 +1,9 @@
 import sys
 
 from carnival.step import Step
-from carnival.host import SSHHost, LocalHost
+from carnival.hosts.base import Host, Connection, Result
+from carnival.hosts.local import LocalHost
+from carnival.hosts.ssh import SshHost
 from carnival.task import TaskBase, StepsTask
 from carnival import cmd
 from carnival import internal_tasks
@@ -18,7 +20,7 @@
 
 __all__ = [
     'Step',
-    'SSHHost', 'LocalHost',
+    'SshHost', 'LocalHost', 'Host', 'Connection', 'Result',
     'TaskBase', 'StepsTask',
     'cmd',
     'log',

diff --git a/carnival/cli.py b/carnival/cli.py
@@ -37,6 +37,7 @@ def main() -> int:
         >>> Usage: python -m carnival [OPTIONS] {help|test}...
         >>> Options:
         >>> --debug        Turn on debug mode
+        >>> --no_validate  Disable step validation
         >>> --help         Show this message and exit.
     """
     global task_types
@@ -49,15 +50,19 @@ def main() -> int:
 
     @click.command()
     @click.option('--debug', is_flag=True, default=False, help="Turn on debug mode")
+    @click.option('--no_validate', is_flag=True, default=False, help="Disable step validation")
     @click.argument('tasks', required=True, type=click.Choice(list(task_types.keys())), nargs=-1)
-    def cli(debug: bool, tasks: typing.Iterable[str]) -> None:
+    def cli(debug: bool, no_validate: bool, tasks: typing.Iterable[str]) -> None:
         if debug is True:
             print("Debug mode on.")
         else:
             sys.excepthook = except_hook
 
+        if no_validate:
+            print("Step validation disabled")
+
         for task in tasks:
-            task_types[task]().run()
+            task_types[task](no_validate=no_validate).run()
 
     cli(complete_var=complete_var)
     return 0
diff --git a/carnival/cmd/cli.py b/carnival/cmd/cli.py
@@ -1,11 +1,16 @@
-from carnival.host import AnyConnection
-from invoke import Result  # type: ignore
+import typing
 
+from carnival import Connection, Result
 
-def run(c: AnyConnection, command: str, warn: bool = True, hide: bool = False) -> Result:
+
+def run(c: Connection, command: str, warn: bool = True, hide: bool = False, cwd: typing.Optional[str] = None) -> Result:
     """
     Запустить комманду
 
     См <https://docs.pyinvoke.org/en/latest/api/runners.html>
     """
-    return c.run(command, pty=True, warn=warn, hide=hide)
+    return c.run(command, warn=warn, hide=hide, cwd=cwd)
+
+
+def is_cmd_exist(c: Connection, cmd_name: str) -> bool:
+    return run(c, f"which {cmd_name}", hide=True, warn=True).ok
diff --git a/carnival/cmd/fs.py b/carnival/cmd/fs.py
@@ -2,9 +2,7 @@
 import re
 
 from carnival import cmd
-from carnival.host import AnyConnection
-
-from invoke import Result  # type: ignore
+from carnival import Connection, Result
 
 
 def _escape_for_regex(text: str) -> str:
@@ -19,7 +17,7 @@ def _escape_for_regex(text: str) -> str:
     return regex
 
 
-def mkdirs(c: AnyConnection, *dirs: str) -> List[Result]:
+def mkdirs(c: Connection, *dirs: str) -> List[Result]:
     """
     Создать директории
 
@@ -28,7 +26,7 @@ def mkdirs(c: AnyConnection, *dirs: str) -> List[Result]:
     return [cmd.cli.run(c, f"mkdir -p {x}", hide=True) for x in dirs]
 
 
-def is_dir_exists(c: AnyConnection, dir_path: str) -> bool:
+def is_dir_exists(c: Connection, dir_path: str) -> bool:
     """
     Узнать существует ли директория
 
@@ -37,7 +35,7 @@ def is_dir_exists(c: AnyConnection, dir_path: str) -> bool:
     return bool(cmd.cli.run(c, f"test -d {dir_path}", warn=True, hide=True).ok)
 
 
-def is_file_contains(c: AnyConnection, filename: str, text: str, exact: bool = False, escape: bool = True) -> bool:
+def is_file_contains(c: Connection, filename: str, text: str, exact: bool = False, escape: bool = True) -> bool:
     """
     Содержит ли файл текст
 
@@ -53,22 +51,22 @@ def is_file_contains(c: AnyConnection, filename: str, text: str, exact: bool = F
         if exact:
             text = "^{}$".format(text)
     egrep_cmd = 'egrep "{}" "{}"'.format(text, filename)
-    return c.run(egrep_cmd, hide=True, warn=True).ok  # type: ignore
+    return c.run(egrep_cmd, hide=True, warn=True).ok
 
 
-def is_file_exists(c: AnyConnection, path: str) -> bool:
+def is_file_exists(c: Connection, path: str) -> bool:
     """
     Проверить существует ли файл
 
     :param path: путь до файла
     """
 
     cmd = 'test -e "$(echo {})"'.format(path)
-    return c.run(cmd, hide=True, warn=True).ok  # type: ignore
+    return c.run(cmd, hide=True, warn=True).ok
 
 
 def ensure_dir_exists(
-    c: AnyConnection,
+    c: Connection,
     path: str,
     user: Optional[str] = None,
     group: Optional[str] = None,

diff --git a/carnival/cmd/system.py b/carnival/cmd/system.py
@@ -1,10 +1,8 @@
 from carnival import cmd
-from carnival.host import AnyConnection
+from carnival import Connection, Result
 
-from invoke import Result  # type: ignore
 
-
-def set_password(c: AnyConnection, username: str, password: str) -> Result:
+def set_password(c: Connection, username: str, password: str) -> Result:
     """
     Установить пароль пользователю
 
@@ -14,22 +12,22 @@ def set_password(c: AnyConnection, username: str, password: str) -> Result:
     return cmd.cli.run(c, f"echo '{username}:{password}' | chpasswd", hide=True)
 
 
-def get_current_user_name(c: AnyConnection) -> str:
+def get_current_user_name(c: Connection) -> str:
     """
     Получить имя текущего пользователя
     """
     id_res: str = cmd.cli.run(c, "id -u -n", hide=True).stdout
     return id_res.strip()
 
 
-def get_current_user_id(c: AnyConnection) -> int:
+def get_current_user_id(c: Connection) -> int:
     """
     Получить id текущего пользователя
     """
     return int(cmd.cli.run(c, "id -u", hide=True).stdout.strip())
 
 
-def is_current_user_root(c: AnyConnection) -> bool:
+def is_current_user_root(c: Connection) -> bool:
     """
     Проверить что текущий пользователь - `root`
     """

diff --git a/carnival/cmd/transfer.py b/carnival/cmd/transfer.py
@@ -1,36 +1,66 @@
 from io import BytesIO
-from typing import Any, Iterable
+from typing import Any
 
-from carnival.host import AnyConnection
+from carnival import Connection, Result, SshHost, Host
+from carnival.hosts.local import localhost_connection
 from carnival.templates import render
-from fabric.transfer import Result, Transfer  # type:ignore
-from patchwork import transfers  # type:ignore
+
+
+from fabric.transfer import Transfer  # type:ignore
+from paramiko.config import SSH_PORT
+
+
+def _get_ssh_host_addess_for_ssh_cli(host: SshHost) -> str:
+    """
+    Return `user@addr` if user given, else `addr`
+    """
+
+    if host.ssh_user:
+        return f"{host.ssh_user}@{host.addr}"
+    return host.addr
 
 
 def rsync(
-    c: AnyConnection,
-    source: str, target: str,
-    exclude: Iterable[str] = (),
-    delete: bool = False, strict_host_keys: bool = True,
-    rsync_opts: str = "--progress -pthrvz",
-    ssh_opts: str = ''
+    host: Host,
+    source: str,
+    target: str,
+
+    rsync_opts: str = "--progress -pthrvz --timeout=60",
+    ssh_opts: str = '',
+    rsync_command: str = "rsync",
+    hide: bool = False,
 ) -> Result:
     """
-    <https://fabric-patchwork.readthedocs.io/en/latest/api/transfers.html#patchwork.transfers.rsync>
+    Залить папку с локального диска на сервер по rsync
+    :param host: сервер
+    :param source: локальный путь до папки
+    :param target: путь куда нужно залить
+    :param rsync_opts: параметры команды rsync
+    :param ssh_opts: параметры ssh
+    :param rsync_command: путь до rsync
+    :param hide: скрыть результаты выполнения
     """
-    return transfers.rsync(
-        c=c,
-        source=source,
-        target=target,
-        exclude=exclude,
-        delete=delete,
-        strict_host_keys=strict_host_keys,
-        rsync_opts=rsync_opts,
-        ssh_opts=ssh_opts,
-    )
-
-
-def get(c: AnyConnection, remote: str, local: str, preserve_mode: bool = True) -> Result:
+
+    assert isinstance(host, SshHost)  # TODO: Think about remove this
+
+    if host.ssh_port != SSH_PORT:
+        ssh_opts = f"-p {host.ssh_port} {ssh_opts}"
+
+    if host.ssh_gateway is not None:
+        if host.ssh_gateway.ssh_gateway is not None:
+            raise ValueError("gateway for gateway s not supported for rsync, please use .ssh/config")
+        ssh_opts = f"-J {_get_ssh_host_addess_for_ssh_cli(host.ssh_gateway)}:{host.ssh_gateway.ssh_port}"
+
+    ssh_opts = ssh_opts.strip()
+    if ssh_opts:
+        ssh_opts = f'-e "ssh {ssh_opts.strip()}"'
+
+    command = f'{rsync_command} {rsync_opts} {ssh_opts} {source} {_get_ssh_host_addess_for_ssh_cli(host)}:{target}'
+
+    return localhost_connection.run(command, hide=hide)
+
+
+def get(c: Connection, remote: str, local: str, preserve_mode: bool = True) -> None:
     """
     Скачать файл с сервера
     <http://docs.fabfile.org/en/2.5/api/transfer.html#fabric.transfer.Transfer.get>
@@ -39,11 +69,12 @@ def get(c: AnyConnection, remote: str, local: str, preserve_mode: bool = True) -
     :param local: путь куда сохранить файл
     :param preserve_mode: сохранить права
     """
-    t = Transfer(c)
-    return t.get(remote=remote, local=local, preserve_mode=preserve_mode)
+    # TODO: c._c ;(
+    t = Transfer(c._c)  # type: ignore
+    t.get(remote=remote, local=local, preserve_mode=preserve_mode)
 
 
-def put(c: AnyConnection, local: str, remote: str, preserve_mode: bool = True) -> Result:
+def put(c: Connection, local: str, remote: str, preserve_mode: bool = True) -> None:
     """
     Закачать файл на сервер
     <http://docs.fabfile.org/en/2.5/api/transfer.html#fabric.transfer.Transfer.put>
@@ -52,11 +83,12 @@ def put(c: AnyConnection, local: str, remote: str, preserve_mode: bool = True) -
     :param remote: путь куда сохранить на сервере
     :param preserve_mode: сохранить права
     """
-    t = Transfer(c)
-    return t.put(local=local, remote=remote, preserve_mode=preserve_mode)
+    # TODO: c._c ;(
+    t = Transfer(c._c)  # type: ignore
+    t.put(local=local, remote=remote, preserve_mode=preserve_mode)
 
 
-def put_template(c: AnyConnection, template_path: str, remote: str, **context: Any) -> Result:
+def put_template(c: Connection, template_path: str, remote: str, **context: Any) -> None:
     """
     Отрендерить файл с помощью jinja-шаблонов и закачать на сервер
     См раздел templates.
@@ -68,5 +100,6 @@ def put_template(c: AnyConnection, template_path: str, remote: str, **context: A
     :param context: контекс для рендеринга jinja2
     """
     filestr = render(template_path=template_path, **context)
-    t = Transfer(c)
-    return t.put(local=BytesIO(filestr.encode()), remote=remote, preserve_mode=False)
+    # TODO: c._c ;(
+    t = Transfer(c._c)  # type: ignore
+    t.put(local=BytesIO(filestr.encode()), remote=remote, preserve_mode=False)