diff --git a/README.rst b/README.rst index 856dd9fc..277b99d2 100644 --- a/README.rst +++ b/README.rst @@ -45,6 +45,7 @@ Features * support of multi user connection; * support of `hooks `_; * community `project samples `_; +* CI support; * user friendly interface. Quickstart diff --git a/doc/api_schema.yaml b/doc/api_schema.yaml index 227bc157..93d7d96a 100755 --- a/doc/api_schema.yaml +++ b/doc/api_schema.yaml @@ -19,9 +19,9 @@ info: url: https://gitlab.com/vstconsulting/polemarch.git Request: - name: Question - url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Ask&issue%5Btitle%5D=Ask%20about%20version%201.3.1 + url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Ask&issue%5Btitle%5D=Ask%20about%20version%201.4.2 - name: Bug report - url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Bug&issue%5Btitle%5D=Bug%20in%20version%201.3.1 + url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Bug&issue%5Btitle%5D=Bug%20in%20version%201.4.2 - name: Feature request url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Feature%20request&issue%5Btitle%5D= x-menu: @@ -54,9 +54,9 @@ info: span_class: fa fa-plug url: /hook x-versions: - application: 1.3.1 - library: 1.3.1 - vstutils: 2.3.1 + application: 1.4.2 + library: 1.4.2 + vstutils: 2.3.10 django: 2.2.1 ansible: 2.8.1 version: v2 @@ -457,7 +457,7 @@ paths: /group/{id}/copy/: post: operationId: group_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -750,7 +750,7 @@ paths: /group/{id}/group/{group_id}/copy/: post: operationId: group_group_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -1357,7 +1357,7 @@ paths: /group/{id}/host/{host_id}/copy/: post: operationId: group_host_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -2761,7 +2761,7 @@ paths: /host/{id}/copy/: post: operationId: host_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -3636,7 +3636,7 @@ paths: /inventory/{id}/copy/: post: operationId: inventory_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -3929,7 +3929,7 @@ paths: /inventory/{id}/group/{group_id}/copy/: post: operationId: inventory_group_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -4536,7 +4536,7 @@ paths: /inventory/{id}/host/{host_id}/copy/: post: operationId: inventory_host_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -5427,7 +5427,7 @@ paths: /project/{id}/copy/: post: operationId: project_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -6430,7 +6430,7 @@ paths: /project/{id}/inventory/{inventory_id}/copy/: post: operationId: project_inventory_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -6740,7 +6740,7 @@ paths: /project/{id}/inventory/{inventory_id}/group/{group_id}/copy/: post: operationId: project_inventory_group_inventory_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -7378,7 +7378,7 @@ paths: /project/{id}/inventory/{inventory_id}/host/{host_id}/copy/: post: operationId: project_inventory_host_inventory_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -9756,7 +9756,7 @@ paths: /team/{id}/copy/: post: operationId: team_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -10147,7 +10147,7 @@ paths: /team/{id}/user/{user_id}/copy/: post: operationId: team_user_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body @@ -10597,7 +10597,7 @@ paths: /user/{id}/copy/: post: operationId: user_copy - description: Copy instance with deps. + description: Endpoint which copy instance with deps. parameters: - name: data in: body diff --git a/polemarch/__init__.py b/polemarch/__init__.py index ae2d8e7c..3f71f8cc 100644 --- a/polemarch/__init__.py +++ b/polemarch/__init__.py @@ -31,6 +31,6 @@ "VST_ROOT_URLCONF": os.getenv("VST_ROOT_URLCONF", 'vstutils.urls'), } -__version__ = "1.4.1" +__version__ = "1.4.2" prepare_environment(**default_settings) diff --git a/polemarch/main/models/utils.py b/polemarch/main/models/utils.py index 8baedde3..9f2533cf 100644 --- a/polemarch/main/models/utils.py +++ b/polemarch/main/models/utils.py @@ -9,6 +9,7 @@ import logging import tempfile import traceback +from pathlib import Path from collections import namedtuple, OrderedDict from subprocess import Popen from functools import reduce @@ -227,8 +228,12 @@ def __generate_arg_file(self, value: Text) -> Tuple[Text, List[tmp_file]]: def __parse_key(self, key: Text, value: Text) -> Tuple[Text, List]: # pylint: disable=unused-argument, if re.match(r"[-]+BEGIN .+ KEY[-]+", value): + # Add new line if not exists and generate tmpfile for private key value + value = value + '/n' if value[-1] != '/n' else value return self.__generate_arg_file(value) - return "{}/{}".format(self.workdir, value), [] + # Return path in project if it's path + path = (Path(self.workdir)/Path(value).expanduser()).resolve() + return str(path), [] def __convert_arg(self, ansible_extra: AnsibleExtra, item: Tuple[Text, Any]) -> Tuple[List, List]: extra_args, files = ansible_extra diff --git a/polemarch/main/repo/_base.py b/polemarch/main/repo/_base.py index e1f84588..814f0b17 100644 --- a/polemarch/main/repo/_base.py +++ b/polemarch/main/repo/_base.py @@ -2,8 +2,8 @@ from __future__ import unicode_literals from typing import Any, Text, Dict, List, Tuple, Union, Iterable, Callable, TypeVar, NoReturn import os -import re import shutil +import pathlib import logging import traceback @@ -128,7 +128,7 @@ def _handle_yaml(self, data: Union[Dict, None]) -> NoReturn: feature_name = 'pm_handle_{}'.format(feature) getattr(self, feature_name, self.pm_handle_unknown)(feature, data) - def _set_tasks_list(self, playbooks_names: Iterable[Text]) -> NoReturn: + def _set_tasks_list(self, playbooks_names: Iterable[pathlib.Path]) -> NoReturn: """ Updates playbooks in project. """ @@ -173,16 +173,16 @@ def _set_project_modules(self) -> None: ModuleClass(path=path, project=project) for path in modules ]) - def _update_tasks(self, files: List[Text]) -> NoReturn: + def _update_tasks(self, files: Iterable[pathlib.Path]) -> NoReturn: ''' Find and update playbooks in project. :param files: list of filenames. ''' - reg = re.compile(self.regex) - playbooks = filter(reg.match, files) - self._set_tasks_list(playbooks) + # reg = re.compile(self.regex) + # playbooks = filter(reg.match, files) + self._set_tasks_list(files) - def _get_files(self, repo: Any = None) -> List: + def _get_files(self, repo: Any = None) -> pathlib.Path: ''' Get all files, where playbooks should be. :param repo: Repo object @@ -191,7 +191,10 @@ def _get_files(self, repo: Any = None) -> List: :rtype: list ''' # pylint: disable=unused-argument - return os.listdir(self.path) + return pathlib.Path(self.path) + + def search_files(self, repo: Any = None, pattern: Text = '**/*') -> Iterable[pathlib.Path]: + return self._get_files(repo).glob(pattern) def _operate(self, operation: Callable, **kwargs) -> Any: return operation(kwargs) @@ -208,7 +211,7 @@ def _make_operations(self, operation: Callable) -> Any: with transaction.atomic(): result = self._operate(operation) self._set_status("OK") - self._update_tasks(self._get_files(result[0])) + self._update_tasks(self.search_files(result[0], '*.yml')) self._set_project_modules() self._handle_yaml(self._load_yaml() or dict()) except Exception as err: @@ -264,7 +267,7 @@ def clone(self) -> Text: for __ in range(attempt): try: repo = self._make_operations(self.make_clone)[0] - return "Received {} files.".format(len(self._get_files(repo))) + return "Received {} files.".format(len(list(self.search_files(repo)))) except: self.delete() raise Exception("Clone didn't perform by {} attempts.".format(attempt)) diff --git a/polemarch/main/repo/vcs.py b/polemarch/main/repo/vcs.py index 38a57d2d..2e542c8e 100644 --- a/polemarch/main/repo/vcs.py +++ b/polemarch/main/repo/vcs.py @@ -1,13 +1,13 @@ # pylint: disable=expression-not-assigned,abstract-method,import-error from __future__ import unicode_literals -from typing import Tuple, Dict, Text, Union, Any +from typing import Tuple, Dict, Text, Union, Any, Iterable import warnings from vstutils.utils import tmp_file_context, raise_context try: import git except: # nocv warnings.warn("Git is not installed or have problems.") -from ._base import _Base, os, logger +from ._base import _Base, os, logger, pathlib ENV_VARS_TYPE = Dict[Text, Union[Text, bool]] # pylint: disable=invalid-name @@ -187,8 +187,21 @@ def _operate(self, operation, **env_vars): env_vars = self._with_key(tmp, env_vars) return super(Git, self)._operate(operation, **env_vars) - def _get_files(self, repo: git.Repo = None): - return dict(repo.index.entries.keys()).keys() + def search_files(self, repo: git.Repo = None, pattern: Text = '**/*') -> Iterable[pathlib.Path]: + recursive = pattern.startswith('**/') + if recursive: + pattern = pattern.replace('**/', '') + for path in dict(repo.index.entries.keys()).keys(): + result = pathlib.Path(path) + if not recursive and result.parent != pathlib.Path('.'): + continue + if result.match(pattern): + yield result + + for sm in repo.submodules: + if recursive or pattern.startswith(sm.name): + for file in self.search_files(sm.module(), pattern.replace(sm.name + '/', '')): + yield pathlib.Path(sm.name)/file def get(self) -> Dict[str, str]: return { diff --git a/polemarch/main/tests/executions.py b/polemarch/main/tests/executions.py index edacce7b..8f4658b1 100644 --- a/polemarch/main/tests/executions.py +++ b/polemarch/main/tests/executions.py @@ -2,6 +2,7 @@ import uuid import tempfile import logging +from pathlib import Path from collections import OrderedDict from datetime import timedelta @@ -540,8 +541,8 @@ def project_workflow(self, repo_type, **kwargs): finally: self.remove_project(**project_data) - def get_file_path(self, name, path): - return "{}/{}".format(path, name) + def get_file_path(self, name: str, path: str) -> Path: + return Path(path).joinpath(*name.split('/')) def generate_playbook(self, path, name='test', count=1, data=test_playbook_content): ''' @@ -564,7 +565,9 @@ def generate_playbook(self, path, name='test', count=1, data=test_playbook_conte _files = ['{}-{}.yml'.format(name, i) for i in range(count or 1)] for filename in _files: file_path = self.get_file_path(filename, path) - with open(file_path, 'w') as playbook: + if not file_path.parent.exists(): + file_path.parent.mkdir(parents=True) + with file_path.open('w') as playbook: playbook.write(data) files.append(filename) return files @@ -1191,7 +1194,7 @@ def test_project_git(self): # Prepare repo self.repo_dir = tempfile.mkdtemp() self.submodule_dir = "{}_submodule".format(self.repo_dir) - self.generate_playbook(self.repo_dir, ['main.yml']) + self.generate_playbook(self.repo_dir, ['main.yml', 'subdir/other.yml'], 2) self.generate_playbook(self.repo_dir, ['.polemarch.yaml'], data=dump(pm_yaml)) self.generate_playbook(self.repo_dir, ['ansible.cfg'], data=test_ansible_cfg) lib_dir = self.submodule_dir @@ -1208,7 +1211,7 @@ def test_project_git(self): 'add', '../{}/.git'.format(self.submodule_dir.split('/')[-1]), 'lib' ) repo.git.submodule('add', '{}/.git'.format(self.submodule_dir), 'sm1') - repo.index.add(["main.yml", ".polemarch.yaml", "ansible.cfg"]) + repo.index.add(["main.yml", "subdir/other.yml", ".polemarch.yaml", "ansible.cfg"]) repo.index.commit("no message") first_revision = repo.head.object.hexsha repo.create_head('new_branch') diff --git a/requirements-doc.txt b/requirements-doc.txt index e18a8cce..01ca3aab 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,2 +1,2 @@ # Docs -vstutils[doc]~=2.3.0 +vstutils[doc]~=2.3.10 diff --git a/requirements-test.txt b/requirements-test.txt index ba3c7c36..900f23c9 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,5 @@ vcrpy-unittest==0.1.6 -coverage<=4.3.4 -mock<=2.0.1 +coverage<=4.5.4 +mock<=3.0.5 fakeldap==0.6.1 tblib==1.3.2 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ba972af2..770a87b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ # Main -vstutils[rpc,ldap,doc,prod]~=2.3.0 -docutils==0.14 +vstutils[rpc,ldap,doc,prod]~=2.3.10 +docutils==0.15.2 markdown2==2.3.8 # Repo types -gitpython==2.1.11 +gitpython==3.0.2 # Hooks requests==2.22.0 diff --git a/setup.py b/setup.py index 418d27e4..54cd2242 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,8 @@ import sys import fnmatch import codecs +import gzip +import shutil # allow setup.py to be run from any path os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) @@ -159,6 +161,9 @@ def minify_static_files(base_dir, files, exclude=None): minified = func(static_file_fd.read(), subfunc) with codecs.open(fext_file, 'w', encoding='utf-8') as static_file_fd: static_file_fd.write(minified) + with open(fext_file, 'rb') as f_in: + with gzip.open("{}.gz".format(fext_file), 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) print('Minfied file {}.'.format(fext_file))