From 1c16d98fe793218199b816e019acdcee92b153d4 Mon Sep 17 00:00:00 2001
From: Sergey Klyuykov <onegreyonewhite@mail.ru>
Date: Thu, 22 Aug 2019 11:12:06 +1000
Subject: [PATCH 1/4] Compress *.js and *.css via gzip on build and add files
 for serving.

---
 doc/api_schema.yaml                | 38 +++++++++++++++---------------
 polemarch/__init__.py              |  2 +-
 polemarch/main/repo/_base.py       | 23 ++++++++++--------
 polemarch/main/repo/vcs.py         | 21 +++++++++++++----
 polemarch/main/tests/executions.py | 13 ++++++----
 requirements-doc.txt               |  2 +-
 requirements.txt                   |  2 +-
 7 files changed, 60 insertions(+), 41 deletions(-)

diff --git a/doc/api_schema.yaml b/doc/api_schema.yaml
index 227bc157..93d7d96a 100755
--- a/doc/api_schema.yaml
+++ b/doc/api_schema.yaml
@@ -19,9 +19,9 @@ info:
       url: https://gitlab.com/vstconsulting/polemarch.git
     Request:
       - name: Question
-        url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Ask&issue%5Btitle%5D=Ask%20about%20version%201.3.1
+        url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Ask&issue%5Btitle%5D=Ask%20about%20version%201.4.2
       - name: Bug report
-        url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Bug&issue%5Btitle%5D=Bug%20in%20version%201.3.1
+        url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Bug&issue%5Btitle%5D=Bug%20in%20version%201.4.2
       - name: Feature request
         url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Feature%20request&issue%5Btitle%5D=
   x-menu:
@@ -54,9 +54,9 @@ info:
           span_class: fa fa-plug
           url: /hook
   x-versions:
-    application: 1.3.1
-    library: 1.3.1
-    vstutils: 2.3.1
+    application: 1.4.2
+    library: 1.4.2
+    vstutils: 2.3.10
     django: 2.2.1
     ansible: 2.8.1
   version: v2
@@ -457,7 +457,7 @@ paths:
   /group/{id}/copy/:
     post:
       operationId: group_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -750,7 +750,7 @@ paths:
   /group/{id}/group/{group_id}/copy/:
     post:
       operationId: group_group_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -1357,7 +1357,7 @@ paths:
   /group/{id}/host/{host_id}/copy/:
     post:
       operationId: group_host_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -2761,7 +2761,7 @@ paths:
   /host/{id}/copy/:
     post:
       operationId: host_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -3636,7 +3636,7 @@ paths:
   /inventory/{id}/copy/:
     post:
       operationId: inventory_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -3929,7 +3929,7 @@ paths:
   /inventory/{id}/group/{group_id}/copy/:
     post:
       operationId: inventory_group_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -4536,7 +4536,7 @@ paths:
   /inventory/{id}/host/{host_id}/copy/:
     post:
       operationId: inventory_host_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -5427,7 +5427,7 @@ paths:
   /project/{id}/copy/:
     post:
       operationId: project_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -6430,7 +6430,7 @@ paths:
   /project/{id}/inventory/{inventory_id}/copy/:
     post:
       operationId: project_inventory_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -6740,7 +6740,7 @@ paths:
   /project/{id}/inventory/{inventory_id}/group/{group_id}/copy/:
     post:
       operationId: project_inventory_group_inventory_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -7378,7 +7378,7 @@ paths:
   /project/{id}/inventory/{inventory_id}/host/{host_id}/copy/:
     post:
       operationId: project_inventory_host_inventory_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -9756,7 +9756,7 @@ paths:
   /team/{id}/copy/:
     post:
       operationId: team_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -10147,7 +10147,7 @@ paths:
   /team/{id}/user/{user_id}/copy/:
     post:
       operationId: team_user_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
@@ -10597,7 +10597,7 @@ paths:
   /user/{id}/copy/:
     post:
       operationId: user_copy
-      description: Copy instance with deps.
+      description: Endpoint which copy instance with deps.
       parameters:
         - name: data
           in: body
diff --git a/polemarch/__init__.py b/polemarch/__init__.py
index ae2d8e7c..3f71f8cc 100644
--- a/polemarch/__init__.py
+++ b/polemarch/__init__.py
@@ -31,6 +31,6 @@
     "VST_ROOT_URLCONF": os.getenv("VST_ROOT_URLCONF", 'vstutils.urls'),
 }
 
-__version__ = "1.4.1"
+__version__ = "1.4.2"
 
 prepare_environment(**default_settings)
diff --git a/polemarch/main/repo/_base.py b/polemarch/main/repo/_base.py
index e1f84588..814f0b17 100644
--- a/polemarch/main/repo/_base.py
+++ b/polemarch/main/repo/_base.py
@@ -2,8 +2,8 @@
 from __future__ import unicode_literals
 from typing import Any, Text, Dict, List, Tuple, Union, Iterable, Callable, TypeVar, NoReturn
 import os
-import re
 import shutil
+import pathlib
 import logging
 import traceback
 
@@ -128,7 +128,7 @@ def _handle_yaml(self, data: Union[Dict, None]) -> NoReturn:
             feature_name = 'pm_handle_{}'.format(feature)
             getattr(self, feature_name, self.pm_handle_unknown)(feature, data)
 
-    def _set_tasks_list(self, playbooks_names: Iterable[Text]) -> NoReturn:
+    def _set_tasks_list(self, playbooks_names: Iterable[pathlib.Path]) -> NoReturn:
         """
         Updates playbooks in project.
         """
@@ -173,16 +173,16 @@ def _set_project_modules(self) -> None:
             ModuleClass(path=path, project=project) for path in modules
         ])
 
-    def _update_tasks(self, files: List[Text]) -> NoReturn:
+    def _update_tasks(self, files: Iterable[pathlib.Path]) -> NoReturn:
         '''
         Find and update playbooks in project.
         :param files: list of filenames.
         '''
-        reg = re.compile(self.regex)
-        playbooks = filter(reg.match, files)
-        self._set_tasks_list(playbooks)
+        # reg = re.compile(self.regex)
+        # playbooks = filter(reg.match, files)
+        self._set_tasks_list(files)
 
-    def _get_files(self, repo: Any = None) -> List:
+    def _get_files(self, repo: Any = None) -> pathlib.Path:
         '''
         Get all files, where playbooks should be.
         :param repo: Repo object
@@ -191,7 +191,10 @@ def _get_files(self, repo: Any = None) -> List:
         :rtype: list
         '''
         # pylint: disable=unused-argument
-        return os.listdir(self.path)
+        return pathlib.Path(self.path)
+
+    def search_files(self, repo: Any = None, pattern: Text = '**/*') -> Iterable[pathlib.Path]:
+        return self._get_files(repo).glob(pattern)
 
     def _operate(self, operation: Callable, **kwargs) -> Any:
         return operation(kwargs)
@@ -208,7 +211,7 @@ def _make_operations(self, operation: Callable) -> Any:
             with transaction.atomic():
                 result = self._operate(operation)
                 self._set_status("OK")
-                self._update_tasks(self._get_files(result[0]))
+                self._update_tasks(self.search_files(result[0], '*.yml'))
                 self._set_project_modules()
                 self._handle_yaml(self._load_yaml() or dict())
         except Exception as err:
@@ -264,7 +267,7 @@ def clone(self) -> Text:
         for __ in range(attempt):
             try:
                 repo = self._make_operations(self.make_clone)[0]
-                return "Received {} files.".format(len(self._get_files(repo)))
+                return "Received {} files.".format(len(list(self.search_files(repo))))
             except:
                 self.delete()
         raise Exception("Clone didn't perform by {} attempts.".format(attempt))
diff --git a/polemarch/main/repo/vcs.py b/polemarch/main/repo/vcs.py
index 38a57d2d..2e542c8e 100644
--- a/polemarch/main/repo/vcs.py
+++ b/polemarch/main/repo/vcs.py
@@ -1,13 +1,13 @@
 # pylint: disable=expression-not-assigned,abstract-method,import-error
 from __future__ import unicode_literals
-from typing import Tuple, Dict, Text, Union, Any
+from typing import Tuple, Dict, Text, Union, Any, Iterable
 import warnings
 from vstutils.utils import tmp_file_context, raise_context
 try:
     import git
 except:  # nocv
     warnings.warn("Git is not installed or have problems.")
-from ._base import _Base, os, logger
+from ._base import _Base, os, logger, pathlib
 
 ENV_VARS_TYPE =  Dict[Text, Union[Text, bool]]  # pylint: disable=invalid-name
 
@@ -187,8 +187,21 @@ def _operate(self, operation, **env_vars):
                 env_vars = self._with_key(tmp, env_vars)
             return super(Git, self)._operate(operation, **env_vars)
 
-    def _get_files(self, repo: git.Repo = None):
-        return dict(repo.index.entries.keys()).keys()
+    def search_files(self, repo: git.Repo = None, pattern: Text = '**/*') -> Iterable[pathlib.Path]:
+        recursive = pattern.startswith('**/')
+        if recursive:
+            pattern = pattern.replace('**/', '')
+        for path in dict(repo.index.entries.keys()).keys():
+            result = pathlib.Path(path)
+            if not recursive and result.parent != pathlib.Path('.'):
+                continue
+            if result.match(pattern):
+                yield result
+
+        for sm in repo.submodules:
+            if recursive or pattern.startswith(sm.name):
+                for file in self.search_files(sm.module(), pattern.replace(sm.name + '/', '')):
+                    yield pathlib.Path(sm.name)/file
 
     def get(self) -> Dict[str, str]:
         return {
diff --git a/polemarch/main/tests/executions.py b/polemarch/main/tests/executions.py
index edacce7b..8f4658b1 100644
--- a/polemarch/main/tests/executions.py
+++ b/polemarch/main/tests/executions.py
@@ -2,6 +2,7 @@
 import uuid
 import tempfile
 import logging
+from pathlib import Path
 from collections import OrderedDict
 from datetime import timedelta
 
@@ -540,8 +541,8 @@ def project_workflow(self, repo_type, **kwargs):
         finally:
             self.remove_project(**project_data)
 
-    def get_file_path(self, name, path):
-        return "{}/{}".format(path, name)
+    def get_file_path(self, name: str, path: str) -> Path:
+        return Path(path).joinpath(*name.split('/'))
 
     def generate_playbook(self, path, name='test', count=1, data=test_playbook_content):
         '''
@@ -564,7 +565,9 @@ def generate_playbook(self, path, name='test', count=1, data=test_playbook_conte
             _files = ['{}-{}.yml'.format(name, i) for i in range(count or 1)]
         for filename in _files:
             file_path = self.get_file_path(filename, path)
-            with open(file_path, 'w') as playbook:
+            if not file_path.parent.exists():
+                file_path.parent.mkdir(parents=True)
+            with file_path.open('w') as playbook:
                 playbook.write(data)
             files.append(filename)
         return files
@@ -1191,7 +1194,7 @@ def test_project_git(self):
         # Prepare repo
         self.repo_dir = tempfile.mkdtemp()
         self.submodule_dir = "{}_submodule".format(self.repo_dir)
-        self.generate_playbook(self.repo_dir, ['main.yml'])
+        self.generate_playbook(self.repo_dir, ['main.yml', 'subdir/other.yml'], 2)
         self.generate_playbook(self.repo_dir, ['.polemarch.yaml'], data=dump(pm_yaml))
         self.generate_playbook(self.repo_dir, ['ansible.cfg'], data=test_ansible_cfg)
         lib_dir = self.submodule_dir
@@ -1208,7 +1211,7 @@ def test_project_git(self):
             'add', '../{}/.git'.format(self.submodule_dir.split('/')[-1]), 'lib'
         )
         repo.git.submodule('add', '{}/.git'.format(self.submodule_dir), 'sm1')
-        repo.index.add(["main.yml", ".polemarch.yaml", "ansible.cfg"])
+        repo.index.add(["main.yml", "subdir/other.yml", ".polemarch.yaml", "ansible.cfg"])
         repo.index.commit("no message")
         first_revision = repo.head.object.hexsha
         repo.create_head('new_branch')
diff --git a/requirements-doc.txt b/requirements-doc.txt
index e18a8cce..01ca3aab 100644
--- a/requirements-doc.txt
+++ b/requirements-doc.txt
@@ -1,2 +1,2 @@
 # Docs
-vstutils[doc]~=2.3.0
+vstutils[doc]~=2.3.10
diff --git a/requirements.txt b/requirements.txt
index ba972af2..e1b64291 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 # Main
-vstutils[rpc,ldap,doc,prod]~=2.3.0
+vstutils[rpc,ldap,doc,prod]~=2.3.10
 docutils==0.14
 markdown2==2.3.8
 

From 124ab39c5f7054f642c45eeacb13a5ac77363ce2 Mon Sep 17 00:00:00 2001
From: Sergey Klyuykov <onegreyonewhite@mail.ru>
Date: Thu, 22 Aug 2019 14:23:01 +1000
Subject: [PATCH 2/4] Compress *.js and *.css via gzip on build and add files
 for serving.

---
 README.rst            | 1 +
 requirements-test.txt | 4 ++--
 requirements.txt      | 4 ++--
 setup.py              | 9 ++++++++-
 4 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/README.rst b/README.rst
index 856dd9fc..277b99d2 100644
--- a/README.rst
+++ b/README.rst
@@ -45,6 +45,7 @@ Features
 * support of multi user connection;
 * support of `hooks <https://about.polemarch.org/en/latest/gui.html#hooks>`_;
 * community `project samples <https://gitlab.com/vstconsulting/polemarch-community-repos>`_;
+* CI support;
 * user friendly interface.
 
 Quickstart
diff --git a/requirements-test.txt b/requirements-test.txt
index ba3c7c36..900f23c9 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -1,5 +1,5 @@
 vcrpy-unittest==0.1.6
-coverage<=4.3.4
-mock<=2.0.1
+coverage<=4.5.4
+mock<=3.0.5
 fakeldap==0.6.1
 tblib==1.3.2
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index e1b64291..770a87b9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,10 @@
 # Main
 vstutils[rpc,ldap,doc,prod]~=2.3.10
-docutils==0.14
+docutils==0.15.2
 markdown2==2.3.8
 
 # Repo types
-gitpython==2.1.11
+gitpython==3.0.2
 
 # Hooks
 requests==2.22.0
diff --git a/setup.py b/setup.py
index 418d27e4..7e4ec2ca 100644
--- a/setup.py
+++ b/setup.py
@@ -4,6 +4,8 @@
 import sys
 import fnmatch
 import codecs
+import gzip
+import shutil
 
 # allow setup.py to be run from any path
 os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))
@@ -111,7 +113,9 @@ def make_extensions(extensions_list, packages):
     ext_count = len(ext_modules)
     nthreads = ext_count if ext_count < 10 else 10
 
-    language_level = 3
+    language_level = 2
+    if 'bdist_wheel' in sys.argv and sys.version_info.major == 3:
+        language_level = 3
     if is_help:
         pass
     elif has_cython and ('compile' in sys.argv or 'bdist_wheel' in sys.argv):
@@ -159,6 +163,9 @@ def minify_static_files(base_dir, files, exclude=None):
                     minified = func(static_file_fd.read(), subfunc)
                 with codecs.open(fext_file, 'w', encoding='utf-8') as static_file_fd:
                     static_file_fd.write(minified)
+                with open(fext_file, 'rb') as f_in:
+                    with gzip.open("{}.gz".format(fext_file), 'wb') as f_out:
+                        shutil.copyfileobj(f_in, f_out)
                 print('Minfied file {}.'.format(fext_file))
 
 

From 60429afaa566de2100f011cd7e48c1de92472ea9 Mon Sep 17 00:00:00 2001
From: Sergey Klyuykov <onegreyonewhite@mail.ru>
Date: Thu, 22 Aug 2019 14:30:26 +1000
Subject: [PATCH 3/4] Fix language level on build

---
 setup.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 7e4ec2ca..54cd2242 100644
--- a/setup.py
+++ b/setup.py
@@ -113,9 +113,7 @@ def make_extensions(extensions_list, packages):
     ext_count = len(ext_modules)
     nthreads = ext_count if ext_count < 10 else 10
 
-    language_level = 2
-    if 'bdist_wheel' in sys.argv and sys.version_info.major == 3:
-        language_level = 3
+    language_level = 3
     if is_help:
         pass
     elif has_cython and ('compile' in sys.argv or 'bdist_wheel' in sys.argv):

From 937370030b0ee0709d462e20805f4e5be5f5179c Mon Sep 17 00:00:00 2001
From: Sergey Klyuykov <onegreyonewhite@mail.ru>
Date: Thu, 22 Aug 2019 15:49:26 +1000
Subject: [PATCH 4/4] Fix problem with private-key execution via API.

SSH wants that private-key has new line always in file.

Closes vstconsulting/polemarch#87
---
 polemarch/main/models/utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/polemarch/main/models/utils.py b/polemarch/main/models/utils.py
index 8baedde3..9f2533cf 100644
--- a/polemarch/main/models/utils.py
+++ b/polemarch/main/models/utils.py
@@ -9,6 +9,7 @@
 import logging
 import tempfile
 import traceback
+from pathlib import Path
 from collections import namedtuple, OrderedDict
 from subprocess import Popen
 from functools import reduce
@@ -227,8 +228,12 @@ def __generate_arg_file(self, value: Text) -> Tuple[Text, List[tmp_file]]:
     def __parse_key(self, key: Text, value: Text) -> Tuple[Text, List]:
         # pylint: disable=unused-argument,
         if re.match(r"[-]+BEGIN .+ KEY[-]+", value):
+            # Add new line if not exists and generate tmpfile for private key value
+            value = value + '/n' if value[-1] != '/n' else value
             return self.__generate_arg_file(value)
-        return "{}/{}".format(self.workdir, value), []
+        # Return path in project if it's path
+        path = (Path(self.workdir)/Path(value).expanduser()).resolve()
+        return str(path), []
 
     def __convert_arg(self, ansible_extra: AnsibleExtra, item: Tuple[Text, Any]) -> Tuple[List, List]:
         extra_args, files = ansible_extra