Merge branch 'issue_162_parse_ansible_cfg' into 'master'

Resolve "Ansible modules from project parsing on sync"

*  Load ansible configuration with `ansible-config`
*  Check modules in project dir and load them with all data
*  Refactoring ansible execution classes
*  Fix pylint checks
*  Optimize project modules search and import

See merge request polemarch/ce!94

.gitlab-ci.yml

@@ -34,17 +34,9 @@ stages:
       - issue_building
     variables:
       - $CI_COMMIT_MESSAGE =~ /Merge branch/
-  only:
     changes:
-      - polemarch/api/*
-      - polemarch/main/*
-      - tests.py
-      - setup.py
-      - setup.cfg
-      - MANIFEST.in
-      - requirements*
-      - tox.ini
-      - .giltab-ci.yml
+      - polemarch/static/*
+      - polemarch/templates/*
   retry: 2
 
 .pack_tamplate: &packing-test
@@ -87,20 +79,9 @@ code_style:
       - issue_building
     variables:
       - $CI_COMMIT_MESSAGE =~ /Merge branch/
-  only:
-    refs:
-      - /^.{0,}issue_.*$/
-      - developer
-      - master
     changes:
-      - polemarch/api/*
-      - polemarch/main/*
-      - tests.py
-      - requirements*
-      - tox.ini
-      - .giltab-ci.yml
-      - .pep8
-      - .pylintrc
+      - polemarch/static/*
+      - polemarch/templates/*
   retry: 2
 
 py27-install:

polemarch/main/models/projects.py

@@ -23,7 +23,7 @@
 from ..exceptions import PMException
 from .base import ManyToManyFieldACL, BQuerySet, BModel
 from .hooks import Hook
-from ..utils import AnsibleModules, SubCacheInterface
+from ..utils import AnsibleModules, AnsibleConfigParser, SubCacheInterface
 
 
 logger = logging.getLogger("polemarch")
@@ -36,6 +36,7 @@ class ProjectQuerySet(AbstractVarsQuerySet):
 
 
 class Project(AbstractModel):
+    # pylint: disable=too-many-public-methods
     PROJECTS_DIR = getattr(settings, "PROJECTS_DIR")
     objects       = ProjectQuerySet.as_manager()
     repo_handlers = objects._queryset_class.repo_handlers
@@ -138,6 +139,15 @@ def type(self):
         except self.variables.model.DoesNotExist:  # nocv
             return 'MANUAL'
 
+    @property
+    def config(self):
+        return self.get_ansible_config_parser().get_data()
+
+    def get_ansible_config_parser(self):
+        if not hasattr(self, 'config_parser'):
+            self.config_parser = AnsibleConfigParser(self.path)
+        return self.config_parser
+
     def get_yaml_subcache(self, suffix=''):
         return SubCacheInterface(''.join(['project', str(self.id), suffix]))
 
@@ -332,7 +342,11 @@ def _load_data(self, data):
         return load(data, Loader=Loader) if data and data != '{}' else {}
 
     def _get_module_data_from_cli(self):
-        modules = AnsibleModules(detailed=True)
+        path = None
+        if self.project:
+            path = self.project.config.get('DEFAULT_MODULE_PATH', [])
+            path = list(filter(lambda p: self.project.path in p, path))
+        modules = AnsibleModules(detailed=True, paths=path)
         module_list = modules.get(self.path)
         module = module_list[0] if module_list else None
         if module:

polemarch/main/models/utils.py

@@ -315,10 +315,5 @@ def __init__(self, target, *pargs, **kwargs):
         super(AnsibleModule, self).__init__(*pargs, **kwargs)
         self.ansible_ref['module-name'] = {'type': 'string'}
 
-    def get_inventory_arg(self, target, extra_args):
-        if self.inventory_object is not None and not self.inventory_object.is_file:
-            return [self.inventory_object.file]
-        return super(AnsibleModule, self).get_inventory_arg(target, extra_args)
-
     def execute(self, group, *args, **extra_args):
         return super(AnsibleModule, self).execute(group, *args, **extra_args)

polemarch/main/repo/_base.py

@@ -10,6 +10,7 @@
 from six.moves.urllib.request import urlretrieve
 from django.db import transaction
 from vstutils.utils import raise_context
+from ..utils import AnsibleModules
 
 logger = logging.getLogger("polemarch")
 
@@ -38,6 +39,12 @@ def _load_yaml(self):
         self.proj.get_yaml_subcache().clear()
         return self.proj.get_yaml()
 
+    def _path_exists(self, path):
+        return os.path.exists(path)
+
+    def _dir_exists(self, path_dir):
+        return self._path_exists(path_dir) and os.path.isdir(path_dir)
+
     def message(self, message, level='debug'):
         getattr(logger, level.lower(), logger.debug)(
             'Syncing project [{}] - {}'.format(self.proj.id, message)
@@ -138,6 +145,33 @@ def _set_tasks_list(self, playbooks_names):
         )
         PlaybookModel.objects.bulk_create(playbook_objects) if playbook_objects else None
 
+    def __get_project_modules(self, module_path):
+        valid_paths = tuple(filter(self._dir_exists, module_path))
+        if not valid_paths:
+            return []
+        modules = AnsibleModules(detailed=False, paths=valid_paths)
+        modules.clear_cache()
+        modules_list = modules.all()
+        modules_list.sort()
+        return modules_list
+
+    @raise_context()
+    def _set_project_modules(self):
+        '''
+        Update project modules
+        '''
+        # pylint: disable=invalid-name
+        project = self.proj
+        project.get_ansible_config_parser().clear_cache()
+        project.modules.all().delete()
+        ModuleClass = self.proj.modules.model
+        paths = project.config.get('DEFAULT_MODULE_PATH', [])
+        paths = filter(lambda mp: project.path in mp, paths)
+        modules = self.__get_project_modules(paths)
+        ModuleClass.objects.bulk_create([
+            ModuleClass(path=path, project=project) for path in modules
+        ])
+
     def _update_tasks(self, files):
         '''
         Find and update playbooks in project.
@@ -176,6 +210,7 @@ def _make_operations(self, operation):
                 result = self._operate(operation)
                 self._set_status("OK")
                 self._update_tasks(self._get_files(result[0]))
+                self._set_project_modules()
                 self._handle_yaml(self._load_yaml() or dict())
         except Exception as err:
             logger.debug(traceback.format_exc())

polemarch/main/tests/executions.py

@@ -11,6 +11,36 @@
 from ..tasks import ScheduledTask
 
 
+test_ansible_cfg = '''
+[defaults]
+library = lib:lib2
+'''
+
+test_module_content = '''
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['stableinterface'],
+                    'supported_by': 'core'}
+
+DOCUMENTATION = """
+---
+module: test_module
+short_description: Test module
+description:
+  - Test module for check functionality.
+version_added: "0.0.2"
+options:
+  name:
+    description:
+      - Test description.
+author: "Sergey Klyuykov"
+notes:
+   - Test module
+"""
+'''
+
 test_playbook_content = '''
 ---
 - hosts: all
@@ -593,6 +623,16 @@ def wip_git(self, project_data):
             playbook = extra_view_data['playbooks'][playbook_name]
             for required_field in ['title', 'help']:
                 self.assertIn(required_field, playbook.keys())
+        # Check ansible.cfg
+        proj = self.get_model_filter('Project', pk=project_data['id']).get()
+        proj_config = getattr(proj, 'config', None)
+        self.assertTrue(proj_config is not None)
+        # Check modules
+        proj_modules = proj.module.filter(path__startswith='polemarch.project.')
+        self.assertEqual(proj_modules.count(), 1)
+        proj_module = proj_modules.first()
+        self.assertEqual(proj_module.name, 'test_module')
+        self.assertEqual(proj_module.data['short_description'], 'Test module')
         return dict(playbook_count=len(self.revisions), execute=True)
 
     def make_test_templates(self, project_data):
@@ -993,8 +1033,15 @@ def test_project_git(self):
         self.repo_dir = tempfile.mkdtemp()
         self.generate_playbook(self.repo_dir, ['main.yml'])
         self.generate_playbook(self.repo_dir, ['.polemarch.yaml'], data=dump(pm_yaml))
+        self.generate_playbook(self.repo_dir, ['ansible.cfg'], data=test_ansible_cfg)
+        lib_dir = self.repo_dir + '/lib'
+        if not os.path.exists(lib_dir):
+            os.makedirs(lib_dir)
+            self.generate_playbook(lib_dir, ['test_module.py'], data=test_module_content)
         repo = git.Repo.init(self.repo_dir)
-        repo.index.add(["main.yml", ".polemarch.yaml"])
+        repo.index.add([
+            "main.yml", ".polemarch.yaml", "ansible.cfg", lib_dir + '/test_module.py'
+        ])
         repo.index.commit("no message")
         first_revision = repo.head.object.hexsha
         repo.create_head('new_branch')

polemarch/main/utils.py

@@ -91,7 +91,7 @@ def _enqueue_output(self, out, queue):
         try:
             line = out.readline()
             while len(line):
-                queue.put_nowait(line)
+                queue.put(line)
                 line = out.readline()
         finally:
             out.close()
@@ -253,13 +253,19 @@ class AnsibleCache(SubCacheInterface):
 
 
 class PMAnsible(PMObject):
-    __slots__ = ()
+    __slots__ = 'execute_path', 'cache',
     # Json regex
     _regex = re.compile(r"([\{\[][^\w\d\.].*[\}\]]$)", re.MULTILINE)
     ref_name = 'object'
+    cache_timeout = 86400*7
+
+    def __init__(self, execute_path='/tmp/'):
+        self.execute_path = execute_path
 
     def get_ansible_cache(self):
-        return AnsibleCache(self.get_ref(cache=True))
+        if not hasattr(self, 'cache'):
+            self.cache = AnsibleCache(self.get_ref(cache=True), self.cache_timeout)
+        return self.cache
 
     def _get_only_json(self, output):
         return json.loads(self._regex.findall(output)[0])
@@ -280,7 +286,7 @@ def get_data(self):
             with open(os.devnull, 'wb') as DEVNULL:
                 cmd = CmdExecutor(stderr=DEVNULL)
                 cmd_command = self.get_args()
-                cmd.execute(cmd_command, '/tmp/')
+                cmd.execute(cmd_command, self.execute_path)
             result = self._get_only_json(cmd.output)
             cache.set(result)
         return result
@@ -304,6 +310,7 @@ class AnsibleArgumentsReference(PMAnsible):
     ]
 
     def __init__(self):
+        super(AnsibleArgumentsReference, self).__init__()
         self.raw_dict = self._extract_from_cli()
 
     def is_valid_value(self, command, argument, value):
@@ -352,20 +359,24 @@ def _extract_from_cli(self):
 
 
 class AnsibleModules(PMAnsible):
-    __slots__ = 'detailed', 'key'
+    __slots__ = 'detailed', 'key', 'module_paths'
     ref_name = 'modules'
 
-    def __init__(self, detailed=False):
+    def __init__(self, detailed=False, paths=None):
         super(AnsibleModules, self).__init__()
         self.detailed = detailed
         self.key = None
+        self.module_paths = paths
 
     def get_args(self):  # nocv
         cmd = super(AnsibleModules, self).get_args()
         if self.detailed:
             cmd += ['--detail']
         if self.key:
             cmd += ['--get', self.key]
+        if isinstance(self.module_paths, (tuple, list)):
+            for path in self.module_paths:
+                cmd += ['--path', path]
         return cmd
 
     def get_ref(self, cache=False):
@@ -404,3 +415,7 @@ def get_inventory_data(self, raw_data):
         with tmp_file_context(data=raw_data) as tmp_file:
             self.path = tmp_file.name
             return self.get_data()
+
+
+class AnsibleConfigParser(PMAnsible):
+    ref_name = 'config'

requirements.txt

@@ -14,5 +14,5 @@ gitpython==2.1.11
 requests==2.20.0
 
 # Ansible required packages
-polemarch-ansible>=1.0.12
+polemarch-ansible>=1.1.0,<1.2
 pywinrm[kerberos]==0.3.0