Release 0.2.1

Changes:
* Import inventory from text
* Autoupdate time parametr in user settings
* Search in templates options and variables
* Fix bug with systemd
* Fix small GUI bugs

See merge request polemarch/ce!81

.gitlab-ci.yml

@@ -77,11 +77,7 @@ stages:
   allow_failure: true
   only:
    - tags
-  before_script:
-  - git remote add ${PUBLISH_REMOTE_NAME} ${PUBLISH_REMOTE_URI}
-  script:
-  - git push -f ${PUBLISH_REMOTE_NAME} master
-  - git push -f ${PUBLISH_REMOTE_NAME} ${CI_COMMIT_TAG}
+
 
 # Branch tests
 ###########################################
@@ -164,8 +160,6 @@ pages:
   only:
     refs:
       - developer
-    changes:
-      - doc/*
 
 release_pypi:
   stage: release
@@ -218,18 +212,7 @@ release_deb:
     paths:
     - dist/
 
-# DEPRECATED
-#publish_gitlab:
-#  <<: *publishing
-#  variables:
-#    PUBLISH_REMOTE_USER: "gitlab-ci-token"
-#    PUBLISH_REMOTE_TOKEN: "${GITLAB_TOKEN}"
-#
-#publish_release:
-#  <<: *publishing
-#  variables:
-#    PUBLISH_REMOTE_USER: "${GITHUB_USER}"
-#    PUBLISH_REMOTE_TOKEN: "${GITHUB_TOKEN}"
-#    PUBLISH_REMOTE_SERVER: 'github.com'
-#  after_script:
-#  - make test ENVS=release
+publish_release:
+  <<: *publishing
+  script:
+  - make test ENVS=release

.pep8

@@ -1,4 +1,4 @@
 [flake8]
-ignore = E221,E222,E121,E123,E126,E226,E24,E704,E116,E731,E722,E741,E402
+ignore = E221,E222,E121,E123,E126,E226,E24,E704,E116,E731,E722,E741,E402,W504,W503,W605
 exclude = ./polemarch/*/migrations/*,./polemarch/main/settings*.py,.tox/*,./etc/*,./*/__init__.py,*__main.py,./t_openstack.py,./polemarch/projects/*
 max-line-length=90

README.rst

@@ -56,6 +56,9 @@ Red Hat/CentOS installation
 That's it. Polemarch web panel on 8080 port. Default administrative account is
 admin/admin.
 
+If you have any problems with `.deb` package, you can install it from PyPi:
+https://polemarch.readthedocs.io/en/stable/quickstart.html#install-from-pypi
+
 Note: If you using authentication by password at some of your machines
 managed by Polemarch, you also must install ``sshpass`` package because it
 required for ansible to autheticate via ssh by password. It available in
@@ -93,6 +96,9 @@ need to install it for your distro:
 That's it. Polemarch web panel on 8080 port. Default administrative account is
 admin/admin.
 
+If you have any problems with `.deb` package, you can install it from PyPi:
+https://polemarch.readthedocs.io/en/stable/quickstart.html#install-from-pypi
+
 Quickstart
 ----------
 

doc/quickstart.rst

@@ -120,8 +120,14 @@ Install from PyPI
            vacuum = True
 
            [worker]
-           pidfile = /tmp/pm_worker.pid
-           logfile = /dev/null
+           logfile = /tmp/{PROG_NAME}_worker.log  # output will be /tmp/polemarch_worker.log
+           pidfile = /tmp/{PROG_NAME}_worker.pid  # output will be /tmp/polemarch_worker.pid
+           loglevel = INFO
+
+      Also if you need to set your own path for logfile or pidfile,
+      different from the path from example, you can do it, but make sure,
+      that user, which starts Polemarch has write-permissions for these directory and file.
+
 
 #. Make migrations:
 

polemarch/__init__.py

@@ -31,6 +31,6 @@
     "VST_ROOT_URLCONF": os.getenv("VST_ROOT_URLCONF", 'vstutils.urls'),
 }
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 prepare_environment(**default_settings)

polemarch/api/v2/serializers.py

@@ -11,7 +11,7 @@
 from vstutils.api import serializers as vst_serializers, fields as vst_fields
 from vstutils.api.serializers import DataSerializer, EmptySerializer
 from vstutils.api.base import Response
-from ...main.utils import AnsibleArgumentsReference
+from ...main.utils import AnsibleArgumentsReference, AnsibleInventoryParser
 
 from ...main.models import Inventory
 from ...main import models
@@ -182,9 +182,12 @@ def update(self, instance, validated_data):
 class _WithPermissionsSerializer(_SignalSerializer):
     perms_msg = "You do not have permission to perform this action."
 
-    def create(self, validated_data):
-        validated_data["owner"] = self.current_user()
-        return super(_WithPermissionsSerializer, self).create(validated_data)
+    def is_valid(self, *args, **kwargs):
+        result = super(_WithPermissionsSerializer, self).is_valid(*args, **kwargs)
+        self.validated_data['owner'] = self.validated_data.get(
+            'owner', self.current_user()
+        )
+        return result
 
     def current_user(self):
         return self.context['request'].user
@@ -262,6 +265,7 @@ class WidgetSettingsSerializer(vst_serializers.JsonObjectSerializer):
 
 
 class UserSettingsSerializer(vst_serializers.JsonObjectSerializer):
+    autoupdateInterval = serializers.IntegerField(default=15000)
     chartLineSettings = ChartLineSettingsSerializer()
     widgetSettings = WidgetSettingsSerializer()
 
@@ -1052,3 +1056,46 @@ class DashboardStatisticSerializer(DataSerializer):
     teams = serializers.IntegerField()
     users = serializers.IntegerField()
     jobs = DashboardJobsSerializer()
+
+
+class InventoryImportSerializer(DataSerializer):
+    inventory_id = vst_fields.RedirectIntegerField(default=None, allow_null=True)
+    name = serializers.CharField(required=True)
+    raw_data = serializers.CharField()
+
+    @transaction.atomic()
+    def create(self, validated_data):
+        parser = AnsibleInventoryParser()
+        inv_json = parser.get_inventory_data(validated_data['raw_data'])
+
+        inventory = Inventory.objects.create(name=validated_data['name'])
+        created_hosts, created_groups = dict(), dict()
+
+        for host in inv_json['hosts']:
+            inv_host = inventory.hosts.create(name=host['name'])
+            inv_host.vars = host['vars']
+            created_hosts[inv_host.name] = inv_host
+
+        for group in inv_json['groups']:
+            children = False if len(group['groups']) == 0 else True
+            inv_group = inventory.groups.create(name=group['name'], children=children)
+            inv_group.vars = group['vars']
+            created_groups[inv_group.name] = inv_group
+
+        for group in inv_json['groups']:
+            inv_group = created_groups[group['name']]
+            g_subs = list()
+            if inv_group.children:
+                for name in group['groups']:
+                    g_subs.append(created_groups[name])
+                inv_group.groups.add(*g_subs)
+            else:
+                for name in group['hosts']:
+                    g_subs.append(created_hosts[name])
+                inv_group.hosts.add(*g_subs)
+
+        return dict(
+            inventory_id=inventory.id,
+            name=inventory.name,
+            raw_data=validated_data['raw_data']
+        )

polemarch/api/v2/views.py

@@ -439,6 +439,13 @@ class InventoryViewSet(_GroupMixin):
     filter_class = filters.InventoryFilter
     copy_related = ['hosts', 'groups']
 
+    @deco.action(methods=["post"], detail=no)
+    def import_inventory(self, request, **kwargs):
+        serializer = sers.InventoryImportSerializer(data=request.data)
+        serializer.is_valid(True)
+        serializer.save()
+        return base.Response(serializer.data, status.HTTP_201_CREATED).resp
+
 
 class __PlaybookViewSet(base.ReadOnlyModelViewSet):
     '''

polemarch/main/models/vars.py

@@ -6,6 +6,7 @@
 
 from functools import reduce
 from collections import OrderedDict
+from django.db import transaction
 from django.db.models import Case, When, Value
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
@@ -122,6 +123,15 @@ def get_hook_data(self, when):
         # pylint: disable=unused-argument
         return OrderedDict(id=self.id, name=self.name)
 
+    @transaction.atomic()
+    def set_vars(self, variables):
+        encr = "[~~ENCRYPTED~~]"
+        encrypted_vars = {k: v for k, v in variables.items() if v == encr}
+        other_vars = {k: v for k, v in variables.items() if v != encr}
+        self.variables.exclude(key__in=encrypted_vars.keys()).delete()
+        for key, value in other_vars.items():
+            self.variables.create(key=key, value=value)
+
     def vars_string(self, variables, separator=" "):
         return separator.join(
             map(lambda kv: "{}={}".format(kv[0], kv[1]), variables.items())
@@ -144,6 +154,10 @@ def get_generated_vars(self):
     def vars(self):
         return self.get_vars()
 
+    @vars.setter
+    def vars(self, value):
+        self.set_vars(value)
+
     @property
     def have_vars(self):
         return bool(len(self.vars))  # nocv

polemarch/main/tests/hosts.py

@@ -1,4 +1,5 @@
 from ._base import BaseTestCase, json
+from ..unittests.ansible import inventory_data, valid_inventory
 
 
 class InvBaseTestCase(BaseTestCase):
@@ -250,3 +251,44 @@ def test_inventories(self):
             'Host', dict(name='inv-host'),
             copy_check=dict(host='hosts')
         )
+
+    def test_import_inventory(self):
+        result = self.get_result(
+            'post', self.get_url('inventory', 'import_inventory'), 201,
+            data=json.dumps(dict(name='test-inventory', raw_data=inventory_data))
+        )
+        self.assertIn('inventory_id', result)
+        bulk_data = [
+            self.get_bulk('inventory', {}, 'get', pk=result['inventory_id']),
+            self.get_mod_bulk(
+                'inventory',
+                '<0[data][id]>',
+                {},
+                'all_hosts',
+                method='get'
+            ),
+            self.get_mod_bulk(
+                'inventory',
+                '<0[data][id]>',
+                {},
+                'all_groups',
+                method='get'
+            ),
+        ]
+        results = self.make_bulk(bulk_data, 'put')
+        self.assertEqual(
+            results[0]['data']['id'],
+            result['inventory_id']
+        )
+        self.assertEqual(
+            results[1]['data']['count'],
+            len(valid_inventory['hosts'])
+        )
+        self.assertEqual(
+            results[2]['data']['count'],
+            len(valid_inventory['groups'])
+        )
+        for host in results[1]['data']['results']:
+            self.assertIn(host['name'], valid_inventory['hosts'].keys())
+        for group in results[2]['data']['results']:
+            self.assertIn(group['name'], valid_inventory['groups'].keys())

polemarch/main/unittests/ansible.py

@@ -1,6 +1,68 @@
 import six
 from django.test import TestCase
 from django.core.management import call_command
+from ..utils import AnsibleInventoryParser
+
+inventory_data = '''
+test-host-single ansible_host=10.10.10.10
+
+[test-group]
+test-host ansible_host=10.10.10.20
+
+[test-group:vars]
+ansible_user=ubuntu
+ansible_ssh_private_key_file=example-key
+
+[child-group]
+test-host-2
+
+[parent-group:children]
+child-group
+'''
+valid_inventory = {
+    'groups': {
+        'test-group': {
+            'name': 'test-group',
+            'groups': [],
+            'hosts': ['test-host'],
+            'vars': {
+                'ansible_user': 'ubuntu',
+                'ansible_ssh_private_key_file': 'example-key',
+            },
+        },
+        'child-group': {
+            'name': 'child-group',
+            'groups': [],
+            'hosts': ['test-host-2'],
+            'vars': {},
+        },
+        'parent-group': {
+            'name': 'parent-group',
+            'groups': ['child-group'],
+            'hosts': [],
+            'vars': {},
+        },
+    },
+    'hosts': {
+        'test-host-single': {
+            'name': 'test-host-single',
+            'vars': {
+                'ansible_host': '10.10.10.10',
+            }
+        },
+        'test-host': {
+            'name': 'test-host',
+            'vars': {
+                'ansible_host': '10.10.10.20',
+            }
+        },
+        'test-host-2': {
+            'name': 'test-host-2',
+            'vars': {}
+        }
+    },
+
+}
 
 
 class AnsibleTestCase(TestCase):
@@ -11,3 +73,31 @@ def test_modules(self):
             'The modules have been successfully updated.\n',
             out.getvalue().replace('\x1b[32;1m', '').replace('\x1b[0m', '')
         )
+
+    def test_inventory_parser(self):
+        parser = AnsibleInventoryParser()
+        inv_json = parser.get_inventory_data(inventory_data)
+        for record in inv_json['groups']:
+            self.assertIn(record['name'], valid_inventory['groups'].keys())
+            self.assertEqual(
+                list(valid_inventory['groups'][record['name']]['hosts']),
+                list(record['hosts'])
+            )
+            self.assertEqual(
+                list(valid_inventory['groups'][record['name']]['groups']),
+                list(record['groups'])
+            )
+            self.assertEqual(
+                list(valid_inventory['groups'][record['name']]['vars'].keys()),
+                list(record['vars'].keys())
+            )
+            for key, value in valid_inventory['groups'][record['name']]['vars'].items():
+                self.assertEqual(record['vars'][key], value)
+        for record in inv_json['hosts']:
+            self.assertIn(record['name'], valid_inventory['hosts'].keys())
+            self.assertEqual(
+                list(valid_inventory['hosts'][record['name']]['vars'].keys()),
+                list(record['vars'].keys())
+            )
+            for key, value in valid_inventory['hosts'][record['name']]['vars'].items():
+                self.assertEqual(record['vars'][key], value)