1.6.2

Changelog:
* Improve `dockerrun` command.
* Optimize performance.
* Fix rerendering of Dashboard chart.

See merge request polemarch/ce!185

doc/installation.rst

@@ -133,8 +133,14 @@ Install from PyPI
            threads = 4
            harakiri = 120
            vacuum = True
+           http-keepalive = true
+           http-auto-chunked = true
+           thread-stacksize = 512
            pidfile = /opt/polemarch/pid/polemarch.pid
            log_file = /opt/polemarch/logs/{PROG_NAME}_web.log
+           # Uncomment it for HTTPS and install `uwsgi` pypi package to env:
+           # addrport = 127.0.0.1:8080
+           # https = 0.0.0.0:443,/etc/polemarch/polemarch.crt,/etc/polemarch/polemarch.key
 
            [worker]
            # output will be /opt/polemarch/logs/polemarch_worker.log

docker_release.yml

@@ -59,13 +59,13 @@
     - meta: flush_handlers
 
     - name: Get latest release tag
-      shell: echo -ne $(git describe --tags `git rev-list --tags --max-count=1`)
+      shell: git describe --tags `git rev-list --tags --max-count=1`
       register: release_latest_version
 
     - name: Set latest tag
       set_fact:
         docker_tag: 'latest'
-      changed_when: release_latest_version.tag == tag
+      changed_when: release_latest_version.stdout == tag
       notify:
         - "build registry"
         - "build dockerhub"

environment/docker_data/settings.ini

@@ -5,3 +5,10 @@ hooks_dir = /hooks
 [uwsgi]
 pidfile = /run/web.pid
 addrport = 0.0.0.0:8080
+vacuum = True
+max-requests = 1000
+max-worker-lifetime = 3600
+worker-reload-mercy = 60
+http-keepalive = true
+http-auto-chunked = true
+thread-stacksize = 1024

polemarch/__init__.py

@@ -31,6 +31,6 @@
     "VST_ROOT_URLCONF": os.getenv("VST_ROOT_URLCONF", 'vstutils.urls'),
 }
 
-__version__ = "1.6.1.post2"
+__version__ = "1.6.2"
 
 prepare_environment(**default_settings)

polemarch/api/v2/filters.py

@@ -2,7 +2,7 @@
 from functools import reduce
 from operator import or_
 from django.db.models import Q
-from django_filters import (CharFilter, NumberFilter, IsoDateTimeFilter)
+from django_filters import CharFilter, NumberFilter, IsoDateTimeFilter
 from vstutils.api.filters import DefaultIDFilter, extra_filter, name_filter, filters
 from ...main import models
 

polemarch/api/v2/serializers.py

@@ -3,26 +3,26 @@
 from typing import Dict, List
 import json
 import uuid
-try:
-    from ruamel.ordereddict import ordereddict as OrderedDict
-except ImportError:  # nocv
-    from collections import OrderedDict
-import six
-from django.contrib.auth.models import User
+from collections import OrderedDict
+from django.contrib.auth import get_user_model
 from django.utils.functional import cached_property
 from django.db import transaction
 from rest_framework import serializers, exceptions, status
-from rest_framework.exceptions import PermissionDenied
 from vstutils.api import serializers as vst_serializers, fields as vst_fields
 from vstutils.api.serializers import DataSerializer, EmptySerializer
 from vstutils.api.base import Response
 from ...main.utils import AnsibleArgumentsReference, AnsibleInventoryParser
+from ...main.settings import LANGUAGES
 
-from ...main.models import Inventory
 from ...main import models
 from ..signals import api_post_save, api_pre_save
 
 
+User = get_user_model()
+
+LANG_CHOICES = [item[0] for item in LANGUAGES]
+
+
 # NOTE: we can freely remove that because according to real behaviour all our
 #  models always have queryset at this stage, so this code actually doing
 # nothing
@@ -33,7 +33,7 @@ class DictField(serializers.CharField):
     def to_internal_value(self, data):  # nocv
         return (
             data
-            if isinstance(data, (six.string_types, six.text_type, dict, list))
+            if isinstance(data, (str, dict, list))
             else self.fail("Unknown type.")
         )
 
@@ -51,7 +51,7 @@ def to_internal_value(self, data):
 
     def to_representation(self, value):
         return (
-            value if not isinstance(value, six.class_types)
+            value if not isinstance(value, type)
             else str(value)
         )
 
@@ -68,10 +68,12 @@ class InventoryAutoCompletionField(vst_fields.AutoCompletionField):
     def to_internal_value(self, data):
         inventory = super(InventoryAutoCompletionField, self).to_internal_value(data)
         try:
-            inventory = Inventory.objects.get(id=int(inventory))
+            inventory = models.Inventory.objects.get(id=int(inventory))
             user = self.context['request'].user
             if not inventory.acl_handler.viewable_by(user):
-                raise PermissionDenied("You don't have permission to inventory.")  # noce
+                raise exceptions.PermissionDenied(
+                    "You don't have permission to inventory."
+                )  # noce
         except (ValueError, KeyError):
             self.check_path(inventory)
         return inventory
@@ -120,7 +122,7 @@ class SetOwnerSerializer(DataSerializer):
 
     def update(self, instance, validated_data):
         if not self.instance.acl_handler.owned_by(self.current_user()):  # noce
-            raise PermissionDenied(self.perms_msg)
+            raise exceptions.PermissionDenied(self.perms_msg)
         user = self.get_user(validated_data)
         self.instance.acl_handler.set_owner(user)
         return user
@@ -132,7 +134,7 @@ def current_user(self) -> User:
         return self.context['request'].user
 
     def to_representation(self, value: User):
-        return dict(user_id=value.id)
+        return dict(user_id=value.pk)
 
     def to_internal_value(self, data: dict):
         return dict(pk=data['user_id'])
@@ -245,9 +247,12 @@ class WidgetSettingsSerializer(vst_serializers.JsonObjectSerializer):
 
 
 class UserSettingsSerializer(vst_serializers.JsonObjectSerializer):
+    lang = serializers.ChoiceField(choices=LANG_CHOICES, default=LANG_CHOICES[0])
     autoupdateInterval = serializers.IntegerField(default=15000)
     chartLineSettings = ChartLineSettingsSerializer()
     widgetSettings = WidgetSettingsSerializer()
+    selectedSkin = serializers.CharField(required=False)
+    skinsSettings = vst_serializers.DataSerializer(required=False)
 
 
 class TeamSerializer(_WithPermissionsSerializer):
@@ -1050,8 +1055,7 @@ def __new__(cls, name, bases, attrs):
         return super(AnsibleSerializerMetaclass, cls).__new__(cls, name, bases, attrs)
 
 
-@six.add_metaclass(AnsibleSerializerMetaclass)
-class _AnsibleSerializer(serializers.Serializer):
+class _AnsibleSerializer(serializers.Serializer, metaclass=AnsibleSerializerMetaclass):
     # pylint: disable=abstract-method
     pass
 
@@ -1116,7 +1120,7 @@ def create(self, validated_data: Dict) -> Dict:
         parser = AnsibleInventoryParser()
         inv_json = parser.get_inventory_data(validated_data['raw_data'])
 
-        inventory = Inventory.objects.create(name=validated_data['name'])
+        inventory = models.Inventory.objects.create(name=validated_data['name'])
         inventory.vars = inv_json['vars']
         created_hosts, created_groups = dict(), dict()
 

polemarch/main/migrations/0002_modules_and_rename.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.10 on 2018-07-09 07:50
 from __future__ import unicode_literals
-import six
 from django.db import migrations, models
 import django.db.models.deletion
 from ..models.base import ForeignKeyACL
@@ -30,7 +29,7 @@ def set_inventories_to_project(apps, schema_editor):
     for templ in Template.objects.all():
         try:
             inv = templ.inventory_object
-            if not isinstance(inv, (six.string_types, six.text_type)):
+            if not isinstance(inv, str):
                 templ.project.inventories.add(inv)
         except:
             pass

polemarch/main/migrations/0046_auto_20180608_0658.py

@@ -1,14 +1,14 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.10 on 2018-06-08 06:58
 from __future__ import unicode_literals
+import io
 from functools import partial
-import six
 from django.db import migrations, models
 from django.db.models.functions import Length
 
 
 def __bulking_lines(history, value, number, obj_class):
-    out = six.StringIO(value)
+    out = io.StringIO(value)
     for line in iter(partial(out.read, 2 * 1024 - 100), ''):
         yield obj_class(history=history, line_number=number, line=line)
 

polemarch/main/models/__init__.py

@@ -1,13 +1,11 @@
-# pylint: disable=unused-argument,no-member
+# pylint: disable=unused-argument
 from __future__ import absolute_import
-from typing import Any, Text, NoReturn, Iterable, Union
+from typing import Any, Text, NoReturn, Iterable, Dict, Union
 import sys
 import json
 import logging
 from collections import OrderedDict
-from pytz import timezone
-import django_celery_beat
-from django_celery_beat.models import IntervalSchedule, CrontabSchedule
+from django_celery_beat.models import IntervalSchedule, CrontabSchedule, PeriodicTask as CPTask
 from django.db.models import signals, IntegerField
 from django.db import transaction
 from django.dispatch import receiver
@@ -212,7 +210,7 @@ def clean_dirs(instance: Project, **kwargs) -> None:
     instance.repo_class.delete()
 
 
-def compare_schedules(new_schedule_data: OrderedDict, old_schedule: Union[CrontabSchedule, IntervalSchedule]):
+def compare_schedules(new_schedule_data: Dict, old_schedule: Union[CrontabSchedule, IntervalSchedule]):
     """
     Method for compare parameters for Schedule from instance and current Periodic Task Schedule Params
     :param new_schedule_data: Dictionary contains data for new Schedule
@@ -246,10 +244,9 @@ def save_to_beat(instance: PeriodicTask, **kwargs) -> NoReturn:
         interval=IntervalSchedule,
         crontab=CrontabSchedule,
     )
-    manager = django_celery_beat.models.PeriodicTask.objects
 
     # Try get Celery Periodic Task, that linked with Polemarch Periodic Task
-    celery_task = manager.filter(
+    celery_task = CPTask.objects.filter(
         name=str(instance.id), task=settings.TASKS_HANDLERS["SCHEDUER"]["BACKEND"]
     ).last()
 
@@ -262,6 +259,8 @@ def save_to_beat(instance: PeriodicTask, **kwargs) -> NoReturn:
     elif instance_pt_type == 'crontab':
         schedule_data = instance.crontab_kwargs
         schedule_data['timezone'] = settings.TIME_ZONE
+    else:
+        raise ValidationError("Unknown periodic task type `{}`.".format(instance.type))  # nocv
 
 
     if celery_task:
@@ -278,12 +277,11 @@ def save_to_beat(instance: PeriodicTask, **kwargs) -> NoReturn:
             for type_name in filter(lambda k: k != instance_pt_type, types_dict.keys()):
                 schedule_old = getattr(celery_task, type_name, None)
                 if schedule_old is not None:
-                    schedule_old_type = type_name
+                    setattr(celery_task, type_name, None)
                     break
 
             # Update data for old schedule type and new schedule type
             setattr(celery_task, instance_pt_type, schedule_new)
-            setattr(celery_task, schedule_old_type, None)
         else:
             # Update celery periodic task schedule
             setattr(celery_task, instance_pt_type, schedule_new)
@@ -294,7 +292,7 @@ def save_to_beat(instance: PeriodicTask, **kwargs) -> NoReturn:
             schedule_old.delete()
     else:
         # Create new Celery Periodic Task, if it doesn't  exist
-        manager.create(
+        CPTask.objects.create(
             name=str(instance.id),
             task=settings.TASKS_HANDLERS["SCHEDUER"]["BACKEND"],
             args=json.dumps([instance.id]),
@@ -315,7 +313,7 @@ def delete_from_beat(instance: PeriodicTask, **kwargs) -> NoReturn:
         return
 
     # Get Celery Periodic Task
-    celery_task = django_celery_beat.models.PeriodicTask.objects.filter(
+    celery_task = CPTask.objects.filter(
         name=str(instance.id), task=settings.TASKS_HANDLERS["SCHEDUER"]["BACKEND"]
     ).last()
 

polemarch/main/models/hosts.py

@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 from typing import Any, List, Tuple, Dict, Text
 import logging
-import six
 from django.db.models import Q
 try:
     from yaml import dump as to_yaml, CDumper as Dumper, ScalarNode
@@ -26,7 +25,7 @@ class InventoryDumper(Dumper):
     yaml_representers[type(None)] = lambda dumper, value: (
         ScalarNode(tag=u'tag:yaml.org,2002:null', value='')
     )
-    yaml_representers[six.text_type] = lambda dumper, value: (
+    yaml_representers[str] = lambda dumper, value: (
         ScalarNode(tag=u'tag:yaml.org,2002:str', value=value)
     )
 

polemarch/main/models/projects.py

@@ -7,7 +7,6 @@
 import traceback
 import time
 import uuid
-import six
 import requests
 from docutils.core import publish_parts as rst_gen
 from markdown2 import Markdown
@@ -278,7 +277,7 @@ def execute_view_data(self) -> Dict[str, Dict[str, Dict]]:
         return view_data
 
     def check_path(self, inventory) -> NoReturn:
-        if not isinstance(inventory, (six.string_types, six.text_type)):  # nocv
+        if not isinstance(inventory, str):  # nocv
             return
         path = "{}/{}".format(self.path, inventory)
         path = os.path.abspath(os.path.expanduser(path))

polemarch/main/models/tasks.py

@@ -8,7 +8,7 @@
 import json
 
 import re
-import six
+import io
 from celery.schedules import crontab
 from django.core.exceptions import ValidationError
 from django.db import transaction
@@ -117,7 +117,7 @@ def ci_run(self):
         self.execute(self.project.owner)
 
     def _convert_to_data(self, value):
-        if isinstance(value, (six.string_types, six.text_type)):
+        if isinstance(value, str):
             return json.loads(value)  # nocv
         elif isinstance(value, (dict, OrderedDict, list)):
             return value
@@ -321,7 +321,7 @@ def _get_history_stats_by(self, qs, grouped_by='day') -> List:
             sum_by_date[hist_stat[grouped_by]] = (
                 sum_by_date.get(hist_stat[grouped_by], 0) + hist_stat['sum']
             )
-        for hist_stat in qs.order_by(grouped_by):
+        for hist_stat in qs.order_by(grouped_by, 'status'):
             hist_stat.update({'all': sum_by_date[hist_stat[grouped_by]]})
             values.append(hist_stat)
         return values
@@ -361,7 +361,7 @@ def start(self, project, kind, mod_name, inventory, **extra) -> Tuple[Any, Dict]
             executor=extra_options['executor'], hidden=project.hidden,
             options=self.__get_additional_options(extra_options)
         )
-        if isinstance(inventory, (six.string_types, six.text_type)):
+        if isinstance(inventory, str):
             history_kwargs['inventory'] = None
         elif isinstance(inventory, int):
             history_kwargs['inventory'] = project.inventories.get(pk=inventory)  # nocv
@@ -531,7 +531,7 @@ def __create_line(self, gnum: int, num: int, val: str, hidden: bool = False) ->
         )
 
     def __bulking_lines(self, value: str, number: int, endl: str) -> Generator:
-        out = six.StringIO(value)
+        out = io.StringIO(value)
         nline = 0
         for line in iter(partial(out.read, 2 * 1024 - 100), ''):
             nline += 1

polemarch/main/tests/executions.py

@@ -9,7 +9,6 @@
 
 import git
 import requests
-import six
 from django.conf import settings
 from django.utils.timezone import now
 from django.core.management import call_command
@@ -537,7 +536,7 @@ def project_workflow(self, repo_type, **kwargs):
             if not execute:
                 return
             kwargs = getattr(self, 'wip_{}'.format(repo_type.lower()), str)(project_data)
-            kwargs = kwargs if not isinstance(kwargs, six.string_types) else dict()
+            kwargs = kwargs if not isinstance(kwargs, str) else dict()
             self.change_owner(project_data)
             self.playbook_tests(project_data, **kwargs)
             self.module_tests(project_data)
@@ -749,7 +748,7 @@ def wip_git(self, project_data):
             for required_field in ['title', 'default', 'format', 'help']:
                 self.assertIn(required_field, field.keys())
             self.assertEqual(field_name.split('_')[-1], field['format'], field)
-            default_type = (six.string_types, six.text_type)
+            default_type = str
             if field['format'] == 'boolean':
                 default_type = bool
             elif field['format'] == 'integer':