1.5.0

Changelog:
* Optimize history list view.
* Fix key parse on execution.
* Fix crontab time in periodic tasks.
* Update design.
* Performance upgrade.
* Optimize package size.

See merge request polemarch/ce!170

.pylintrc

@@ -32,7 +32,7 @@ unsafe-load-any-extension=no
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code
-extension-pkg-whitelist=vstutils.custom_model,vstutils.api.fields,vstutils.tools
+extension-pkg-whitelist=
 
 # Allow optimization of some AST trees. This will activate a peephole AST
 # optimizer, which will apply various small optimizations. For instance, it can
@@ -65,7 +65,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=no-name-in-module,useless-super-delegation,len-as-condition,super-init-not-called,keyword-arg-before-vararg,no-else-return,no-self-argument,inconsistent-return-statements,unsubscriptable-object,too-many-branches,deprecated-lambda,old-style-class,no-init,expression-not-assigned,broad-except,logging-format-interpolation,model-no-explicit-unicode,too-many-ancestors,bad-continuation,bad-whitespace,redefined-builtin,missing-docstring,redefined-variable-type,no-self-use,line-too-long,suppressed-message,cmp-method,no-absolute-import,xrange-builtin,using-cmp-argument,basestring-builtin,backtick,unpacking-in-except,old-raise-syntax,getslice-method,long-builtin,print-statement,reduce-builtin,filter-builtin-not-iterating,import-star-module-level,unichr-builtin,dict-iter-method,range-builtin-not-iterating,file-builtin,old-division,standarderror-builtin,coerce-builtin,setslice-method,old-ne-operator,long-suffix,execfile-builtin,oct-method,metaclass-assignment,intern-builtin,apply-builtin,dict-view-method,raw_input-builtin,raising-string,coerce-method,unicode-builtin,next-method-called,hex-method,nonzero-method,round-builtin,cmp-builtin,reload-builtin,buffer-builtin,useless-suppression,zip-builtin-not-iterating,indexing-exception,map-builtin-not-iterating,delslice-method,old-octal-literal,input-builtin,parameter-unpacking,model-has-unicode,bare-except,too-few-public-methods,fixme,dangerous-default-value,attribute-defined-outside-init,pointless-string-statement,too-many-instance-attributes,arguments-differ,binary-op-exception,bad-classmethod-argument,locally-disabled,file-ignored,multiple-statements,superfluous-parens,bad-mcs-classmethod-argument,useless-object-inheritance
+disable=unexpected-keyword-arg,no-name-in-module,useless-super-delegation,len-as-condition,super-init-not-called,keyword-arg-before-vararg,no-else-return,no-self-argument,inconsistent-return-statements,unsubscriptable-object,too-many-branches,deprecated-lambda,old-style-class,no-init,expression-not-assigned,broad-except,logging-format-interpolation,model-no-explicit-unicode,too-many-ancestors,bad-continuation,bad-whitespace,redefined-builtin,missing-docstring,redefined-variable-type,no-self-use,line-too-long,suppressed-message,cmp-method,no-absolute-import,xrange-builtin,using-cmp-argument,basestring-builtin,backtick,unpacking-in-except,old-raise-syntax,getslice-method,long-builtin,print-statement,reduce-builtin,filter-builtin-not-iterating,import-star-module-level,unichr-builtin,dict-iter-method,range-builtin-not-iterating,file-builtin,old-division,standarderror-builtin,coerce-builtin,setslice-method,old-ne-operator,long-suffix,execfile-builtin,oct-method,metaclass-assignment,intern-builtin,apply-builtin,dict-view-method,raw_input-builtin,raising-string,coerce-method,unicode-builtin,next-method-called,hex-method,nonzero-method,round-builtin,cmp-builtin,reload-builtin,buffer-builtin,useless-suppression,zip-builtin-not-iterating,indexing-exception,map-builtin-not-iterating,delslice-method,old-octal-literal,input-builtin,parameter-unpacking,model-has-unicode,bare-except,too-few-public-methods,fixme,dangerous-default-value,attribute-defined-outside-init,pointless-string-statement,too-many-instance-attributes,arguments-differ,binary-op-exception,bad-classmethod-argument,locally-disabled,file-ignored,multiple-statements,superfluous-parens,bad-mcs-classmethod-argument,useless-object-inheritance
 
 
 [REPORTS]
@@ -277,13 +277,12 @@ ignore-mixin-members=yes
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=vstutils.tools,vstutils.custom_model,vstutils.urls,vstutils.api.base,vstutils.api.views,vstutils.gui.views,vstutils.api.permissions,vstutils.api.filters,vstutils.middleware,vstutils.environment,vstutils.utils,vstutils.exceptions,vstutils.models,vstutils.api.decorators,vstutils.api.swagger,vstutils.api.serializers,
-# ignored-modules=vstutils
+ignored-modules=vstutils.custom_model,vstutils.api.views,
 
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
 # qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local,vstutils.custom_model
+ignored-classes=optparse.Values,thread._local,_thread._local,
 
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular

doc/config.rst

@@ -367,9 +367,9 @@ Installation of additional packages to Polemarch
 ------------------------------------------------
 
 .. warning::
-    .rpm or .dep installation methods are depracated.
+    .rpm or .deb installation methods are depracated.
 
-If you want to install some additional package to Polemarch from .rpm or .dep,
+If you want to install some additional package to Polemarch from .rpm or .deb,
 you should run next command:
 
 .. sourcecode:: bash

polemarch/__init__.py

@@ -31,6 +31,6 @@
     "VST_ROOT_URLCONF": os.getenv("VST_ROOT_URLCONF", 'vstutils.urls'),
 }
 
-__version__ = "1.4.4"
+__version__ = "1.5.0"
 
 prepare_environment(**default_settings)

polemarch/api/v2/filters.py

@@ -1,4 +1,7 @@
 # pylint: disable=import-error
+from functools import reduce
+from operator import or_
+from django.db.models import Q
 from django_filters import (CharFilter, NumberFilter, IsoDateTimeFilter)
 from vstutils.api.filters import DefaultIDFilter, extra_filter, name_filter, filters
 from ...main import models
@@ -15,6 +18,21 @@ def variables_filter(queryset, field, value):
     return queryset.var_filter(**kwargs)
 
 
+def filter_name_endswith(queryset, field, value):
+    # pylint: disable=unused-argument
+    return queryset.filter(
+        reduce(or_, (
+            Q(path__endswith='.{}'.format(v))
+            for v in value.split(',')
+        ))
+    )
+
+
+def playbook_filter(queryset, field, value):
+    # pylint: disable=unused-argument
+    return queryset.filter(playbook__in=value.split(','))
+
+
 class VariableFilter(DefaultIDFilter):
     key = CharFilter(method=name_filter, help_text=name_help.replace('name', 'key name'))
     value = CharFilter(method=name_filter, help_text='A value of instance.')
@@ -54,11 +72,13 @@ class Meta:
 class ModuleFilter(filters.FilterSet):
     path__not = CharFilter(method=name_filter, help_text='Full path to module.')
     path      = CharFilter(method=name_filter, help_text='Full path to module.')
+    name      = CharFilter(method=filter_name_endswith, help_text='Name of module.')
 
     class Meta:
         model = models.Module
         fields = (
             'path',
+            'name',
         )
 
 
@@ -115,12 +135,14 @@ class Meta:
 class TaskFilter(_BaseFilter):
     playbook__not = CharFilter(method=name_filter, help_text='Playbook filename.')
     playbook      = CharFilter(method=name_filter, help_text='Playbook filename.')
+    pb_filter     = CharFilter(method=playbook_filter, help_text='Playbook filename - filter for prefetch.')
 
     class Meta:
         model = models.Task
         fields = ('id',
                   'name',
-                  'playbook',)
+                  'playbook',
+                  'pb_filter',)
 
 
 class HistoryFilter(_BaseFilter):

polemarch/api/v2/permissions.py

@@ -8,6 +8,8 @@ def has_permission(self, request, view):
 
     def get_user_permission(self, request, view, obj):  # nocv
         # pylint: disable=unused-argument
+        if hasattr(obj, 'owner') and obj.owner == request.user:
+            return True
         return False
 
     def has_object_permission(self, request, view, obj):

polemarch/api/v2/serializers.py

@@ -171,14 +171,14 @@ class _WithPermissionsSerializer(_SignalSerializer):
 
     def is_valid(self, *args, **kwargs):
         result = super(_WithPermissionsSerializer, self).is_valid(*args, **kwargs)
-        if not hasattr(self, 'instance'):  # nocv
+        if not hasattr(self, 'instance') or self.instance is None:  # noce
             self.validated_data['owner'] = self.validated_data.get(
                 'owner', self.current_user()
             )
         return result
 
     def current_user(self) -> User:
-        return self.context['request'].user  # nocv
+        return self.context['request'].user  # noce
 
 
 class UserSerializer(vst_serializers.UserSerializer):
@@ -770,7 +770,7 @@ class Meta:
                   'owner',)
 
 
-class ProjectCreateMasterSerializer(vst_serializers.VSTSerializer):
+class ProjectCreateMasterSerializer(vst_serializers.VSTSerializer, _WithPermissionsSerializer):
     types = models.list_to_choices(models.Project.repo_handlers.keys())
     auth_types = ['NONE', 'KEY', 'PASSWORD']
     branch_auth_types = {t: "hidden" for t in models.Project.repo_handlers.keys()}
@@ -884,7 +884,7 @@ def update(self, instance: models.ProjectTemplate, validated_data) -> models.Pro
             repo_auth=instance.repo_auth,
             auth_data=instance.auth_data or '',
         )
-        serializer = ProjectCreateMasterSerializer(data=data)
+        serializer = ProjectCreateMasterSerializer(data=data, context=self.context)
         serializer.is_valid(raise_exception=True)
         serializer.save()
         return serializer.instance

polemarch/api/v2/views.py

@@ -389,6 +389,7 @@ class GroupViewSet(_BaseGroupViewSet, _GroupMixin):
     __doc__ = _BaseGroupViewSet.__doc__
 
     def nested_allow_check(self):
+        # pylint: disable=no-member
         exception = _BaseGroupViewSet.serializer_class_one.ValidationException
         if not self.nested_parent_object.children and self.nested_name == 'group':
             raise exception("Group is not children.")
@@ -428,6 +429,7 @@ class InventoryViewSet(_GroupMixin):
 
     @deco.action(methods=["post"], detail=no)
     def import_inventory(self, request, **kwargs):
+        # pylint: disable=no-member
         serializer = self.get_serializer(data=request.data)
         serializer.is_valid(True)
         serializer.save()

polemarch/main/models/__init__.py

@@ -7,7 +7,7 @@
 from collections import OrderedDict
 import django_celery_beat
 from django_celery_beat.models import IntervalSchedule, CrontabSchedule
-from django.db.models import signals, IntegerField
+from django.db.models import signals, IntegerField, Q
 from django.dispatch import receiver
 from django.db.models.functions import Cast
 from django.core.validators import ValidationError
@@ -230,6 +230,8 @@ def save_to_beat(instance: PeriodicTask, **kwargs) -> NoReturn:
     elif instance.type == "CRONTAB":
         cron_data = instance.crontab_kwargs
         schedule, _ = CrontabSchedule.objects.get_or_create(**cron_data)
+        schedule.timezone = settings.TIME_ZONE
+        schedule.save()
         manager.create(crontab=schedule,
                        name=str(instance.id),
                        task=task,
@@ -356,3 +358,10 @@ def update_ptasks_with_templates(instance: Template, **kwargs) -> NoReturn:
 def cancel_task_on_delete_history(instance: History, **kwargs) -> NoReturn:
     exchange = KVExchanger(CmdExecutor.CANCEL_PREFIX + str(instance.id))
     exchange.send(True, 60) if instance.working else None
+
+
+@receiver(signals.post_migrate)
+def update_crontab_timezone_ptasks(*args, **kwargs):
+    qs = CrontabSchedule.objects.exclude(timezone=settings.TIME_ZONE)
+    qs.filter(periodictask__name__startswith='polemarch').update(timezone=settings.TIME_ZONE)
+    qs.filter(periodictask__name__startswith='pmlib').update(timezone=settings.TIME_ZONE)

polemarch/main/models/tasks.py

@@ -250,14 +250,17 @@ def inventory(self) -> InvOrString:
 
     @inventory.setter
     def inventory(self, inventory: InvOrString) -> NoReturn:
-        if isinstance(inventory, Inventory):
-            self._inventory = inventory  # nocv
-        elif isinstance(inventory, (six.string_types, six.text_type, int)):
+        if isinstance(inventory, Inventory):  # nocv
+            self._inventory = inventory
+            self.inventory_file = None
+        elif isinstance(inventory, (str, int)):
             try:
                 self._inventory = self.project.inventories.get(pk=int(inventory))
+                self.inventory_file = None
             except (ValueError, Inventory.DoesNotExist):
                 self.project.check_path(inventory)
                 self.inventory_file = inventory
+                self._inventory = None
 
     @property
     def crontab_kwargs(self) -> Dict:

polemarch/main/models/utils.py

@@ -13,7 +13,6 @@
 from collections import namedtuple, OrderedDict
 from subprocess import Popen
 from functools import reduce
-import six
 from django.utils import timezone
 from vstutils.utils import tmp_file, KVExchanger, raise_context
 from vstutils.tools import get_file_value
@@ -109,7 +108,7 @@ def execute(self, cmd: Iterable[Text], cwd: Text):
         pm_ansible_path = ' '.join(self.pm_ansible())
         new_cmd = list()
         for one_cmd in cmd:
-            if isinstance(one_cmd, six.string_types):
+            if isinstance(one_cmd, str):
                 with raise_context():
                     one_cmd = one_cmd.decode('utf-8')
             new_cmd.append(one_cmd)
@@ -145,7 +144,7 @@ def __init__(self, inventory: Union[Inventory, int, Text], cwd: Text = "/tmp", t
             self.tmpdir = tmpdir
             self._file = None
             self.is_file = True
-            if isinstance(inventory, (six.string_types, six.text_type)):
+            if isinstance(inventory, str):
                 self.raw, self.keys = self.get_from_file(inventory)
             else:
                 self.raw, self.keys = self.get_from_int(inventory)
@@ -177,18 +176,20 @@ def file(self) -> Union[tmp_file, Text]:
         @property
         def file_name(self) -> Text:
             # pylint: disable=no-member
-            if isinstance(self.file, (six.string_types, six.text_type)):
+            if isinstance(self.file, str):
                 return self.file
             return self.file.name
 
         def close(self) -> NoReturn:
             # pylint: disable=no-member
             map(lambda key_file: key_file.close(), self.keys) if self.keys else None
-            if not isinstance(self.file, (six.string_types, six.text_type)):
+            if not isinstance(self.file, str):
                 self._file.close()
 
     def __init__(self, *args, **kwargs):
         self.args = args
+        if 'verbose' in kwargs:
+            kwargs['verbose'] = int(float(kwargs.get('verbose', 0)))
         self.kwargs = kwargs
         self.__will_raise_exception = False
         self.ref_type = self.ref_types[self.command_type]
@@ -229,7 +230,7 @@ def __parse_key(self, key: Text, value: Text) -> Tuple[Text, List]:
         # pylint: disable=unused-argument,
         if re.match(r"[-]+BEGIN .+ KEY[-]+", value):
             # Add new line if not exists and generate tmpfile for private key value
-            value = value + '/n' if value[-1] != '/n' else value
+            value = value + '\n' if value[-1] != '\n' else value
             return self.__generate_arg_file(value)
         # Return path in project if it's path
         path = (Path(self.workdir)/Path(value).expanduser()).resolve()
@@ -436,5 +437,5 @@ def __init__(self, target: Text, *pargs, **kwargs):
         super(AnsibleModule, self).__init__(*pargs, **kwargs)
         self.ansible_ref['module-name'] = {'type': 'string'}
 
-    def execute(self, group: Text, *args, **extra_args):
+    def execute(self, group: Text = 'all', *args, **extra_args):
         return super(AnsibleModule, self).execute(group, *args, **extra_args)