Merge pull request #1703 from arenadata/develop

Release 2022.04.18

Makefile

@@ -3,7 +3,7 @@ BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
 
 ADCMBASE_IMAGE ?= hub.arenadata.io/adcm/base
 ADCMTEST_IMAGE ?= hub.arenadata.io/adcm/test
-ADCMBASE_TAG ?= 20220323204419
+ADCMBASE_TAG ?= 20220415154808
 APP_IMAGE ?= hub.adsw.io/adcm/adcm
 APP_TAG ?= $(subst /,_,$(BRANCH_NAME))
 
@@ -63,7 +63,7 @@ pytest: ## Run functional tests
 	-e BUILD_TAG=${BUILD_TAG} -e ADCMPATH=/adcm/ -e PYTHONPATH=${PYTHONPATH}:python/ \
 	-e SELENOID_HOST="${SELENOID_HOST}" -e SELENOID_PORT="${SELENOID_PORT}" \
 	hub.adsw.io/library/functest:3.8.6.slim.buster-x64 /bin/sh -e \
-	./pytest.sh -m "not full" --adcm-image='hub.adsw.io/adcm/adcm:$(subst /,_,$(BRANCH_NAME))'
+	./pytest.sh -m "not full and not extra_rbac" --adcm-image='hub.adsw.io/adcm/adcm:$(subst /,_,$(BRANCH_NAME))'
 
 pytest_release: ## Run functional tests on release
 	docker pull hub.adsw.io/library/functest:3.8.6.slim.buster.firefox-x64

conf/adcm/config.yaml

@@ -2,7 +2,7 @@
 
   type: adcm
   name: ADCM
-  version: 1.3
+  version: 1.4
 
   config:
     - name: "global"
@@ -59,6 +59,8 @@
           description: |
             Mitogen for Ansible is a completely redesigned UNIX connection layer and module runtime for Ansible.
           type: boolean
+          ui_options:
+            invisible: true
           default: false
         - name: "forks"
           display_name: "Forks"

python/api/action/views.py

@@ -53,7 +53,7 @@ def get_obj(**kwargs):
 
 
 class ActionList(PermissionListMixin, GenericUIView):
-    queryset = Action.objects.all()
+    queryset = Action.objects.filter(upgrade__isnull=True)
     serializer_class = serializers.ActionSerializer
     serializer_class_ui = serializers.ActionUISerializer
     filterset_class = ActionFilter
@@ -123,7 +123,7 @@ def get(self, request, *args, **kwargs):  # pylint: disable=too-many-locals
 
 
 class ActionDetail(PermissionListMixin, GenericUIView):
-    queryset = Action.objects.all()
+    queryset = Action.objects.filter(upgrade__isnull=True)
     serializer_class = serializers.ActionDetailSerializer
     serializer_class_ui = serializers.ActionUISerializer
     permission_classes = (DjangoOnlyObjectPermissions,)
@@ -140,7 +140,12 @@ def get(self, request, *args, **kwargs):
             request.user, f'{ct.app_label}.view_{ct.model}', model, id=object_id
         )
         # TODO: we can access not only the actions of this object
-        action = get_object_for_user(request.user, 'cm.view_action', Action, id=action_id)
+        action = get_object_for_user(
+            request.user,
+            'cm.view_action',
+            self.get_queryset(),
+            id=action_id,
+        )
         if isinstance(obj, Host) and action.host_action:
             objects = {'host': obj}
         else:

python/api/serializers.py

@@ -14,10 +14,11 @@
 from rest_framework.reverse import reverse
 from rest_framework_extensions.settings import extensions_api_settings
 
+from api.config.serializers import ConfigSerializerUI
 from api.utils import check_obj, hlink, UrlField
-from cm.adcm_config import ui_config
+from cm.adcm_config import ui_config, get_prototype_config, get_action_variant
 from cm.errors import raise_AdcmEx
-from cm.models import Upgrade, GroupConfig
+from cm.models import Upgrade, GroupConfig, Cluster, HostProvider, PrototypeConfig
 from cm.upgrade import do_upgrade
 
 
@@ -36,6 +37,36 @@ class UpgradeSerializer(serializers.Serializer):
     from_edition = serializers.JSONField(required=False)
     state_available = serializers.JSONField(required=False)
     state_on_success = serializers.CharField(required=False)
+    ui_options = serializers.SerializerMethodField()
+    config = serializers.SerializerMethodField()
+
+    def get_ui_options(self, instance):
+        if instance.action:
+            return instance.action.ui_options
+        return {}
+
+    def get_config(self, instance):
+        if instance.action is None:
+            return {'attr': {}, 'config': []}
+
+        if 'cluster_id' in self.context:
+            obj = check_obj(Cluster, self.context['cluster_id'])
+            proto = obj.prototype
+        elif 'provider_id' in self.context:
+            obj = check_obj(HostProvider, self.context['provider_id'])
+            proto = obj.prototype
+        else:
+            obj = None
+            proto = self.context['prototype']
+
+        action_conf = PrototypeConfig.objects.filter(
+            prototype=instance.action.prototype, action=instance.action
+        ).order_by('id')
+        _, _, _, attr = get_prototype_config(proto, instance.action)
+        if obj:
+            get_action_variant(obj, action_conf)
+        conf = ConfigSerializerUI(action_conf, many=True, context=self.context, read_only=True)
+        return {'attr': attr, 'config': conf.data}
 
 
 class UpgradeLinkSerializer(UpgradeSerializer):
@@ -50,10 +81,13 @@ def get_kwargs(self, obj):
 class DoUpgradeSerializer(serializers.Serializer):
     id = serializers.IntegerField(read_only=True)
     upgradable = serializers.BooleanField(read_only=True)
+    config = serializers.JSONField(required=False, default=dict)
+    task_id = serializers.IntegerField(read_only=True)
 
     def create(self, validated_data):
         upgrade = check_obj(Upgrade, validated_data.get('upgrade_id'), 'UPGRADE_NOT_FOUND')
-        return do_upgrade(validated_data.get('obj'), upgrade)
+        config = validated_data.get('config')
+        return do_upgrade(validated_data.get('obj'), upgrade, config)
 
 
 class StringListSerializer(serializers.ListField):

python/cm/adcm_schema.yaml

@@ -173,10 +173,40 @@ upgrade_obj:
     versions: version_dict
     states: states_dict
     from_edition: any_or_list
+    scripts: upgrade_task_list
+    hc_acl: action_hc_acl_list
+    venv: string
+    ui_options: json
+    config: config_obj
+
   required_items:
     - name
     - versions
 
+upgrade_task_list:
+  match: list
+  item: upgrade_task_action
+
+upgrade_task_action:
+  match: dict
+  items:
+    name: string
+    script: string
+    script_type: upgrade_script_type
+    display_name: string
+    params: json
+    on_fail: post_action_or_string
+  required_items:
+    - name
+    - script
+    - script_type
+
+upgrade_script_type:
+  match: set
+  variants:
+    - internal
+    - ansible
+
 version_dict:
   match: dict
   items:

python/cm/api.py

@@ -16,6 +16,7 @@
 from django.core.exceptions import MultipleObjectsReturned
 from django.db import transaction
 from django.utils import timezone
+from version_utils import rpm
 
 import cm.issue
 import cm.status_api
@@ -49,11 +50,35 @@
     PrototypeImport,
     ServiceComponent,
     TaskLog,
+    Bundle,
 )
-from cm.upgrade import check_license, version_in
+
 from rbac.models import re_apply_object_policy
 
 
+def check_license(bundle: Bundle) -> None:
+    if bundle.license == 'unaccepted':
+        msg = 'License for bundle "{}" {} {} is not accepted'
+        err('LICENSE_ERROR', msg.format(bundle.name, bundle.version, bundle.edition))
+
+
+def version_in(version: str, ver: PrototypeImport) -> bool:
+    # log.debug('version_in: %s < %s > %s', ver.min_version, version, ver.max_version)
+    if ver.min_strict:
+        if rpm.compare_versions(version, ver.min_version) <= 0:
+            return False
+    elif ver.min_version:
+        if rpm.compare_versions(version, ver.min_version) < 0:
+            return False
+    if ver.max_strict:
+        if rpm.compare_versions(version, ver.max_version) >= 0:
+            return False
+    elif ver.max_version:
+        if rpm.compare_versions(version, ver.max_version) > 0:
+            return False
+    return True
+
+
 def check_proto_type(proto, check_type):
     if proto.type != check_type:
         msg = 'Prototype type should be {}, not {}'

python/cm/bundle.py

@@ -444,6 +444,9 @@ def copy_stage_upgrade(stage_upgrades, bundle):
         )
         upg.bundle = bundle
         upgrades.append(upg)
+        if su.action:
+            prototype = Prototype.objects.get(name=su.action.prototype.name, bundle=bundle)
+            upg.action = Action.objects.get(prototype=prototype, name=su.action.name)
     Upgrade.objects.bulk_create(upgrades)
 
 
@@ -456,7 +459,7 @@ def prepare_bulk(origin_objects, Target, prototype, fields):
     return target_objects
 
 
-def copy_stage_actons(stage_actions, prototype):
+def copy_stage_actions(stage_actions, prototype):
     actions = prepare_bulk(
         stage_actions,
         Action,
@@ -559,7 +562,7 @@ def copy_stage_component(stage_components, stage_proto, prototype, bundle):
     Prototype.objects.bulk_create(componets)
     for sp in StagePrototype.objects.filter(type='component', parent=stage_proto):
         p = Prototype.objects.get(name=sp.name, type='component', parent=prototype, bundle=bundle)
-        copy_stage_actons(StageAction.objects.filter(prototype=sp), p)
+        copy_stage_actions(StageAction.objects.filter(prototype=sp), p)
         copy_stage_config(StagePrototypeConfig.objects.filter(prototype=sp), p)
 
 
@@ -639,7 +642,7 @@ def copy_stage(bundle_hash, bundle_proto):
 
     for sp in stage_prototypes:
         p = Prototype.objects.get(name=sp.name, type=sp.type, bundle=bundle)
-        copy_stage_actons(StageAction.objects.filter(prototype=sp), p)
+        copy_stage_actions(StageAction.objects.filter(prototype=sp), p)
         copy_stage_config(StagePrototypeConfig.objects.filter(prototype=sp), p)
         copy_stage_component(
             StagePrototype.objects.filter(parent=sp, type='component'), sp, p, bundle

python/cm/config.py

@@ -49,6 +49,8 @@
 
 ANSIBLE_VAULT_HEADER = '$ANSIBLE_VAULT;1.1;AES256'
 
+DEFAULT_SALT = b'"j\xebi\xc0\xea\x82\xe0\xa8\xba\x9e\x12E>\x11D'
+
 if os.path.exists(SECRETS_FILE):
     with open(SECRETS_FILE, encoding='utf_8') as f:
         data = json.load(f)

python/cm/errors.py

@@ -117,6 +117,7 @@
     'NOT_IMPLEMENTED': ("not implemented yet", rfs.HTTP_501_NOT_IMPLEMENTED, ERR),
     'NO_JOBS_RUNNING': ("no jobs running", rfs.HTTP_409_CONFLICT, ERR),
     'BAD_QUERY_PARAMS': ("bad query params", rfs.HTTP_400_BAD_REQUEST, ERR),
+    'WRONG_PASSWORD': ("Incorrect password during loading", rfs.HTTP_400_BAD_REQUEST, ERR),
     'DUMP_LOAD_CLUSTER_ERROR': (
         "Dumping or loading error with cluster",
         rfs.HTTP_409_CONFLICT,

python/cm/issue.py

@@ -119,7 +119,7 @@ def check_hc(cluster):
         for co in ClusterObject.objects.filter(cluster=cluster):
             for comp in Prototype.objects.filter(parent=co.prototype, type='component'):
                 const = comp.constraint
-                if len(const) == 2 and const[0] == 0 and const[1] == '+':
+                if len(const) == 2 and const[0] == 0 and (const[1] == '+' or const[1] == 'odd'):
                     continue
                 log.debug('void host components for %s', proto_ref(co.prototype))
                 return False

python/cm/job.py

@@ -18,13 +18,12 @@
 import subprocess
 from configparser import ConfigParser
 from datetime import timedelta, datetime
-from typing import List, Tuple, Optional, Hashable, Any
+from typing import List, Tuple, Optional, Hashable, Any, Union
 
 from background_task import background
 from django.db import transaction
 from django.utils import timezone
 
-from rbac.roles import re_apply_policy_for_jobs
 from cm import api, inventory, adcm_config, variant, config
 from cm.adcm_config import process_file_type
 from cm.api_context import ctx
@@ -53,6 +52,7 @@
     get_object_cluster,
 )
 from cm.status_api import post_event
+from rbac.roles import re_apply_policy_for_jobs
 
 
 def start_task(
@@ -654,12 +654,21 @@ def restore_hc(task: TaskLog, action: Action, status: str):
     api.save_hc(cluster, host_comp_list)
 
 
+def set_before_upgrade_state(action: Action, obj: Union[Cluster, HostProvider]) -> None:
+    """Save before state after upgrade"""
+    if action.upgrade is not None:
+        obj.before_upgrade['state'] = obj.state
+        obj.save()
+
+
 def finish_task(task: TaskLog, job: JobLog, status: str):
     action = task.action
     obj = task.task_object
     state, multi_state_set, multi_state_unset = get_state(action, job, status)
     with transaction.atomic():
         DummyData.objects.filter(id=1).update(date=timezone.now())
+        if hasattr(action, 'upgrade'):
+            set_before_upgrade_state(action, obj)
         set_action_state(action, task, obj, state, multi_state_set, multi_state_unset)
         restore_hc(task, action, status)
         task.unlock_affected()

python/cm/management/commands/dumpcluster.py

@@ -14,10 +14,17 @@
 
 import json
 import sys
+import base64
+import getpass
+from cryptography.fernet import Fernet
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
 
+from cm.config import ANSIBLE_SECRET, DEFAULT_SALT
 from cm.models import (
     Bundle,
     Cluster,
@@ -290,6 +297,21 @@ def get_host_component(host_component_id):
     return host_component
 
 
+def encrypt_data(pass_from_user, result):
+    password = pass_from_user.encode()
+    kdf = PBKDF2HMAC(
+        algorithm=hashes.SHA256(),
+        length=32,
+        salt=DEFAULT_SALT,
+        iterations=390000,
+        backend=default_backend(),
+    )
+    key = base64.urlsafe_b64encode(kdf.derive(password))
+    f = Fernet(key)
+    encrypted = f.encrypt(result)
+    return encrypted
+
+
 def dump(cluster_id, output):
     """
     Saving objects to file in JSON format
@@ -355,14 +377,17 @@ def dump(cluster_id, output):
     ):
         host_component = get_host_component(host_component_obj.id)
         data['host_components'].append(host_component)
-
-    result = json.dumps(data, indent=2)
+    data['adcm_password'] = ANSIBLE_SECRET
+    result = json.dumps(data, indent=2).encode('utf-8')
+    password = getpass.getpass()
+    encrypted = encrypt_data(password, result)
 
     if output is not None:
-        with open(output, 'w', encoding='utf_8') as f:
-            f.write(result)
+        with open(output, 'wb') as f:
+            f.write(encrypted)
+        sys.stdout.write(f'Dump successfully done to file {output}\n')
     else:
-        sys.stdout.write(result)
+        sys.stdout.write(encrypted.decode('utf8'))
 
 
 class Command(BaseCommand):