Merge pull request #235 from indigo-dc/devel

Devel

IM/InfrastructureInfo.py

@@ -306,17 +306,16 @@ def update_radl(self, radl, deployed_vms):
         """
 
         with self._lock:
-            # Add new networks only
-            for s in radl.systems + radl.networks + radl.ansible_hosts:
+            original_radl = self.radl.clone()
+            # Add new networks ad ansible_hosts only
+            for s in radl.networks + radl.ansible_hosts:
                 if not self.radl.add(s.clone(), "ignore"):
-                    InfrastructureInfo.logger.warn(
-                        "Ignoring the redefinition of %s %s" % (type(s), s.getId()))
+                    InfrastructureInfo.logger.warn("Ignoring the redefinition of %s %s" % (type(s), s.getId()))
 
-            # Add or update configures
+            # Add or update configures and systems
             for s in radl.configures + radl.systems:
                 self.radl.add(s.clone(), "replace")
-                InfrastructureInfo.logger.warn(
-                    "(Re)definition of %s %s" % (type(s), s.getId()))
+                InfrastructureInfo.logger.warn("(Re)definition of %s %s" % (type(s), s.getId()))
 
             # Append contextualize
             self.radl.add(radl.contextualize)
@@ -335,7 +334,12 @@ def update_radl(self, radl, deployed_vms):
                         self.private_networks[private_net] = d.cloud_id
 
         # Check the RADL
-        self.radl.check()
+        try:
+            self.radl.check()
+        except Exception as ex:
+            # If something is not correct restore the original one and raise the error
+            self.radl = original_radl
+            raise(ex)
 
     def complete_radl(self, radl):
         """

IM/connectors/Azure.py

@@ -26,6 +26,7 @@
 try:
     from azure.mgmt.resource import ResourceManagementClient
     from azure.mgmt.storage import StorageManagementClient
+    from azure.storage.blob import BlockBlobService
     from azure.mgmt.compute import ComputeManagementClient
     from azure.mgmt.network import NetworkManagementClient
     from azure.mgmt.dns import DnsManagementClient
@@ -405,11 +406,12 @@ def get_azure_vm_create_json(self, storage_account, vm_name, nics, radl, instanc
         if system.getValue('availability_zone'):
             location = system.getValue('availability_zone')
 
-        # Allways use the new credentials
+        # Always use the new credentials
         system.updateNewCredentialValues()
         user_credentials = system.getCredentials()
 
         os_disk_name = "osdisk-" + str(uuid.uuid1())
+        disks = [vm_name + os_disk_name + ".vhd"]
 
         vm = {
             'location': location,
@@ -443,10 +445,23 @@ def get_azure_vm_create_json(self, storage_account, vm_name, nics, radl, instanc
             },
         }
 
+        tags = {}
+        if system.getValue('instance_tags'):
+            keypairs = system.getValue('instance_tags').split(",")
+            for keypair in keypairs:
+                parts = keypair.split("=")
+                key = parts[0].strip()
+                value = parts[1].strip()
+                tags[key] = value
+
+        if tags:
+            vm['tags'] = tags
+
         cont = 1
         data_disks = []
         while system.getValue("disk." + str(cont) + ".size"):
             disk_size = system.getFeature("disk." + str(cont) + ".size").getValue('G')
+            disks.append("{}disk{}.vhd".format(vm_name, cont))
             self.log_info("Adding a %s GB disk." % disk_size)
             data_disks.append({
                 'name': '%s_disk_%d' % (vm_name, cont),
@@ -463,7 +478,7 @@ def get_azure_vm_create_json(self, storage_account, vm_name, nics, radl, instanc
         if data_disks:
             vm['storage_profile']['data_disks'] = data_disks
 
-        return vm
+        return vm, disks
 
     def create_nets(self, radl, credentials, subscription_id, group_name):
         network_client = NetworkManagementClient(credentials, subscription_id)
@@ -529,17 +544,31 @@ def create_vms(self, inf, radl, requested_radl, num_vm, location, storage_accoun
             group_name = "rg-%s" % (vm_name)
 
             try:
+                tags = {}
+                if radl.systems[0].getValue('instance_tags'):
+                    keypairs = radl.systems[0].getValue('instance_tags').split(",")
+                    for keypair in keypairs:
+                        parts = keypair.split("=")
+                        key = parts[0].strip()
+                        value = parts[1].strip()
+                        tags[key] = value
+
+                args = {'location': location}
+                if tags:
+                    args['tags'] = tags
+
                 # Create resource group for the VM
-                resource_client.resource_groups.create_or_update(group_name, {'location': location})
+                resource_client.resource_groups.create_or_update(group_name, args)
 
                 vm = VirtualMachine(inf, group_name + '/' + vm_name, self.cloud, radl, requested_radl, self)
                 vm.info.systems[0].setValue('instance_id', group_name + '/' + vm_name)
 
                 nics = self.create_nics(radl, credentials, subscription_id, group_name, subnets)
 
                 instance_type = self.get_instance_type(radl.systems[0], credentials, subscription_id)
-                vm_parameters = self.get_azure_vm_create_json(storage_account_name, vm_name,
-                                                              nics, radl, instance_type)
+                vm_parameters, disks = self.get_azure_vm_create_json(storage_account_name, vm_name,
+                                                                     nics, radl, instance_type)
+                vm.disks = disks
 
                 compute_client = ComputeManagementClient(credentials, subscription_id)
                 async_vm_creation = compute_client.virtual_machines.create_or_update(group_name,
@@ -556,11 +585,21 @@ def create_vms(self, inf, radl, requested_radl, num_vm, location, storage_accoun
                 # Delete Resource group and everything in it
                 if group_name:
                     self.delete_resource_group(group_name, resource_client)
+                    self.delete_vm_disks(vm, credentials, subscription_id)
 
             i += 1
 
         return vms
 
+    @staticmethod
+    def get_storage_account_name(inf_id):
+        # Storage account name must be between 3 and 24 characters in length and use
+        # numbers and lower-case letters only
+        storage_account_name = "s%s" % inf_id
+        storage_account_name = storage_account_name.replace("-", "")
+        storage_account_name = storage_account_name[:24]
+        return storage_account_name
+
     def launch(self, inf, radl, requested_radl, num_vm, auth_data):
         location = self.DEFAULT_LOCATION
         if radl.systems[0].getValue('availability_zone'):
@@ -572,11 +611,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
 
         resource_client = ResourceManagementClient(credentials, subscription_id)
 
-        # Storage account name must be between 3 and 24 characters in length and use
-        # numbers and lower-case letters only
-        storage_account_name = "s%s" % inf.id
-        storage_account_name = storage_account_name.replace("-", "")
-        storage_account_name = storage_account_name[:24]
+        storage_account_name = self.get_storage_account_name(inf.id)
 
         with inf._lock:
             # Create resource group for the Infrastructure if it does not exists
@@ -646,6 +681,14 @@ def updateVMInfo(self, vm, auth_data):
             compute_client = ComputeManagementClient(credentials, subscription_id)
             # Get one the virtual machine by name
             virtual_machine = compute_client.virtual_machines.get(group_name, vm_name, expand='instanceView')
+
+            if virtual_machine.storage_profile.data_disks:
+                for data_disk in virtual_machine.storage_profile.data_disks:
+                    self.log_debug(data_disk.vhd.name)
+
+            if virtual_machine.storage_profile.os_disk:
+                self.log_debug(virtual_machine.storage_profile.os_disk.name)
+
         except Exception as ex:
             self.log_warn("The VM does not exists.")
             # check if the RG still exists
@@ -778,6 +821,8 @@ def finalize(self, vm, last, auth_data):
                         return False, "Error terminating the VM: %s" % msg
                 else:
                     self.log_info("RG: %s does not exist. Do not remove." % "rg-%s" % vm.inf.id)
+            else:
+                self.delete_vm_disks(vm, credentials, subscription_id)
 
         except Exception as ex:
             self.log_exception("Error terminating the VM")
@@ -865,3 +910,29 @@ def delete_resource_group(self, group_name, resource_client, max_retries=3):
             self.log_info("Resource group %s successfully deleted." % group_name)
 
         return deleted, msg
+
+    def delete_vm_disks(self, vm, credentials, subscription_id):
+        try:
+            if "disks" in vm.__dict__.keys():
+                storage_account_name = self.get_storage_account_name(vm.inf.id)
+
+                storage_client = StorageManagementClient(credentials, subscription_id)
+                keys = storage_client.storage_accounts.list_keys("rg-%s" % vm.inf.id, storage_account_name)
+
+                key = None
+                for key in keys.keys:
+                    break
+                if not key:
+                    self.log_error("Error deleting VM disks: No key found.")
+                    return (False, "Error deleting VM disks: No key found.")
+
+                block_blob_service = BlockBlobService(account_name=storage_account_name, account_key=key.value)
+
+                for disk in vm.disks:
+                    self.log_debug("Deleting disk: %s" % disk)
+                    block_blob_service.delete_blob("vhds", disk)
+        except Exception as ex:
+            self.log_exception("Error deleting VM disks")
+            return (False, "Error deleting VM disks" + str(ex))
+
+        return True, ""

IM/connectors/EC2.py

@@ -596,6 +596,15 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
                     res.append((False, "Error managing the keypair."))
                 return res
 
+            tags = {}
+            if system.getValue('instance_tags'):
+                keypairs = system.getValue('instance_tags').split(",")
+                for keypair in keypairs:
+                    parts = keypair.split("=")
+                    key = parts[0].strip()
+                    value = parts[1].strip()
+                    tags[key] = value
+
             all_failed = True
 
             i = 0
@@ -711,6 +720,8 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
                             if len(reservation.instances) == 1:
                                 instance = reservation.instances[0]
                                 instance.add_tag("IM-USER", im_username)
+                                for key, value in tags.items():
+                                    instance.add_tag(key, value)
                                 ec2_vm_id = region_name + ";" + instance.id
 
                                 self.log_debug("RADL:")
@@ -796,7 +807,7 @@ def attach_volumes(self, instance, vm):
         """
         try:
             if instance.state == 'running' and "volumes" not in vm.__dict__.keys():
-                # Flag to se that this VM has created (or is creating) the
+                # Flag to set that this VM has created (or is creating) the
                 # volumes
                 vm.volumes = True
                 conn = instance.connection

IM/connectors/GCE.py

@@ -456,6 +456,16 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
                 'external_ip': None,
                 'location': region}
 
+        tags = {}
+        if system.getValue('instance_tags'):
+            keypairs = system.getValue('instance_tags').split(",")
+            for keypair in keypairs:
+                parts = keypair.split("=")
+                key = parts[0].strip()
+                value = parts[1].strip()
+                tags[key] = value
+            args['ex_metadata'] = tags
+
         # include the SSH_KEYS
         username = system.getValue('disk.0.os.credentials.username')
         private = system.getValue('disk.0.os.credentials.private_key')

IM/connectors/OpenNebula.py

@@ -646,6 +646,18 @@ def getONETemplate(self, radl, sgs, auth_data):
             %s
         ''' % (name, cpu, cpu, memory, arch, disks, ConfigOpenNebula.TEMPLATE_OTHER)
 
+        user_template = ""
+        if system.getValue('instance_tags'):
+            keypairs = system.getValue('instance_tags').split(",")
+            for keypair in keypairs:
+                parts = keypair.split("=")
+                key = parts[0].strip()
+                value = parts[1].strip()
+                user_template += '%s = "%s", ' % (key, value)
+
+        if user_template:
+            res += "\nUSER_TEMPLATE = [%s]\n" % user_template[:-2]
+
         res += self.get_networks_template(radl, sgs, auth_data)
 
         # include the SSH_KEYS

IM/connectors/OpenStack.py

@@ -534,6 +534,16 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
                 'ex_security_groups': sgs,
                 'name': "%s-%s" % (name, int(time.time() * 100))}
 
+        tags = {}
+        if system.getValue('instance_tags'):
+            keypairs = system.getValue('instance_tags').split(",")
+            for keypair in keypairs:
+                parts = keypair.split("=")
+                key = parts[0].strip()
+                value = parts[1].strip()
+                tags[key] = value
+            args['ex_metadata'] = tags
+
         keypair = None
         keypair_name = None
         keypair_created = False

ansible_install.yaml

@@ -50,6 +50,9 @@
       yum: name=IM,ansible,python-pip,python-jinja2,sshpass,openssh-clients,unzip,MySQL-python,python-sqlite3dbm
       when: ansible_os_family == "RedHat"
 
+    - name: Install Azure libraries
+      pip: executable=pip name=msrest,msrestazure,azure-common,azure-mgmt-storage,azure-mgmt-compute,azure-mgmt-network,azure-mgmt-resource,azure-mgmt-dns,azure-storage
+
 ################################################ Configure Ansible  ###################################################
 
     - name: Create /etc/ansible

changelog

@@ -416,3 +416,5 @@ IM 1.7.1:
 IM 1.7.2:
     * Fix Error setting the INF_CACHE_TIME conf variable time.
     * Add support to availability_zone in tosca.policies.Placement.
+    * Enable to set instance_tags in connectors.
+    * Fix error in Azure conn: VM disks are not deleted when VM is finalized, only when Infrastructure is destroyed.

doc/source/conf.py

@@ -39,6 +39,7 @@
     'sphinx.ext.autodoc',
     'sphinx.ext.mathjax',
     'sphinx.ext.viewcode',
+    'sphinx.ext.graphviz'
 ]
 
 # Math

doc/source/manual.rst

@@ -125,7 +125,7 @@ Then install the downloaded RPMs::
 
 Azure python SDK is not available in CentOS. So if you need the Azure plugin you have to manually install them using pip::
 
-	$ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns
+	$ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage
 
 From Deb package (Tested with Ubuntu 14.04 and 16.04)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

doc/source/xmlrpc.rst

@@ -62,6 +62,17 @@ This is the list of method names:
    Return the aggregated state associated to the 
    infrastructure with ID ``infId``. 
 
+   The ``state`` can be
+
+   * ``pending``, At least one VM is still in initialization stage;
+   * ``running``, All the VMs are created successfully and running, but at least one of them are still in the configuration stage;
+   * ``configured``, All the VMs are running and contextualized;
+   * ``unconfigured``, All the VMs are running but at least one of them are not correctly contextualized;
+   * ``stopped``, All the VMs are stopped or suspended;
+   * ``off``, All the VMs are shutdown or removed from the infrastructure;
+   * ``failed``, There are at least one VM in status ``failed``.
+   * ``unknown``, There are at least one VM in status ``unknown``.
+
 ``GetInfrastructureRADL``
    :parameter 0: ``infId``: integer
    :parameter 1: ``auth``: array of structs
@@ -94,7 +105,8 @@ This is the list of method names:
    * ``failed``, an error happened during the launching; or
    * ``unknown``, unable to obtain the status.
 
-   The next figure shows a state diagram of virtual machine status.
+   The next figure shows a state diagram of virtual machine status. This figure is illustrative
+   as if may differ in case of Cloud Providers.
 
    .. digraph:: stategraph
 

docker-py3/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install setuptools pip --upgrade -I
 RUN pip3 install pyOpenSSL --upgrade -I
 
 # Install pip optional libraries
-RUN pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns cheroot xmltodict
+RUN pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage cheroot xmltodict
 
 # Install IM
 RUN apt-get update && apt-get install --no-install-recommends -y gcc libssl-dev libffi-dev libsqlite3-dev && \

docker/Dockerfile

@@ -22,7 +22,7 @@ RUN wget https://launchpad.net/ubuntu/+archive/primary/+files/python-msrest_0.4.
     rm -rf /var/lib/apt/lists/*
 
 # Install IM
-RUN wget https://github.com/grycap/RADL/releases/download/v1.1.0/python-radl_1.1.0-1_all.deb && \ 
+RUN wget https://github.com/grycap/RADL/releases/download/v1.1.1/python-radl_1.1.1-1_all.deb && \ 
     wget https://github.com/indigo-dc/im/releases/download/v1.7.2/python-im_1.7.2-1_all.deb && \
     wget https://github.com/indigo-dc/tosca-parser/releases/download/0.9.1/python-tosca-parser_0.9.1-1_all.deb && \
     dpkg -i python-radl_*_all.deb ; \

setup.py

@@ -63,6 +63,6 @@
       description="IM is a tool to manage virtual infrastructures on Cloud deployments",
       platforms=["any"],
       install_requires=["ansible >= 2.0", "paramiko >= 1.14", "PyYAML", suds_pkg, sqlite_pkg, "cheroot",
-                        "boto >= 2.29", "apache-libcloud >= 2.3.0", "RADL >= 1.1.0", "bottle", "netaddr",
+                        "boto >= 2.29", "apache-libcloud >= 2.3.0", "RADL >= 1.1.1", "bottle", "netaddr",
                         "requests", "scp", "tosca-parser"]
       )