Merge pull request #81 from dell/91_replication_enhancements

SRDF Volume add and Remove + Secure Snap

Author: rawstorage

ChangeLog

@@ -1,6 +1,27 @@
 PyU4V Change Log
 ================
 
+Version 9.1.2.0 - released 19/12/19
+===================================
+Provisioning Enhancements
+-------------------------
+- add_new_volume_to_storage_group : added functionality to be able to add
+  volumes to SRDF protected storage groups, new volumes will automatically be
+  protected with SRDF, volumes are added to the remote storage group to fully
+  automate the provisioning and simplify operations. Storage group must be
+  local to the unisphere instance and only works with storage groups that
+  contain R1 or R11 devices.
+- remove_volume_from_storage_group enhanced to be able to automatically remove
+  volumes from local and remote storage groups deleting srdf pairs as part of
+  the process, no need to suspend, the automation takes care of everything,
+  storage group must be local to the unisphere instance and only works with
+  storage groups that contain R1 or R11 devices.
+
+Replication Enhancements
+-------------------------
+- create_storage_group_snapshot added secure option to enable creation of
+  secure snapshot
+
 Version 9.1.1.0 - released 12/12/19
 ===================================
 
@@ -86,6 +107,7 @@ Provisioning Enhancements
 - format_director_port : Format separate director port into single string
 - get_any_director_port : Get a non-GuestOS port from a director
 
+
 Fixes
 -----
 - When searching for volumes by volume name, if more than one volume matches
@@ -331,6 +353,7 @@ PyU4V/replication.py
 - delete_storagegroup_srdf() has been refactored and marked for deprecation,
   please use alternative function delete_storage_group_srdf()
 
+
 PyU4V/utils/console.py
 ----------------------
 - choose_from_list() has been marked for deprecation, please implement any

PyU4V/__init__.py

@@ -23,7 +23,7 @@
 from .univmax_conn import U4VConn  # noqa: F401
 
 __title__ = 'pyu4v'
-__version__ = '9.1.1.0'
+__version__ = '9.1.2.0'
 __author__ = 'Dell EMC or its subsidiaries'
 __license__ = 'Apache 2.0'
 __copyright__ = 'Copyright 2019 Dell EMC Inc'

PyU4V/provisioning.py

@@ -1773,7 +1773,7 @@ def add_new_vol_to_storagegroup(self, sg_id, num_vols, vol_size,
         :param cap_unit: capacity unit (MB, GB, TB, CYL) -- str
         :param _async: if call should be async -- bool
         :param vol_name: name to give to the volume, optional -- str
-        :param create_new_volumes: new volumes only, no ro-use -- bool
+        :param create_new_volumes: new volumes only, no re-use -- bool
         :returns: storage group details -- dict
         """
         return self.add_new_volume_to_storage_group(
@@ -1782,7 +1782,9 @@ def add_new_vol_to_storagegroup(self, sg_id, num_vols, vol_size,
 
     def add_new_volume_to_storage_group(
             self, storage_group_id, num_vols, vol_size, cap_unit, _async=False,
-            vol_name=None, create_new_volumes=None):
+            vol_name=None, create_new_volumes=None, remote_array_1_id=None,
+            remote_array_1_sgs=None, remote_array_2_id=None,
+            remote_array_2_sgs=None):
         """Expand an existing storage group by adding new volumes.
 
         :param storage_group_id: storage group id -- str
@@ -1792,6 +1794,17 @@ def add_new_volume_to_storage_group(
         :param _async: if call should be async -- bool
         :param vol_name: name to give to the volume, optional -- str
         :param create_new_volumes: new volumes only, no ro-use -- bool
+        :param remote_array_1_id: 12 digit serial number of remote array,
+               optional -- str
+        :param remote_array_1_sgs: list of storage groups on remote array to
+               add Remote device, Unisphere instance must be local to R1
+               storage group otherwise volumes will only be added to the
+               local group -- str or list
+        :param remote_array2_id: optional digit serial number of remote array,
+               only used in multihop SRDF, e.g. R11, or R1 - R21 - R2 optional
+               -- str
+        :param remote_array2_sgs: storage groups on remote array, optional
+               -- str or list
         :returns: storage group details -- dict
         """
         add_volume_param = {'emulation': 'FBA'}
@@ -1817,7 +1830,18 @@ def add_new_volume_to_storage_group(
         expand_sg_data = ({'editStorageGroupActionParam': {
             'expandStorageGroupParam': {
                 'addVolumeParam': add_volume_param}}})
-
+        if remote_array_1_id and remote_array_1_sgs:
+            if not isinstance(remote_array_1_sgs, list):
+                remote_array_1_sgs = [remote_array_1_sgs]
+            add_volume_param.update({'remoteSymmSGInfoParam': {
+                'remote_symmetrix_1_id': remote_array_1_id,
+                'remote_symmetrix_1_sgs': remote_array_1_sgs}})
+            if remote_array_2_id and remote_array_2_sgs:
+                if not isinstance(remote_array_2_sgs, list):
+                    remote_array_2_sgs = [remote_array_2_sgs]
+                add_volume_param['remoteSymmSGInfoParam'].update({
+                    'remote_symmetrix_2_id': remote_array_2_id,
+                    'remote_symmetrix_2_sgs': remote_array_2_sgs})
         if _async:
             expand_sg_data.update(ASYNC_UPDATE)
         return self.modify_storage_group(storage_group_id, expand_sg_data)
@@ -1841,19 +1865,57 @@ def remove_vol_from_storagegroup(self, sg_id, vol_id, _async=False):
         """
         return self.remove_volume_from_storage_group(sg_id, vol_id, _async)
 
-    def remove_volume_from_storage_group(self, storage_group_id, vol_id,
-                                         _async=False):
+    def remove_volume_from_storage_group(
+            self, storage_group_id, vol_id, _async=False,
+            remote_array_1_id=None, remote_array_1_sgs=None,
+            remote_array_2_id=None, remote_array_2_sgs=None):
         """Remove a volume from a given storage group.
 
         :param storage_group_id: storage group id -- str
         :param vol_id: device id -- str
         :param _async: if call should be async -- bool
+        :param remote_array_1_id: 12 digit serial number of remote array,
+               optional -- str
+        :param remote_array_1_sgs: list of storage groups on remote array to
+               add Remote device, Unisphere instance must be local to R1
+               storage group otherwise volumes will only be added to the
+               local group -- str or list
+        :param remote_array2_id: optional digit serial number of remote array,
+               only used in multihop SRDF, e.g. R11, or R1 - R21 - R2 optional
+               -- str
+        :param remote_array2_sgs: storage groups on remote array, optional
+               -- str or list
         :returns: storage group details -- dict
         """
         if not isinstance(vol_id, list):
             vol_id = [vol_id]
         payload = ({'editStorageGroupActionParam': {
             'removeVolumeParam': {'volumeId': vol_id}}})
+
+        if remote_array_1_id and remote_array_1_sgs:
+            if not isinstance(remote_array_1_sgs, list):
+                remote_array_1_sgs = [remote_array_1_sgs]
+            payload.update(
+                {'editStorageGroupActionParam': {
+                    'removeVolumeParam': {
+                        'volumeId': vol_id,
+                        'remoteSymmSGInfoParam': {
+                            'remote_symmetrix_1_id': remote_array_1_id,
+                            'remote_symmetrix_1_sgs': remote_array_1_sgs}}}
+                 })
+            if remote_array_2_id and remote_array_2_sgs:
+                if not isinstance(remote_array_2_sgs, list):
+                    remote_array_2_sgs = [remote_array_2_sgs]
+                payload.update(
+                    {'editStorageGroupActionParam': {
+                        'removeVolumeParam': {
+                            'volumeId': vol_id,
+                            'remoteSymmSGInfoParam': {
+                                'remote_symmetrix_1_id': remote_array_1_id,
+                                'remote_symmetrix_1_sgs': remote_array_1_sgs,
+                                'remote_symmetrix_2_id': remote_array_2_id,
+                                'remote_symmetrix_2_sgs': remote_array_1_sgs
+                            }}}})
         if _async:
             payload.update(ASYNC_UPDATE)
         return self.modify_storage_group(storage_group_id, payload)

PyU4V/replication.py

@@ -215,7 +215,8 @@ def create_storagegroup_snap(
                                                   hours)
 
     def create_storage_group_snapshot(
-            self, storage_group_id, snap_name, ttl=None, hours=False):
+            self, storage_group_id, snap_name, ttl=None, hours=False,
+            secure=False):
         """Create a snapVx snapshot of a storage group.
 
         To establish a new generation of an existing SnapVX snapshot for a
@@ -226,11 +227,16 @@ def create_storage_group_snapshot(
         :param snap_name: snapshot name -- str
         :param ttl: Time To Live -- str
         :param hours: if TTL is in hours instead of days -- bool
+        :param secure: sets secure snapshot, snapshot created with secure
+               option can not be deleted before ttl expires -- bool
         :returns: snapshot details -- dict
         """
         payload = {'snapshotName': snap_name}
         if ttl:
-            payload.update({'timeToLive': ttl})
+            if secure:
+                payload.update({'secure': ttl})
+            else:
+                payload.update({'timeToLive': ttl})
             if hours:
                 payload.update({'timeInHours': 'True'})
         return self.create_resource(

PyU4V/tests/ci_tests/base.py

@@ -152,7 +152,7 @@ def create_rdf_sg(self):
             srdf_mode='Synchronous', establish=True,
             _async=True)
         self.conn.common.wait_for_job_complete(job)
-        local_volume = self.provision.get_vols_from_storagegroup(
+        local_volume = self.provision.get_volumes_from_storage_group(
             sg_name)[0]
         srdf_group_number = (
             self.replication.get_storage_group_srdf_group_list(
@@ -177,15 +177,15 @@ def cleanup_rdfg(self, sg_name, srdf_group_number):
         if 'Synchronized' in current_rdf_state_list:
             self.replication.suspend_storage_group_srdf(
                 storage_group_id=sg_name, srdf_group_number=srdf_group_number)
-        local_volume_list = self.provision.get_vols_from_storagegroup(
+        local_volume_list = self.provision.get_volumes_from_storage_group(
             sg_name)
         self.replication.delete_storage_group_srdf(
             storage_group_id=sg_name)
         self.provision.delete_storage_group(storage_group_id=sg_name)
         for local_volume in local_volume_list:
             self.provision.delete_volume(device_id=local_volume)
         self.conn.set_array_id(array_id=self.conn.remote_array)
-        remote_volume_list = self.provision.get_vols_from_storagegroup(
+        remote_volume_list = self.provision.get_volumes_from_storage_group(
             sg_name)
         self.provision.delete_storage_group(storage_group_id=sg_name)
         for remote_volume in remote_volume_list:

PyU4V/tests/ci_tests/test_pyu4v_ci_provisioning.py

@@ -1927,6 +1927,18 @@ def test_add_new_vol_to_storagegroup(self):
         for volume in volume_list:
             self.addCleanup(self.delete_volume, storage_group_name, volume)
 
+    def test_add_new_vol_to_srdf_storage_group(self):
+        """Tests adding a volume to a storage group that is replicated"""
+        sg_name, srdf_group_number, device_id, remote_volume = (
+            self.create_rdf_sg())
+        self.provisioning.add_new_volume_to_storage_group(
+            storage_group_id=sg_name, vol_size=1, num_vols=1, cap_unit='GB',
+            remote_array_1_id=self.conn.remote_array,
+            remote_array_1_sgs=[sg_name])
+        storage_group_details = self.provisioning.get_storage_group(
+            storage_group_name=sg_name)
+        self.assertEqual(2, storage_group_details[constants.NUM_OF_VOLS])
+
     def test_remove_volume_from_storage_group(self):
         """Test remove_volume_from_storage_group."""
         volume_name = self.generate_name()
@@ -1944,6 +1956,25 @@ def test_remove_volume_from_storage_group(self):
         self.provisioning.add_existing_volume_to_storage_group(
             storage_group_name, device_id)
 
+    def test_remove_volume_from_srdf_storage_group(self):
+        """Tests adding a volume to a storage group that is replicated"""
+        sg_name, srdf_group_number, device_id, remote_volume = (
+            self.create_rdf_sg())
+        self.provisioning.add_new_volume_to_storage_group(
+            storage_group_id=sg_name, vol_size=1, num_vols=1, cap_unit='GB',
+            remote_array_1_id=self.conn.remote_array,
+            remote_array_1_sgs=[sg_name])
+        storage_group_details = self.provisioning.get_storage_group(
+            storage_group_name=sg_name)
+        self.assertEqual(2, storage_group_details[constants.NUM_OF_VOLS])
+        self.provisioning.remove_volume_from_storage_group(
+            storage_group_id=sg_name, vol_id=device_id,
+            remote_array_1_id=self.conn.remote_array,
+            remote_array_1_sgs=[sg_name])
+        storage_group_details = self.provisioning.get_storage_group(
+            storage_group_name=sg_name)
+        self.assertEqual(1, storage_group_details[constants.NUM_OF_VOLS])
+
     def test_remove_vol_from_storagegroup(self):
         """Test remove_vol_from_storagegroup."""
         volume_name = self.generate_name()

PyU4V/tests/unit_tests/pyu4v_common_data.py

@@ -20,6 +20,7 @@ class CommonData(object):
 
     array = '000197800123'
     remote_array = '000197800124'
+    remote_array2 = '000197800125'
     srp = 'SRP_1'
     slo = 'Diamond'
     workload = 'DSS'

PyU4V/tests/unit_tests/test_pyu4v_provisioning.py

@@ -1455,6 +1455,39 @@ def test_add_new_vol_to_storage_group_name_async(self):
             mock_mod.assert_called_once_with(
                 self.data.storagegroup_name, payload)
 
+    def test_add_new_vol_to_storage_group_srdf_multihop_srdf(self):
+        """Test adding new volume to replicated storage group."""
+        remote_array = '000197800124'
+        remote_array2 = '000197800125'
+        num_of_volumes = 1
+        volume_size = 10
+        payload = {'editStorageGroupActionParam': {
+            'expandStorageGroupParam': {
+                'addVolumeParam': {
+                    'emulation': 'FBA',
+                    'create_new_volumes': False,
+                    'volumeAttributes': [{
+                        'num_of_vols': 1,
+                        'volume_size': 10,
+                        'capacityUnit': 'GB'}],
+                    'remoteSymmSGInfoParam': {
+                        'remote_symmetrix_1_id': remote_array,
+                        'remote_symmetrix_1_sgs': ['PU-mystoragegroup-SG'],
+                        'remote_symmetrix_2_id': remote_array2,
+                        'remote_symmetrix_2_sgs': ['PU-mystoragegroup-SG']
+                    }}}}}
+        with mock.patch.object(
+                self.provisioning, 'modify_storage_group') as mock_mod:
+            self.provisioning.add_new_volume_to_storage_group(
+                storage_group_id=self.data.storagegroup_name,
+                num_vols=num_of_volumes, vol_size=volume_size, cap_unit='GB',
+                remote_array_1_id=self.data.remote_array,
+                remote_array_1_sgs=self.data.storagegroup_name,
+                remote_array_2_id=self.data.remote_array2,
+                remote_array_2_sgs=self.data.storagegroup_name)
+            mock_mod.assert_called_once_with(
+                self.data.storagegroup_name, payload)
+
     def test_remove_vol_from_storage_group_volume_string(self):
         """Test remove_vol_from_storage_group single volume."""
         payload = {'editStorageGroupActionParam': {
@@ -1468,6 +1501,31 @@ def test_remove_vol_from_storage_group_volume_string(self):
             mock_mod.assert_called_once_with(
                 self.data.storagegroup_name, payload)
 
+    def test_remove_vol_from_replicated_storage_group_multihop(self):
+        """Test remove_vol_from_storage_group single volume."""
+        payload = {
+            'editStorageGroupActionParam': {
+                'removeVolumeParam': {
+                    'volumeId': [self.data.device_id],
+                    'remoteSymmSGInfoParam': {
+                        'remote_symmetrix_1_id': self.data.remote_array,
+                        'remote_symmetrix_1_sgs': [
+                            self.data.storagegroup_name],
+                        'remote_symmetrix_2_id': self.data.remote_array2,
+                        'remote_symmetrix_2_sgs': [self.data.storagegroup_name]
+                    }}}}
+        with mock.patch.object(
+                self.provisioning, 'modify_storage_group') as mock_mod:
+            self.provisioning.remove_volume_from_storage_group(
+                storage_group_id=self.data.storagegroup_name,
+                vol_id=self.data.device_id,
+                remote_array_1_id=self.data.remote_array,
+                remote_array_1_sgs=self.data.storagegroup_name,
+                remote_array_2_id=self.data.remote_array2,
+                remote_array_2_sgs=self.data.storagegroup_name)
+            mock_mod.assert_called_once_with(
+                self.data.storagegroup_name, payload)
+
     def test_remove_vol_from_storage_group_volume_list(self):
         """Test remove_vol_from_storage_group multiple volumes."""
         payload = {'executionOption': 'ASYNCHRONOUS',

PyU4V/tests/unit_tests/test_pyu4v_replication.py

@@ -146,6 +146,17 @@ def test_create_storage_group_snap(self):
                 self.data.storagegroup_name, 'snap_name', ttl=2, hours=True)
             self.assertEqual(2, mock_create.call_count)
 
+    def test_create_storage_group_snapshot_secure(self):
+        """Test create_storage_group_snapshot."""
+        with mock.patch.object(
+                self.replication, 'create_resource') as mock_create:
+            self.replication.create_storage_group_snapshot(
+                self.data.storagegroup_name, 'snap_name')
+            self.replication.create_storage_group_snapshot(
+                self.data.storagegroup_name, 'snap_name', ttl=2, hours=True,
+                secure=True)
+            self.assertEqual(2, mock_create.call_count)
+
     def test_get_storagegroup_snapshot_generation_list(self):
         """Test get_storagegroup_snapshot_generation_list."""
         gen_list = self.replication.get_storagegroup_snapshot_generation_list(

PyU4V/utils/constants.py

@@ -30,7 +30,7 @@
 DELETE = 'DELETE'
 
 # Unisphere REST URI constants
-PYU4V_VERSION = '9.1.1.0'
+PYU4V_VERSION = '9.1.2.0'
 UNISPHERE_VERSION = '91'
 VERSION = 'version'
 ITERATOR = 'Iterator'

README.rst

@@ -26,7 +26,7 @@ PyU4V Version 9.1
 +-----------------------+----------------------------+
 | **Author**            | Dell EMC                   |
 +-----------------------+----------------------------+
-| **PyU4V Version**     | 9.1.1.0                    |
+| **PyU4V Version**     | 9.1.2.0                    |
 +-----------------------+----------------------------+
 | **Unisphere Version** | 9.1.0.5                    |
 +-----------------------+----------------------------+

docs/build/doctrees/PyU4V.doctree

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 5eaf754d565a9e60b6e3dcf2f18ef4bb
+config: b2097566cc03ccad42629cea0334c864
 tags: 645f666f9bcd5a90fca523b33c5a78b7