[OPT-954] Add skip mapping function to Lucidchart

sl_util/sl_util/iterations_utils.py

@@ -38,6 +38,14 @@ def compare_unordered_list_or_string(a: Union[str, List], b: Union[str, List]) -
     else:
         return False
 
-      
+
 def remove_nones(list: List) -> List:
     return [e for e in list if e != None]
+
+
+def remove_keys(dictionary: dict, keys_to_remove: [str]) -> dict:
+    filtered = dictionary.copy()
+    for key_to_remove in keys_to_remove:
+        if key_to_remove in filtered:
+            filtered.pop(key_to_remove)
+    return filtered

sl_util/tests/unit/test_iterations_utils.py

@@ -1,4 +1,4 @@
-from sl_util.sl_util.iterations_utils import remove_from_list
+from sl_util.sl_util.iterations_utils import remove_from_list, remove_keys
 
 
 def remove_function(element, original_array, removed_array):
@@ -63,3 +63,16 @@ def test_basic_remove_with_custom_remove_function(self):
         assert len(removed_array) is 2
         assert 2 in removed_array
         assert 4 in removed_array
+
+    def test_remove_keys(self):
+        # Given a dict
+        numbers = {'1': 'One', '2': 'Two', '3': 'Three', '4': 'Four'}
+
+        # And a list with the keys to remove
+        ids = ['2', '4']
+
+        # When we remove the keys
+        result = remove_keys(numbers, ids)
+
+        # Then the result is as expected
+        assert result == {'1': 'One', '3': 'Three'}

slp_visio/slp_visio/load/objects/diagram_objects.py

@@ -34,13 +34,15 @@ def get_component_category(self):
 
     def __str__(self) -> str:
         return '{id: ' + str(self.id) + ', ' \
-               + 'name: ' + self.name + ', ' \
-               + 'parent_id: ' + str(self.parent.id if self.parent else None) + '}'
+            + 'name: ' + self.name + ', ' \
+            + 'type: ' + self.type + ', ' \
+            + 'parent_id: ' + str(self.parent.id if self.parent else None) + '}'
 
     def __repr__(self) -> str:
         return '{id: ' + str(self.id) + ', ' \
-               + 'name: ' + self.name + ', ' \
-               + 'parent_id: ' + str(self.parent.id if self.parent else None) + '}'
+            + 'name: ' + self.name + ', ' \
+            + 'type: ' + self.type + ', ' \
+            + 'parent_id: ' + str(self.parent.id if self.parent else None) + '}'
 
 
 class DiagramConnector:

slp_visio/slp_visio/load/strategies/component/impl/create_component_by_shape_text.py

@@ -25,7 +25,7 @@ def create_component(self, shape: Shape, origin=None, representer: VisioShapeRep
             return DiagramComponent(
                 id=shape.ID,
                 name=normalize_label(name),
-                type=normalize_label(self.get_component_type(shape)),
+                type=self.get_component_type(shape),
                 origin=origin,
                 representation=representer.build_representation(shape),
                 unique_id=get_unique_id_text(shape))

slp_visio/slp_visio/parse/lucid_parser.py

@@ -1,9 +1,11 @@
 from typing import Union, List
 
 from sl_util.sl_util import secure_regex
+from sl_util.sl_util.iterations_utils import remove_keys
 from slp_visio.slp_visio.load.objects.diagram_objects import DiagramComponent, Diagram
 from slp_visio.slp_visio.load.visio_mapping_loader import VisioMappingFileLoader
 from slp_visio.slp_visio.parse.visio_parser import VisioParser
+from slp_visio.slp_visio.util.visio import normalize_label
 
 AWS_REGEX = [r".*2017$", r".*AWS19$", r".*AWS2021$"]
 AZURE_REGEX = [r"^AC.*Block$", r"^AE.*Block$", r"^AGS.*Block$", r"^AVM.*Block$", r".*Azure2019$", r".*Azure2021$"]
@@ -22,26 +24,21 @@ class LucidParser(VisioParser):
     def __init__(self, project_id: str, project_name: str, diagram: Diagram, mapping_loader: VisioMappingFileLoader):
         super().__init__(project_id, project_name, diagram, mapping_loader)
 
-    def __get_ids_to_skip(self) -> [str]:
-        """
-        Returns the ids of the components that are mapped as trustzones or as components.
-        :return:
-        """
-        return list(super()._get_trustzone_mappings().keys()) + list(super()._get_component_mappings().keys())
-
     def _get_component_mappings(self) -> [dict]:
         """
         Returns the component mappings.
         After the component mappings are determined, the catch all mappings is determined.
         :return:
         """
-        component_mappings = super()._get_component_mappings()
-        catch_all_components = self.__get_catch_all_mappings(ids_to_skip=self.__get_ids_to_skip())
+        component_mappings: dict = super()._get_component_mappings()
+        tz_mapped: dict = super()._get_trustzone_mappings()
+        mapped_ids = list(tz_mapped.keys()) + list(component_mappings.keys())
+        catch_all_components = self.__get_catch_all_mappings(ids_to_skip=mapped_ids)
 
-        component_mappings = self.__prune__skip__components(component_mappings)
-        catch_all_components = self.__prune__skip__components(catch_all_components)
+        pruned_component_mappings = self.__prune_skip_components(component_mappings)
+        pruned_catch_all_components = self.__prune_skip_components(catch_all_components)
 
-        return {**catch_all_components, **component_mappings}
+        return {**pruned_catch_all_components, **pruned_component_mappings}
 
     def __get_catch_all_mappings(self, ids_to_skip) -> [dict]:
         result = {}
@@ -65,10 +62,12 @@ def __get_catch_all_config(self):
 
     def __get_skip_config(self) -> List[str]:
         return self.mapping_loader.configuration.get('skip')
-
-    def __prune__skip__components(self, components):
 
-        components_to_skip = self.__get_skip_config()
-        if components_to_skip is not None:
-            components = {key: value for key, value in components.items() if value.get('type') not in components_to_skip}
-        return components
+    def __prune_skip_components(self, mappings):
+        skip_config = self.__get_skip_config()
+        if skip_config:
+            skip_config_normalized = [normalize_label(s) for s in skip_config]
+            ids_to_skip = {key for key, value in mappings.items()
+                           if normalize_label(value.get('label')) in skip_config_normalized}
+            return remove_keys(mappings, ids_to_skip)
+        return mappings

slp_visio/tests/unit/load/strategies/component/impl/test_create_component_by_shape_text.py

@@ -1,7 +1,7 @@
 from unittest.mock import MagicMock
 
 import pytest
-from _pytest.mark import param
+from pytest import param
 
 from slp_visio.slp_visio.load.representation.simple_component_representer import SimpleComponentRepresenter
 from slp_visio.slp_visio.load.strategies.component.impl.create_component_by_shape_text import CreateComponentByShapeText
@@ -68,5 +68,30 @@ def test_get_lucid_component_type(self, id_, shape_name, expected):
         strategy = CreateComponentByShapeText()
         component_type = strategy.get_lucid_component_type(shape)
 
-        # THEN no diagram is returned
+        # THEN the component type is as expected
         assert component_type == expected
+
+    @pytest.mark.parametrize('shape_name,expected', {
+        param('com.lucidchart.AmazonElasticContainerServiceAWS19.121', 'AmazonElasticContainerServiceAWS19',
+              id='id==tail'),
+        param('com.lucidchart.AmazonElasticContainerServiceAWS19', 'AmazonElasticContainerServiceAWS19',
+              id='no tail'),
+        param('com.lucidchart.AmazonElasticContainerServiceAWS19.121', 'AmazonElasticContainerServiceAWS19',
+              id='id!=tail'),
+        param('com.lucidchart.AmazonElasticContainerServiceAWS19.121.44', 'AmazonElasticContainerServiceAWS19',
+              id='double tail')
+    })
+    def test_create_component_lucid(self, shape_name, expected):
+        # GIVEN a visio component shape
+        shape = MagicMock(ID='10', shape_name=shape_name, text='My Elastic Container',
+                          master_shape=MagicMock(text='Elastic Container Master'),
+                          master_page=MagicMock(master_unique_id='777'), center_x_y=(0.5, 2.5),
+                          cells={'Width': MagicMock(value=8), 'Height': MagicMock(value=12)})
+
+        # WHEN the component is created
+        strategy = CreateComponentByShapeText()
+        component = strategy.create_component(shape, representer=SimpleComponentRepresenter())
+
+        # THEN the component is as expected
+        assert component.type == expected
+        assert component.name == 'My Elastic Container'

slp_visio/tests/unit/parse/test_lucid_parser.py

@@ -114,50 +114,24 @@ def test_skip_mapped_shapes(self, mocked_trustzone_mappings, mocked_component_ma
         assert '3' in component_mappings
         assert isinstance(component_mappings['3'], MagicMock)
 
-    @pytest.mark.parametrize('skip_config', [
-        pytest.param({}, id='None configuration'),
-        pytest.param({'skip': ''}, id='Configured as empty'),
-        pytest.param({'skip': ['EC2', 'emty-component', 'AmazonEC22017']}, id='Configured as wrong type'),
-        pytest.param({'skip': 'ce2'}, id='Configured as typo in type string ')
-    ])
-    @patch.object(VisioParser, '_get_component_mappings', return_value={'5': {'label': 'AmazonEC2', 'type': 'ec2'}, '19': {'label': 'AmazonAPIGatewayAWS2021', 'type': 'empty-component'}})
-    def test_wrong_skip_config(self, visio_get_component_mappings, skip_config):
-        """
-        Test the skip config misconfiguration without catch all configuration.
-
-        This test function checks the behavior of different skip configs including no configuration.
-
-        Test Cases:
-        1. Test with no skip configuration.
-        2. Test with skip empty mapping.
-        3. Test with a wrong type or not present type.
-        4. Test with a typo.
-        """
-        # GIVEN a mapping loader with skip configuration
-        mapping_loader = MagicMock(configuration=skip_config)
-
-        diagram = MagicMock()
-        lucid_parser = LucidParser(*(MagicMock(),) * 2, diagram, mapping_loader)
-
-        # WHEN _get_component_mappings is called in LucidParser
-        component_mappings = lucid_parser._get_component_mappings()
-
-        # THEN none components are mapped
-        assert len(component_mappings) == 2
-
-    @pytest.mark.parametrize('skip_config', [
-        pytest.param({'skip': ['ec2', 'empty-component']}, id='Configured as wrong type'),
+    @pytest.mark.parametrize('skip_config,expected', [
+        pytest.param({'skip': ['AmazonEC22017', 'AmazonAPIGatewayAWS2021']}, [], id='with version'),
+        pytest.param({'skip': ['AmazonEC2', 'AmazonAPIGateway']}, [], id='without version'),
+        pytest.param({}, ['5', '19'], id='without skip'),
+        pytest.param({'skip': ''}, ['5', '19'], id='empty skip'),
+        pytest.param({'skip': ['amazonEC22017', 'amazonAPIGatewayAWS2021']}, ['5', '19'],
+                     id='typo in skip with version'),
+        pytest.param({'skip': ['amazonEC2', 'amazonAPIGatewayAWS']}, ['5', '19'], id='typo in skip without version')
     ])
-    @patch.object(VisioParser, '_get_component_mappings', return_value={'5': {'label': 'AmazonEC2', 'type': 'ec2'}, '19': {'label': 'AmazonAPIGatewayAWS2021', 'type': 'empty-component'}})
-    def test_good_skip_config(self, visio_get_component_mappings, skip_config):
+    @patch.object(VisioParser, '_get_component_mappings', return_value={'5': {'label': 'AmazonEC2', 'type': 'ec2'},
+                                                                        '19': {'label': 'AmazonAPIGatewayAWS2021',
+                                                                               'type': 'empty-component'}})
+    def test_get_component_mappings_skip_config(self, visio_get_component_mappings, skip_config, expected):
         """
         Test the skip config without catch all configuration.
 
         This test function checks the behavior of remove components present in skip configuration.
 
-        Test Cases:
-        1. Test with valid input.
-
         """
         # GIVEN a mapping loader with skip configuration
         mapping_loader = MagicMock(configuration=skip_config)
@@ -168,25 +142,32 @@ def test_good_skip_config(self, visio_get_component_mappings, skip_config):
         # WHEN _get_component_mappings is called in LucidParser
         component_mappings = lucid_parser._get_component_mappings()
 
-        # THEN none components are mapped
-        assert len(component_mappings) == 0
-
-    @pytest.mark.parametrize('skip_config', [
-        pytest.param({'skip': ['ec2', 'empty-component']}, id='Configured as wrong type'),
+        # THEN the components mapped are as expected
+        assert sorted(list(component_mappings.keys())) == sorted(expected)
+
+    @pytest.mark.parametrize('skip_config, expected', [
+        pytest.param({}, ['5', '14', '19', '23'], id='no skip'),
+        pytest.param({'skip': ['AmazonEC2']}, ['14', '19', '23'], id='mapped+skip'),
+        pytest.param({'skip': ['Azure Storage', 'AmazonAPIGatewayAWS2021'], 'catch-all': 'empty-component'},
+                     ['5', '14'], id='catchall+skip'),
+        pytest.param({'skip': ['AmazonEC2', 'Azure Storage', 'CorporateDataCenterContainer2017'],
+                      'catch-all': 'empty-component'},
+                     ['19'], id='mapped+catchall+skip'),
     ])
-    @patch.object(VisioParser, '_get_component_mappings', return_value={'5': {'label': 'AmazonEC2', 'type': 'ec2'}})
-    @patch.object(LucidParser, '_LucidParser__get_catch_all_mappings', return_value={'14': {'label': 'CorporateDataCenterContainer2017', 'type': 'empty-component'}, '19': {'label': 'AmazonAPIGatewayAWS2021', 'type': 'empty-component'}, '23': {'label': 'Azure Storage', 'type': 'azure-storage'}})
-    def test_good_skip_config_with_catch_all_config(self, visio_get_component_mappings, lucid__get_catch_all_mappings, skip_config):
-
+    @patch.object(VisioParser, '_get_component_mappings',
+                  return_value={'5': {'label': 'AmazonEC2', 'type': 'ec2'}})
+    @patch.object(LucidParser, '_LucidParser__get_catch_all_mappings',
+                  return_value={'14': {'label': 'CorporateDataCenterContainer2017', 'type': 'empty-component'},
+                                '19': {'label': 'AmazonAPIGatewayAWS2021', 'type': 'empty-component'},
+                                '23': {'label': 'Azure Storage', 'type': 'azure-storage'}})
+    def test_skip_config_with_catch_all_config(self, visio_get_component_mappings, lucid__get_catch_all_mappings,
+                                               skip_config, expected):
         """
-        Test the skip config without catch all configuration.
+        Test the skip config with catch all configuration.
 
         This test function checks the behavior of remove components present in skip configuration, this also removes
         components mapped previously by catch all configuration.
 
-        Test Cases:
-        1. Test with valid input.
-
         """
         # GIVEN a mapping loader with skip configuration
         mapping_loader = MagicMock(configuration=skip_config)
@@ -197,5 +178,5 @@ def test_good_skip_config_with_catch_all_config(self, visio_get_component_mappin
         # WHEN _get_component_mappings is called in LucidParser
         component_mappings = lucid_parser._get_component_mappings()
 
-        # THEN one component is mapped
-        assert len(component_mappings) == 1
+        # THEN the components mapped are as expected
+        assert sorted(list(component_mappings.keys())) == sorted(expected)