diff --git a/sl_util/sl_util/iterations_utils.py b/sl_util/sl_util/iterations_utils.py index 610136d7..94cdff3d 100644 --- a/sl_util/sl_util/iterations_utils.py +++ b/sl_util/sl_util/iterations_utils.py @@ -38,6 +38,14 @@ def compare_unordered_list_or_string(a: Union[str, List], b: Union[str, List]) - else: return False - + def remove_nones(list: List) -> List: return [e for e in list if e != None] + + +def remove_keys(dictionary: dict, keys_to_remove: [str]) -> dict: + filtered = dictionary.copy() + for key_to_remove in keys_to_remove: + if key_to_remove in filtered: + filtered.pop(key_to_remove) + return filtered diff --git a/sl_util/tests/unit/test_iterations_utils.py b/sl_util/tests/unit/test_iterations_utils.py index f7e70af8..42f18edd 100644 --- a/sl_util/tests/unit/test_iterations_utils.py +++ b/sl_util/tests/unit/test_iterations_utils.py @@ -1,4 +1,4 @@ -from sl_util.sl_util.iterations_utils import remove_from_list +from sl_util.sl_util.iterations_utils import remove_from_list, remove_keys def remove_function(element, original_array, removed_array): @@ -63,3 +63,16 @@ def test_basic_remove_with_custom_remove_function(self): assert len(removed_array) is 2 assert 2 in removed_array assert 4 in removed_array + + def test_remove_keys(self): + # Given a dict + numbers = {'1': 'One', '2': 'Two', '3': 'Three', '4': 'Four'} + + # And a list with the keys to remove + ids = ['2', '4'] + + # When we remove the keys + result = remove_keys(numbers, ids) + + # Then the result is as expected + assert result == {'1': 'One', '3': 'Three'} diff --git a/slp_visio/slp_visio/load/objects/diagram_objects.py b/slp_visio/slp_visio/load/objects/diagram_objects.py index d84314be..53683ae1 100644 --- a/slp_visio/slp_visio/load/objects/diagram_objects.py +++ b/slp_visio/slp_visio/load/objects/diagram_objects.py @@ -34,13 +34,15 @@ def get_component_category(self): def __str__(self) -> str: return '{id: ' + str(self.id) + ', ' \ - + 'name: ' + self.name + ', ' \ - + 'parent_id: ' + str(self.parent.id if self.parent else None) + '}' + + 'name: ' + self.name + ', ' \ + + 'type: ' + self.type + ', ' \ + + 'parent_id: ' + str(self.parent.id if self.parent else None) + '}' def __repr__(self) -> str: return '{id: ' + str(self.id) + ', ' \ - + 'name: ' + self.name + ', ' \ - + 'parent_id: ' + str(self.parent.id if self.parent else None) + '}' + + 'name: ' + self.name + ', ' \ + + 'type: ' + self.type + ', ' \ + + 'parent_id: ' + str(self.parent.id if self.parent else None) + '}' class DiagramConnector: diff --git a/slp_visio/slp_visio/load/strategies/component/impl/create_component_by_shape_text.py b/slp_visio/slp_visio/load/strategies/component/impl/create_component_by_shape_text.py index be2fb0ef..402b6389 100644 --- a/slp_visio/slp_visio/load/strategies/component/impl/create_component_by_shape_text.py +++ b/slp_visio/slp_visio/load/strategies/component/impl/create_component_by_shape_text.py @@ -25,7 +25,7 @@ def create_component(self, shape: Shape, origin=None, representer: VisioShapeRep return DiagramComponent( id=shape.ID, name=normalize_label(name), - type=normalize_label(self.get_component_type(shape)), + type=self.get_component_type(shape), origin=origin, representation=representer.build_representation(shape), unique_id=get_unique_id_text(shape)) diff --git a/slp_visio/slp_visio/parse/lucid_parser.py b/slp_visio/slp_visio/parse/lucid_parser.py index 4cc0eeb4..2387e205 100644 --- a/slp_visio/slp_visio/parse/lucid_parser.py +++ b/slp_visio/slp_visio/parse/lucid_parser.py @@ -1,9 +1,11 @@ from typing import Union, List from sl_util.sl_util import secure_regex +from sl_util.sl_util.iterations_utils import remove_keys from slp_visio.slp_visio.load.objects.diagram_objects import DiagramComponent, Diagram from slp_visio.slp_visio.load.visio_mapping_loader import VisioMappingFileLoader from slp_visio.slp_visio.parse.visio_parser import VisioParser +from slp_visio.slp_visio.util.visio import normalize_label AWS_REGEX = [r".*2017$", r".*AWS19$", r".*AWS2021$"] AZURE_REGEX = [r"^AC.*Block$", r"^AE.*Block$", r"^AGS.*Block$", r"^AVM.*Block$", r".*Azure2019$", r".*Azure2021$"] @@ -22,26 +24,21 @@ class LucidParser(VisioParser): def __init__(self, project_id: str, project_name: str, diagram: Diagram, mapping_loader: VisioMappingFileLoader): super().__init__(project_id, project_name, diagram, mapping_loader) - def __get_ids_to_skip(self) -> [str]: - """ - Returns the ids of the components that are mapped as trustzones or as components. - :return: - """ - return list(super()._get_trustzone_mappings().keys()) + list(super()._get_component_mappings().keys()) - def _get_component_mappings(self) -> [dict]: """ Returns the component mappings. After the component mappings are determined, the catch all mappings is determined. :return: """ - component_mappings = super()._get_component_mappings() - catch_all_components = self.__get_catch_all_mappings(ids_to_skip=self.__get_ids_to_skip()) + component_mappings: dict = super()._get_component_mappings() + tz_mapped: dict = super()._get_trustzone_mappings() + mapped_ids = list(tz_mapped.keys()) + list(component_mappings.keys()) + catch_all_components = self.__get_catch_all_mappings(ids_to_skip=mapped_ids) - component_mappings = self.__prune__skip__components(component_mappings) - catch_all_components = self.__prune__skip__components(catch_all_components) + pruned_component_mappings = self.__prune_skip_components(component_mappings) + pruned_catch_all_components = self.__prune_skip_components(catch_all_components) - return {**catch_all_components, **component_mappings} + return {**pruned_catch_all_components, **pruned_component_mappings} def __get_catch_all_mappings(self, ids_to_skip) -> [dict]: result = {} @@ -65,10 +62,12 @@ def __get_catch_all_config(self): def __get_skip_config(self) -> List[str]: return self.mapping_loader.configuration.get('skip') - - def __prune__skip__components(self, components): - components_to_skip = self.__get_skip_config() - if components_to_skip is not None: - components = {key: value for key, value in components.items() if value.get('type') not in components_to_skip} - return components + def __prune_skip_components(self, mappings): + skip_config = self.__get_skip_config() + if skip_config: + skip_config_normalized = [normalize_label(s) for s in skip_config] + ids_to_skip = {key for key, value in mappings.items() + if normalize_label(value.get('label')) in skip_config_normalized} + return remove_keys(mappings, ids_to_skip) + return mappings diff --git a/slp_visio/tests/unit/load/strategies/component/impl/test_create_component_by_shape_text.py b/slp_visio/tests/unit/load/strategies/component/impl/test_create_component_by_shape_text.py index b1c085bf..4cab981d 100644 --- a/slp_visio/tests/unit/load/strategies/component/impl/test_create_component_by_shape_text.py +++ b/slp_visio/tests/unit/load/strategies/component/impl/test_create_component_by_shape_text.py @@ -1,7 +1,7 @@ from unittest.mock import MagicMock import pytest -from _pytest.mark import param +from pytest import param from slp_visio.slp_visio.load.representation.simple_component_representer import SimpleComponentRepresenter from slp_visio.slp_visio.load.strategies.component.impl.create_component_by_shape_text import CreateComponentByShapeText @@ -68,5 +68,30 @@ def test_get_lucid_component_type(self, id_, shape_name, expected): strategy = CreateComponentByShapeText() component_type = strategy.get_lucid_component_type(shape) - # THEN no diagram is returned + # THEN the component type is as expected assert component_type == expected + + @pytest.mark.parametrize('shape_name,expected', { + param('com.lucidchart.AmazonElasticContainerServiceAWS19.121', 'AmazonElasticContainerServiceAWS19', + id='id==tail'), + param('com.lucidchart.AmazonElasticContainerServiceAWS19', 'AmazonElasticContainerServiceAWS19', + id='no tail'), + param('com.lucidchart.AmazonElasticContainerServiceAWS19.121', 'AmazonElasticContainerServiceAWS19', + id='id!=tail'), + param('com.lucidchart.AmazonElasticContainerServiceAWS19.121.44', 'AmazonElasticContainerServiceAWS19', + id='double tail') + }) + def test_create_component_lucid(self, shape_name, expected): + # GIVEN a visio component shape + shape = MagicMock(ID='10', shape_name=shape_name, text='My Elastic Container', + master_shape=MagicMock(text='Elastic Container Master'), + master_page=MagicMock(master_unique_id='777'), center_x_y=(0.5, 2.5), + cells={'Width': MagicMock(value=8), 'Height': MagicMock(value=12)}) + + # WHEN the component is created + strategy = CreateComponentByShapeText() + component = strategy.create_component(shape, representer=SimpleComponentRepresenter()) + + # THEN the component is as expected + assert component.type == expected + assert component.name == 'My Elastic Container' diff --git a/slp_visio/tests/unit/parse/test_lucid_parser.py b/slp_visio/tests/unit/parse/test_lucid_parser.py index b7d38d61..1a846730 100644 --- a/slp_visio/tests/unit/parse/test_lucid_parser.py +++ b/slp_visio/tests/unit/parse/test_lucid_parser.py @@ -114,50 +114,24 @@ def test_skip_mapped_shapes(self, mocked_trustzone_mappings, mocked_component_ma assert '3' in component_mappings assert isinstance(component_mappings['3'], MagicMock) - @pytest.mark.parametrize('skip_config', [ - pytest.param({}, id='None configuration'), - pytest.param({'skip': ''}, id='Configured as empty'), - pytest.param({'skip': ['EC2', 'emty-component', 'AmazonEC22017']}, id='Configured as wrong type'), - pytest.param({'skip': 'ce2'}, id='Configured as typo in type string ') - ]) - @patch.object(VisioParser, '_get_component_mappings', return_value={'5': {'label': 'AmazonEC2', 'type': 'ec2'}, '19': {'label': 'AmazonAPIGatewayAWS2021', 'type': 'empty-component'}}) - def test_wrong_skip_config(self, visio_get_component_mappings, skip_config): - """ - Test the skip config misconfiguration without catch all configuration. - - This test function checks the behavior of different skip configs including no configuration. - - Test Cases: - 1. Test with no skip configuration. - 2. Test with skip empty mapping. - 3. Test with a wrong type or not present type. - 4. Test with a typo. - """ - # GIVEN a mapping loader with skip configuration - mapping_loader = MagicMock(configuration=skip_config) - - diagram = MagicMock() - lucid_parser = LucidParser(*(MagicMock(),) * 2, diagram, mapping_loader) - - # WHEN _get_component_mappings is called in LucidParser - component_mappings = lucid_parser._get_component_mappings() - - # THEN none components are mapped - assert len(component_mappings) == 2 - - @pytest.mark.parametrize('skip_config', [ - pytest.param({'skip': ['ec2', 'empty-component']}, id='Configured as wrong type'), + @pytest.mark.parametrize('skip_config,expected', [ + pytest.param({'skip': ['AmazonEC22017', 'AmazonAPIGatewayAWS2021']}, [], id='with version'), + pytest.param({'skip': ['AmazonEC2', 'AmazonAPIGateway']}, [], id='without version'), + pytest.param({}, ['5', '19'], id='without skip'), + pytest.param({'skip': ''}, ['5', '19'], id='empty skip'), + pytest.param({'skip': ['amazonEC22017', 'amazonAPIGatewayAWS2021']}, ['5', '19'], + id='typo in skip with version'), + pytest.param({'skip': ['amazonEC2', 'amazonAPIGatewayAWS']}, ['5', '19'], id='typo in skip without version') ]) - @patch.object(VisioParser, '_get_component_mappings', return_value={'5': {'label': 'AmazonEC2', 'type': 'ec2'}, '19': {'label': 'AmazonAPIGatewayAWS2021', 'type': 'empty-component'}}) - def test_good_skip_config(self, visio_get_component_mappings, skip_config): + @patch.object(VisioParser, '_get_component_mappings', return_value={'5': {'label': 'AmazonEC2', 'type': 'ec2'}, + '19': {'label': 'AmazonAPIGatewayAWS2021', + 'type': 'empty-component'}}) + def test_get_component_mappings_skip_config(self, visio_get_component_mappings, skip_config, expected): """ Test the skip config without catch all configuration. This test function checks the behavior of remove components present in skip configuration. - Test Cases: - 1. Test with valid input. - """ # GIVEN a mapping loader with skip configuration mapping_loader = MagicMock(configuration=skip_config) @@ -168,25 +142,32 @@ def test_good_skip_config(self, visio_get_component_mappings, skip_config): # WHEN _get_component_mappings is called in LucidParser component_mappings = lucid_parser._get_component_mappings() - # THEN none components are mapped - assert len(component_mappings) == 0 - - @pytest.mark.parametrize('skip_config', [ - pytest.param({'skip': ['ec2', 'empty-component']}, id='Configured as wrong type'), + # THEN the components mapped are as expected + assert sorted(list(component_mappings.keys())) == sorted(expected) + + @pytest.mark.parametrize('skip_config, expected', [ + pytest.param({}, ['5', '14', '19', '23'], id='no skip'), + pytest.param({'skip': ['AmazonEC2']}, ['14', '19', '23'], id='mapped+skip'), + pytest.param({'skip': ['Azure Storage', 'AmazonAPIGatewayAWS2021'], 'catch-all': 'empty-component'}, + ['5', '14'], id='catchall+skip'), + pytest.param({'skip': ['AmazonEC2', 'Azure Storage', 'CorporateDataCenterContainer2017'], + 'catch-all': 'empty-component'}, + ['19'], id='mapped+catchall+skip'), ]) - @patch.object(VisioParser, '_get_component_mappings', return_value={'5': {'label': 'AmazonEC2', 'type': 'ec2'}}) - @patch.object(LucidParser, '_LucidParser__get_catch_all_mappings', return_value={'14': {'label': 'CorporateDataCenterContainer2017', 'type': 'empty-component'}, '19': {'label': 'AmazonAPIGatewayAWS2021', 'type': 'empty-component'}, '23': {'label': 'Azure Storage', 'type': 'azure-storage'}}) - def test_good_skip_config_with_catch_all_config(self, visio_get_component_mappings, lucid__get_catch_all_mappings, skip_config): - + @patch.object(VisioParser, '_get_component_mappings', + return_value={'5': {'label': 'AmazonEC2', 'type': 'ec2'}}) + @patch.object(LucidParser, '_LucidParser__get_catch_all_mappings', + return_value={'14': {'label': 'CorporateDataCenterContainer2017', 'type': 'empty-component'}, + '19': {'label': 'AmazonAPIGatewayAWS2021', 'type': 'empty-component'}, + '23': {'label': 'Azure Storage', 'type': 'azure-storage'}}) + def test_skip_config_with_catch_all_config(self, visio_get_component_mappings, lucid__get_catch_all_mappings, + skip_config, expected): """ - Test the skip config without catch all configuration. + Test the skip config with catch all configuration. This test function checks the behavior of remove components present in skip configuration, this also removes components mapped previously by catch all configuration. - Test Cases: - 1. Test with valid input. - """ # GIVEN a mapping loader with skip configuration mapping_loader = MagicMock(configuration=skip_config) @@ -197,5 +178,5 @@ def test_good_skip_config_with_catch_all_config(self, visio_get_component_mappin # WHEN _get_component_mappings is called in LucidParser component_mappings = lucid_parser._get_component_mappings() - # THEN one component is mapped - assert len(component_mappings) == 1 \ No newline at end of file + # THEN the components mapped are as expected + assert sorted(list(component_mappings.keys())) == sorted(expected)