Merge pull request #328 from iriusrisk/feature/OPT-1018

[feature-OPT-1018] to dev

sl_util/sl_util/dict_utils.py

@@ -0,0 +1,8 @@
+from deepdiff import DeepDiff
+
+
+def compare_dict(expected: dict, actual: dict, exclude_paths=None, exclude_regex=None) -> (dict, dict):
+    diff = DeepDiff(expected, actual, ignore_order=True, exclude_paths=exclude_paths, exclude_regex_paths=exclude_regex)
+    if diff:
+        return diff.t1, diff.t2
+    return {}, {}

sl_util/sl_util/xml_to_dict.py

@@ -0,0 +1,71 @@
+from collections import defaultdict
+
+from defusedxml import ElementTree
+
+
+def get_attrs(attrs):
+    result = {}
+    if attrs is not None:
+        for key, value in attrs.items():
+            result[remove_curly_info(key)] = value
+    return result
+
+
+def get_tag(t):
+    return remove_curly_info(t.tag)
+
+
+def remove_curly_info(value):
+    if '}' in value:
+        return value.split('}')[1]
+    else:
+        return value
+
+
+def add_attributes(d, t, tag_name):
+    d[tag_name].update(get_attrs(t.attrib))
+
+
+def add_text(children, d, t, tag_name):
+    text = t.text.strip()
+    if children or t.attrib:
+        if text:
+            d[tag_name]['text'] = text
+    else:
+        d[tag_name] = text
+
+
+def add_children(children, tag_name, separated_attributes):
+    dd = defaultdict(list)
+    for dc in [xml2dict(ch, separated_attributes) for ch in children]:
+        for k, v in dc.items():
+            dd[k].append(v)
+    return {tag_name: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}}
+
+
+def xml2dict(t, separated_attributes=False) -> dict:
+    tag_name = get_tag(t)
+    d = {tag_name: {} if t.attrib else None}
+    children = list(t)
+    if children:
+        d = add_children(children, tag_name, separated_attributes=separated_attributes)
+    if t.text:
+        add_text(children, d, t, tag_name)
+
+    if t.attrib:
+        if separated_attributes:
+            d['attrib'] = get_attrs(t.attrib)
+        else:
+            add_attributes(d, t, tag_name)
+
+    return d
+
+
+class XmlToDict:
+
+    def __init__(self, xml: str):
+        self.xml = xml
+
+    def to_dict(self) -> dict:
+        xml_data = ElementTree.XML(self.xml)
+        return xml2dict(xml_data)

sl_util/tests/resources/test_resource_paths.py

@@ -0,0 +1,7 @@
+import os
+
+path = os.path.dirname(__file__)
+
+# xml testing files
+random_data_xml = f'{path}/xml/random_data.xml'
+random_data_xml_json = f'{path}/xml/random_data_xml.json'

sl_util/tests/resources/xml/random_data.xml

@@ -0,0 +1,33 @@
+<root>
+    <flower>sunlight</flower>
+    <with waste="hurried">men</with>
+    <reason ship="nuts">-862244492</reason>
+    <exclaimed>
+        <widely>composition</widely>
+        <!--excited-->
+        <regular>spirit</regular>
+        <record>
+            <another experience="twice">
+                <!--sentence rays person-->
+                <teacher food="hay">1184223729.943545</teacher>
+                <count>magic</count>
+                <title>648067907.3277977</title>
+                <strange immediately="bread">change</strange>
+                <!--mainly row-->
+                <instrument>particular</instrument>
+                <finest>-1459099875.7289388</finest>
+            </another>
+            <hay smell="plates">loose</hay>
+            <tobacco>552630041.6824405</tobacco>
+            <peace suppose="motor">-45766333</peace>
+            <!--badly know nobody distance mighty-->
+            <birthday>stretch</birthday>
+            <food native="good">within</food>
+        </record>
+        <cookies tobacco="conversation">-1213510200.834167</cookies>
+        <dinner>-557912696.1980629</dinner>
+        <yard alone="willing">birds</yard>
+    </exclaimed>
+    <identity>459467385.41047287</identity>
+    <clean recognize="dried">1248105488</clean>
+</root>

sl_util/tests/resources/xml/random_data_xml.json

@@ -0,0 +1,62 @@
+{
+  "root": {
+    "clean": {
+      "recognize": "dried",
+      "text": "1248105488"
+    },
+    "exclaimed": {
+      "cookies": {
+        "text": "-1213510200.834167",
+        "tobacco": "conversation"
+      },
+      "dinner": "-557912696.1980629",
+      "record": {
+        "another": {
+          "count": "magic",
+          "experience": "twice",
+          "finest": "-1459099875.7289388",
+          "instrument": "particular",
+          "strange": {
+            "immediately": "bread",
+            "text": "change"
+          },
+          "teacher": {
+            "food": "hay",
+            "text": "1184223729.943545"
+          },
+          "title": "648067907.3277977"
+        },
+        "birthday": "stretch",
+        "food": {
+          "native": "good",
+          "text": "within"
+        },
+        "hay": {
+          "smell": "plates",
+          "text": "loose"
+        },
+        "peace": {
+          "suppose": "motor",
+          "text": "-45766333"
+        },
+        "tobacco": "552630041.6824405"
+      },
+      "regular": "spirit",
+      "widely": "composition",
+      "yard": {
+        "alone": "willing",
+        "text": "birds"
+      }
+    },
+    "flower": "sunlight",
+    "identity": "459467385.41047287",
+    "reason": {
+      "ship": "nuts",
+      "text": "-862244492"
+    },
+    "with": {
+      "text": "men",
+      "waste": "hurried"
+    }
+  }
+}

sl_util/tests/unit/test_xml_to_dict.py

@@ -0,0 +1,41 @@
+import json
+from unittest import TestCase
+
+import pytest
+
+from sl_util.sl_util.dict_utils import compare_dict
+from sl_util.sl_util.file_utils import get_byte_data
+from sl_util.sl_util.xml_to_dict import XmlToDict
+from sl_util.tests.resources.test_resource_paths import random_data_xml, random_data_xml_json
+
+
+class TestXmlToDict(TestCase):
+
+    def test_to_dict(self):
+        # GIVEN the source MTMT data
+        xml = get_byte_data(random_data_xml).decode()
+        # AND the parser
+        parser = XmlToDict(xml)
+        # AND the expected result
+        expected_data = json.loads(get_byte_data(random_data_xml_json).decode())
+
+        # WHEN we convert to json
+        result_data = parser.to_dict()
+
+        # THEN the result is as expected
+        result, expected = compare_dict(result_data, expected_data)
+        assert result == expected
+
+    def test_to_dict_invalid_data(self):
+        # GIVEN the source MTMT data
+        xml = "INVALID XML CONTENT"
+        # AND the parser
+        parser = XmlToDict(xml)
+
+        # WHEN we convert to json
+        # THEN a ParseError is raised
+        with pytest.raises(Exception) as error:
+            parser.to_dict()
+        # AND the error is as expected
+        assert error.typename == 'ParseError'
+        assert str(error.value.msg) == 'syntax error: line 1, column 0'

slp_base/slp_base/provider_type.py

@@ -16,8 +16,8 @@ class DiagramType(str, Provider):
              ['application/vnd.ms-visio.drawing.main+xml', 'application/octet-stream'])
     LUCID = ("LUCID", "Lucidchart", RepresentationType.DIAGRAM,
              ['application/vnd.ms-visio.drawing.main+xml', 'application/octet-stream', 'application/zip'])
-    DRAWIO = ("DRAWIO", "Drawio", RepresentationType.DIAGRAM,
-              ['application/octet-stream', 'application/xml', 'text/plain'])
+    # DRAWIO = ("DRAWIO", "Drawio", RepresentationType.DIAGRAM,
+    #           ['application/octet-stream', 'application/xml', 'text/plain'])
 
 
 class EtmType(str, Provider):

slp_drawio/slp_drawio/drawio_processor.py

@@ -12,7 +12,7 @@ class DrawioProcessor(OTMProcessor):
     Drawio implementation of OTMProcessor
     """
 
-    def __init__(self, project_id: str, project_name: str, source, mappings: [bytes],  diag_type=DiagramType.DRAWIO):
+    def __init__(self, project_id: str, project_name: str, source, mappings: [bytes],  diag_type=None):
         self.project_id = project_id
         self.project_name = project_name
         self.source = source

slp_drawio/slp_drawio/load/drawio_loader.py

@@ -17,13 +17,18 @@ class DrawioLoader(ProviderLoader):
     Builder for a drawio class from the xml data
     """
 
+    def __init__(self, project_id: str, source):
+        self.project_id = project_id
+        self.source = source
+        self.diagram = None
+
     def load(self):
         try:
-            source = DrawIOToDict(self.source).to_dict()
+            source_dict = DrawIOToDict(self.source).to_dict()
 
-            representation: DiagramRepresentation = DiagramRepresentationLoader(self.project_id, source).load()
-            components: [DiagramComponent] = DiagramComponentLoader(source).load()
-            dataflows: [DiagramDataflow] = DiagramDataflowLoader(source).load()
+            representation: DiagramRepresentation = DiagramRepresentationLoader(self.project_id, source_dict).load()
+            components: [DiagramComponent] = DiagramComponentLoader(source_dict).load()
+            dataflows: [DiagramDataflow] = DiagramDataflowLoader(source_dict).load()
 
             self.diagram: Diagram = Diagram(representation, components, dataflows)
         except Exception as e:
@@ -32,10 +37,5 @@ def load(self):
             message = e.__str__()
             raise LoadingSourceFileError('Source file cannot be loaded', detail, message)
 
-    def __init__(self, project_id: str, source):
-        self.project_id = project_id
-        self.source = source
-        self.diagram = None
-
     def get_diagram(self) -> Diagram:
         return self.diagram

slp_drawio/slp_drawio/load/drawio_to_dict.py

@@ -1,12 +1,41 @@
+import base64
+import logging
+import zlib
 from tempfile import SpooledTemporaryFile
+from urllib.parse import unquote
+
+from defusedxml import ElementTree
+
+from sl_util.sl_util.xml_to_dict import XmlToDict
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_ENCODING = 'utf8'
+DIAGRAM_TAG = 'diagram'
 
 
 class DrawIOToDict:
 
     def __init__(self, source):
         file: SpooledTemporaryFile = source.file
-        content: bytes = file.read()
-        self.source = content.decode()
+        self.encoding = 'utf8'
+        self.content: str = file.read().decode()
+        self.decode_b64_tag('diagram')
 
     def to_dict(self) -> dict:
-        return {}
+        return XmlToDict(self.content).to_dict()
+
+    def decode_b64_tag(self, tag):
+        tree = ElementTree.XML(self.content)
+        tree_tag = tree.find(tag)
+        children = list(tree_tag.iter())
+        if len(children) <= 1:
+            logger.debug(f'{tag} tag is encoded')
+            data = base64.b64decode(tree_tag.text, validate=True)
+            xml = zlib.decompress(data, wbits=-15).decode()
+            xml = unquote(xml)
+            diagram_tree = ElementTree.fromstring(xml)
+            tree_tag.append(diagram_tree)
+            tree_tag.text = None
+            self.content = ElementTree.tostring(tree, encoding=self.encoding)
+            logger.debug(f'{tag} tag decoded successfully')

slp_drawio/slp_drawio/parse/drawio_parser.py

@@ -32,12 +32,7 @@ def map_components_and_trustzones(self):
         DiagramMapper(self.diagram, self.mapping).map()
 
     def __build_otm(self):
-        otm = OTMBuilder(self.project_id, self.project_name, DiagramType.DRAWIO).build()
+        # TODO waiting the parser implementation
+        pass
 
-        otm.representations = [self.diagram.representation.otm]
-        otm.components = [c.otm for c in self.diagram.components]
-        otm.dataflows = [d.otm for d in self.diagram.dataflows]
-        otm.trustzones = [t.otm for t in self.diagram.trustzones]
-
-        return otm
 

slp_drawio/tests/resources/drawio/aws_minimal.drawio

@@ -0,0 +1 @@
+<mxfile host="Electron" modified="2023-10-10T12:38:05.500Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/19.0.3 Chrome/102.0.5005.63 Electron/19.0.3 Safari/537.36" etag="1qYMq70-eEWiWxYW-9pH" version="19.0.3" type="device"><diagram id="pOeIjdqQ_qid1hbLa7JX" name="Page-1">7VbbcpswEP0aP8bDNTiPMdhJ22TaxnGSN48MMqgRiBHCxv36rkBgbmmSmXSmDxljrD0Su9LZ3WMmphsXVxyl0S0LMJ0YWlBMTG9iGLpl6PAjkWOFXGhWBYScBGrRCViR31iBmkJzEuCss1AwRgVJu6DPkgT7ooMhztmhu2zHaDdqikI8AFY+okP0kQQiqtCZ4Zzwa0zCqI6sn19UMzGqF6uTZBEK2KEFmYuJ6XLGRDWKCxdTSV7NS/Xc8oXZZmMcJ+ItD9jEeVjPsuKeRhuP/VqHj0/emfKyRzRXB758XAHgUpYHat/iWJORMpKIklB7DhfEc7WJDTOutKaG3QP6ttMF9KElfXSBvu10Ab3vXu/F1/sbbAEDq+Ne68XXWhuEy5yzXFCSYLcpPQ3AkKOAQEpcRhkHLGEJsDePREzB0mF4iIjAqxT5ktUDtA1gO5YIVfy6UduKeOkVylsgiMWVjzITmC/2uEpItYZSlGZk2zzFsZ/zjOzxHc4q5xKFQkzlOC5C2bNTdMisachZnpbb/wKxRmc3MNz4sjA2iArpSHD2jOuDTgwTPktZfPMdobRHwB5zQaCvLikJpX/BZDikLIp3pUdghSThTWl5pqaYGAsRoCzCgTrSsBfqwoaouGhBqjeuMIux4EdYomYbxVFCZZjKPpzaXq+xqN3yNYiU1ISN71M3wkA15HhzFuvrrw/2z6fn4/333Ea338748swYNOegI0lc6le7vBTkkTiEqJRs4e5TAunjQg5ZnOZQOhmMPSTQFmV4oxuzAr7TNAmHVP5VON7Or92h1xqyOxshd/YB3I7uf8jtHQ4JVP6n5n1qXkvzeFUVI2KnW85ifvlPxa4J0Yid/jFiZ1n/ndiZr4vdaO11/gdq3m/QFtMfLCOiTJ63ZUKw+NXE+FgWWLdyRyrHnGLf6CUesrW0Z7ZpvdwOb1RW473JNM5LFWin0xlk0zmf2sN0XpjvziaYp3fWcq715m8u/gA=</diagram></mxfile>

slp_drawio/tests/resources/drawio/aws_minimal.drawio.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<mxfile host="Electron" modified="2023-10-10T12:38:11.657Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/19.0.3 Chrome/102.0.5005.63 Electron/19.0.3 Safari/537.36" etag="IaZ6VWqQhPlsWStpV9DA" version="19.0.3" type="device">
+  <diagram id="pOeIjdqQ_qid1hbLa7JX" name="Page-1">
+    <mxGraphModel dx="1421" dy="904" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="5i7VU8sxTlh_DojUgWXD-1" value="AWS Cloud" style="points=[[0,0],[0.25,0],[0.5,0],[0.75,0],[1,0],[1,0.25],[1,0.5],[1,0.75],[1,1],[0.75,1],[0.5,1],[0.25,1],[0,1],[0,0.75],[0,0.5],[0,0.25]];outlineConnect=0;gradientColor=none;html=1;whiteSpace=wrap;fontSize=12;fontStyle=0;container=1;pointerEvents=0;collapsible=0;recursiveResize=0;shape=mxgraph.aws4.group;grIcon=mxgraph.aws4.group_aws_cloud_alt;strokeColor=#232F3E;fillColor=none;verticalAlign=top;align=left;spacingLeft=30;fontColor=#232F3E;dashed=0;" parent="1" vertex="1">
+          <mxGeometry x="210" y="230" width="130" height="130" as="geometry" />
+        </mxCell>
+        <mxCell id="xUHJV5QXkyTOu5aMK-rF-2" value="" style="image;html=1;image=img/lib/clip_art/computers/Database_128x128.png" parent="5i7VU8sxTlh_DojUgWXD-1" vertex="1">
+          <mxGeometry x="25" y="40" width="80" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="5i7VU8sxTlh_DojUgWXD-2" value="Region" style="points=[[0,0],[0.25,0],[0.5,0],[0.75,0],[1,0],[1,0.25],[1,0.5],[1,0.75],[1,1],[0.75,1],[0.5,1],[0.25,1],[0,1],[0,0.75],[0,0.5],[0,0.25]];outlineConnect=0;gradientColor=none;html=1;whiteSpace=wrap;fontSize=12;fontStyle=0;container=1;pointerEvents=0;collapsible=0;recursiveResize=0;shape=mxgraph.aws4.group;grIcon=mxgraph.aws4.group_region;strokeColor=#147EBA;fillColor=none;verticalAlign=top;align=left;spacingLeft=30;fontColor=#147EBA;dashed=1;" parent="1" vertex="1">
+          <mxGeometry x="440" y="230" width="130" height="130" as="geometry" />
+        </mxCell>
+        <mxCell id="xUHJV5QXkyTOu5aMK-rF-3" value="" style="outlineConnect=0;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;shape=mxgraph.aws3.ec2;fillColor=#F58534;gradientColor=none;" parent="5i7VU8sxTlh_DojUgWXD-2" vertex="1">
+          <mxGeometry x="26.75" y="27" width="76.5" height="93" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>