Merge pull request #358 from iriusrisk/hotfix/OPT-1105

[hotfix/OPT-1105] to main

sl_util/sl_util/file_utils.py

@@ -7,6 +7,8 @@
 from magic import Magic
 from starlette.datastructures import UploadFile
 
+SUPPORTED_ENCODINGS = ['utf-8', 'utf-16', 'utf-8-ignore']
+
 
 def copy_to_disk(diag_file: tempfile.SpooledTemporaryFile, suffix: str):
     with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as ntf:
@@ -37,8 +39,14 @@ def get_byte_data_from_upload_file(upload_file: UploadFile) -> bytes:
     return upload_file.file.read()
 
 
-def read_byte_data(data: bytes, encoding: str = 'utf-8') -> str:
-    return data.decode(encoding)
+def read_byte_data(data: bytes) -> str:
+    for encoding in SUPPORTED_ENCODINGS:
+        try:
+            return data.decode(encoding=encoding)
+        except UnicodeError:
+            pass
+
+    raise UnicodeError(f'File content cannot be decoded, supported encodings: {SUPPORTED_ENCODINGS}')
 
 
 def get_file_type_by_content(file_content: bytes) -> str:

sl_util/sl_util/json_utils.py

@@ -1,21 +1,27 @@
 import json
 import logging
+from typing import Union
 
 import yaml
 
 from otm.otm.entity.otm import OTM
+from sl_util.sl_util.file_utils import read_byte_data
 
 logger = logging.getLogger(__name__)
 
 
+def __yaml_data_as_str(data: Union[str, bytes]) -> str:
+    return data if isinstance(data, str) else read_byte_data(data)
+
+
 def get_otm_as_json(otm: OTM):
     logger.info("getting OTM contents as JSON")
     return json.dumps(otm.json(), indent=2)
 
 
-def yaml_data_as_str(data) -> str:
-    return data if isinstance(data, str) else data.decode()
+def read_yaml(data: bytes, loader=yaml.SafeLoader) -> dict:
+    return yaml.load(__yaml_data_as_str(data), Loader=loader)
 
 
-def yaml_reader(data, loader=yaml.BaseLoader):
-    return yaml.load(yaml_data_as_str(data), Loader=loader)
+def read_json(data: bytes) -> dict:
+    return json.loads(__yaml_data_as_str(data))

slp_cft/slp_cft/load/cft_loader.py

@@ -1,10 +1,9 @@
 import logging
 
 from deepmerge import always_merger
-
 from yaml import BaseLoader, ScalarNode
 
-from sl_util.sl_util.json_utils import yaml_reader
+from sl_util.sl_util.json_utils import read_yaml
 from slp_base.slp_base.errors import LoadingIacFileError
 from slp_base.slp_base.provider_loader import ProviderLoader
 
@@ -33,7 +32,7 @@ class CloudformationLoader(ProviderLoader):
 
     def __init__(self, sources):
         self.sources = sources
-        self.yaml_reader = yaml_reader
+        self.yaml_reader = read_yaml
         self.cloudformation = None
 
     def load(self):
@@ -57,7 +56,7 @@ def __merge_cft_data(self, cft_data):
 
     def __load_cft_data(self, source) -> dict:
         try:
-            logger.debug(f"Loading iac data and reading as string")
+            logger.debug("Loading iac data and reading as string")
 
             cft_data = self.yaml_reader(source, loader=get_loader())
 

slp_drawio/tests/validate/test_drawio_validator.py

@@ -1,11 +1,9 @@
 from unittest.mock import patch, MagicMock
 
 import pytest
-from starlette.datastructures import UploadFile, Headers
 
 from sl_util.sl_util import secure_regex as re
 from sl_util.sl_util.file_utils import get_byte_data
-from sl_util.tests.util.file_utils import get_upload_file
 from slp_base import DiagramFileNotValidError, CommonError
 from slp_drawio.slp_drawio.validate.drawio_validator import DrawioValidator
 from slp_drawio.tests.resources.test_resource_paths import wrong_mxgraphmodel_drawio, wrong_mxfile_drawio, \

slp_tfplan/slp_tfplan/load/tfplan_loader.py

@@ -4,14 +4,14 @@
 from networkx import nx_agraph, DiGraph
 
 from sl_util.sl_util.file_utils import read_byte_data
-from sl_util.sl_util.json_utils import yaml_reader
+from sl_util.sl_util.json_utils import read_json
 from slp_base import ProviderLoader, LoadingIacFileError
 from slp_tfplan.slp_tfplan.load.tfplan_to_resource_dict import TfplanToResourceDict
 
 
 def load_tfplan(source: bytes) -> Dict:
     try:
-        return yaml_reader(source)
+        return read_json(source)
     except Exception:
         pass
 

slp_tfplan/slp_tfplan/load/tfplan_to_resource_dict.py

@@ -4,7 +4,7 @@
 
 
 def is_not_cloned_resource(resource: Dict) -> bool:
-    return 'index' not in resource or resource['index'] == '0' or resource['index'] == 'zero'
+    return 'index' not in resource or resource['index'] == '0' or resource['index'] == 0 or resource['index'] == 'zero'
 
 
 def get_resource_id(resource: Dict) -> str:

slp_tfplan/slp_tfplan/validate/tfplan_validator.py

@@ -13,8 +13,8 @@
 logger = logging.getLogger(__name__)
 
 MIN_FILE_SIZE = 20
-MAX_TFPLAN_FILE_SIZE = 5 * 1024 * 1024  # 5MB
-MAX_TFGRAPH_FILE_SIZE = 2 * 1024 * 1024  # 2MB
+MAX_TFPLAN_FILE_SIZE = 50 * 1024 * 1024  # 50MB
+MAX_TFGRAPH_FILE_SIZE = 50 * 1024 * 1024  # 50MB
 
 TFPLAN_MIME_TYPE = 'application/json'
 TFGRAPH_MIME_TYPE = 'text/plain'

slp_tfplan/tests/unit/load/test_tfplan_loader.py

@@ -36,7 +36,7 @@ def mock_load_graph(mocker, mocked_graph):
 class TestTFPlanLoader:
 
     @patch('slp_tfplan.slp_tfplan.load.tfplan_loader.load_tfgraph')
-    @patch('yaml.load')
+    @patch('json.loads')
     def test_load_tfplan_and_graph(self, yaml_mock, from_agraph_mock):
         # GIVEN a valid plain Terraform Plan file with no modules
         yaml_mock.side_effect = [build_tfplan(resources=generate_resources(2))]
@@ -55,7 +55,7 @@ def test_load_tfplan_and_graph(self, yaml_mock, from_agraph_mock):
         # AND the TFGRAPH is also loaded
         assert tfplan_loader.get_tfgraph().graph['label'] == graph_label
 
-    @patch('yaml.load')
+    @patch('json.loads')
     def test_load_no_modules(self, yaml_mock):
         # GIVEN a valid plain Terraform Plan file with no modules
         yaml_mock.side_effect = [build_tfplan(resources=generate_resources(2))]
@@ -79,7 +79,7 @@ def test_load_no_modules(self, yaml_mock):
 
             assert_resource_values(resource['resource_values'])
 
-    @patch('yaml.load')
+    @patch('json.loads')
     def test_load_only_modules(self, yaml_mock):
         # GIVEN a valid plain Terraform Plan file with only modules
         yaml_mock.side_effect = [build_tfplan(
@@ -110,7 +110,7 @@ def test_load_only_modules(self, yaml_mock):
 
                 resource_index += 1
 
-    @patch('yaml.load')
+    @patch('json.loads')
     def test_load_nested_modules(self, yaml_mock):
         # GIVEN a valid plain Terraform Plan file with nested modules
         yaml_mock.side_effect = [build_tfplan(
@@ -136,7 +136,7 @@ def test_load_nested_modules(self, yaml_mock):
 
         assert_resource_values(resource['resource_values'])
 
-    @patch('yaml.load')
+    @patch('json.loads')
     def test_load_complex_structure(self, yaml_mock):
         # GIVEN a valid plain Terraform Plan file with modules and root-level resources
         yaml_mock.side_effect = [build_tfplan(
@@ -169,7 +169,7 @@ def test_load_complex_structure(self, yaml_mock):
 
         assert_resource_values(resource['resource_values'])
 
-    @patch('yaml.load')
+    @patch('json.loads')
     def test_load_resources_same_name(self, yaml_mock):
         # GIVEN a valid plain Terraform Plan file with only one module
         tfplan = build_tfplan(
@@ -196,7 +196,7 @@ def test_load_resources_same_name(self, yaml_mock):
         assert resources[0]['resource_id'] == 'r1-addr'
         assert resources[0]['resource_name'] == 'cm1-addr.r1-name'
 
-    @patch('yaml.load')
+    @patch('json.loads')
     def test_load_modules_same_name(self, yaml_mock):
         # GIVEN a valid plain Terraform Plan file with only one module
         tfplan = build_tfplan(
@@ -233,7 +233,7 @@ def test_load_modules_same_name(self, yaml_mock):
         assert resources[0]['resource_id'] == 'cm1-addr.r1-addr'
         assert resources[0]['resource_name'] == 'cm1-addr.r1-name'
 
-    @patch('yaml.load')
+    @patch('json.loads')
     def test_load_no_resources(self, yaml_mock):
         # GIVEN a valid Terraform Plan file with no resources
         yaml_mock.side_effect = [{'planned_values': {'root_module': {}}}]
@@ -245,7 +245,7 @@ def test_load_no_resources(self, yaml_mock):
         # THEN TfplanLoader.terraform is an empty dictionary
         assert tfplan_loader.terraform == {}
 
-    @patch('yaml.load')
+    @patch('json.loads')
     def test_load_empty_tfplan(self, yaml_mock):
         # GIVEN an empty TFPLAN
         yaml_mock.side_effect = [{}]

slp_tfplan/tests/util/builders.py

@@ -18,8 +18,8 @@
 
 TFPLAN_MINIMUM_STRUCTURE = {'planned_values': {'root_module': {}}}
 MIN_FILE_SIZE = 20
-MAX_TFPLAN_FILE_SIZE = 5 * 1024 * 1024  # 5MB
-MAX_TFGRAPH_FILE_SIZE = 2 * 1024 * 1024  # 2MB
+MAX_TFPLAN_FILE_SIZE = 50 * 1024 * 1024  # 50MB
+MAX_TFGRAPH_FILE_SIZE = 50 * 1024 * 1024  # 50MB
 
 
 #######