From 30025ac3a5b20a7f8da9b59a3a0d46d4fc293100 Mon Sep 17 00:00:00 2001 From: David Antolin Alvarez Date: Thu, 8 Feb 2024 18:31:26 +0100 Subject: [PATCH 1/5] [OPT-1105] Fixed wrong type in drawio diagram height and width --- slp_tfplan/slp_tfplan/validate/tfplan_validator.py | 4 ++-- slp_tfplan/tests/util/builders.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/slp_tfplan/slp_tfplan/validate/tfplan_validator.py b/slp_tfplan/slp_tfplan/validate/tfplan_validator.py index 012c8bfd..42e30144 100644 --- a/slp_tfplan/slp_tfplan/validate/tfplan_validator.py +++ b/slp_tfplan/slp_tfplan/validate/tfplan_validator.py @@ -13,8 +13,8 @@ logger = logging.getLogger(__name__) MIN_FILE_SIZE = 20 -MAX_TFPLAN_FILE_SIZE = 5 * 1024 * 1024 # 5MB -MAX_TFGRAPH_FILE_SIZE = 2 * 1024 * 1024 # 2MB +MAX_TFPLAN_FILE_SIZE = 20 * 1024 * 1024 # 20MB +MAX_TFGRAPH_FILE_SIZE = 20 * 1024 * 1024 # 20MB TFPLAN_MIME_TYPE = 'application/json' TFGRAPH_MIME_TYPE = 'text/plain' diff --git a/slp_tfplan/tests/util/builders.py b/slp_tfplan/tests/util/builders.py index bdfd83c2..dfc7f232 100644 --- a/slp_tfplan/tests/util/builders.py +++ b/slp_tfplan/tests/util/builders.py @@ -18,8 +18,8 @@ TFPLAN_MINIMUM_STRUCTURE = {'planned_values': {'root_module': {}}} MIN_FILE_SIZE = 20 -MAX_TFPLAN_FILE_SIZE = 5 * 1024 * 1024 # 5MB -MAX_TFGRAPH_FILE_SIZE = 2 * 1024 * 1024 # 2MB +MAX_TFPLAN_FILE_SIZE = 20 * 1024 * 1024 # 20MB +MAX_TFGRAPH_FILE_SIZE = 20 * 1024 * 1024 # 20MB ####### From 2b03b0d6db14b7be96bcf957d3b85c0cbff38a90 Mon Sep 17 00:00:00 2001 From: David Antolin Alvarez Date: Tue, 13 Feb 2024 11:56:22 +0100 Subject: [PATCH 2/5] [OPT-1105] Supported multiple encodings in file_utils.py --- sl_util/sl_util/file_utils.py | 12 ++++++++++-- sl_util/sl_util/json_utils.py | 3 ++- slp_drawio/tests/validate/test_drawio_validator.py | 2 -- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/sl_util/sl_util/file_utils.py b/sl_util/sl_util/file_utils.py index 35e3b9fd..170e5e76 100644 --- a/sl_util/sl_util/file_utils.py +++ b/sl_util/sl_util/file_utils.py @@ -7,6 +7,8 @@ from magic import Magic from starlette.datastructures import UploadFile +SUPPORTED_ENCODINGS = ['utf-8', 'utf-16', 'utf-8-ignore'] + def copy_to_disk(diag_file: tempfile.SpooledTemporaryFile, suffix: str): with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as ntf: @@ -37,8 +39,14 @@ def get_byte_data_from_upload_file(upload_file: UploadFile) -> bytes: return upload_file.file.read() -def read_byte_data(data: bytes, encoding: str = 'utf-8') -> str: - return data.decode(encoding) +def read_byte_data(data: bytes) -> str: + for encoding in SUPPORTED_ENCODINGS: + try: + return data.decode(encoding=encoding) + except UnicodeError: + pass + + raise UnicodeError(f'File content cannot be decoded, supported encodings: {SUPPORTED_ENCODINGS}') def get_file_type_by_content(file_content: bytes) -> str: diff --git a/sl_util/sl_util/json_utils.py b/sl_util/sl_util/json_utils.py index b00f1a0c..654a4adc 100644 --- a/sl_util/sl_util/json_utils.py +++ b/sl_util/sl_util/json_utils.py @@ -4,6 +4,7 @@ import yaml from otm.otm.entity.otm import OTM +from sl_util.sl_util.file_utils import read_byte_data logger = logging.getLogger(__name__) @@ -14,7 +15,7 @@ def get_otm_as_json(otm: OTM): def yaml_data_as_str(data) -> str: - return data if isinstance(data, str) else data.decode() + return data if isinstance(data, str) else read_byte_data(data) def yaml_reader(data, loader=yaml.BaseLoader): diff --git a/slp_drawio/tests/validate/test_drawio_validator.py b/slp_drawio/tests/validate/test_drawio_validator.py index 712a0eea..cc4c715c 100644 --- a/slp_drawio/tests/validate/test_drawio_validator.py +++ b/slp_drawio/tests/validate/test_drawio_validator.py @@ -1,11 +1,9 @@ from unittest.mock import patch, MagicMock import pytest -from starlette.datastructures import UploadFile, Headers from sl_util.sl_util import secure_regex as re from sl_util.sl_util.file_utils import get_byte_data -from sl_util.tests.util.file_utils import get_upload_file from slp_base import DiagramFileNotValidError, CommonError from slp_drawio.slp_drawio.validate.drawio_validator import DrawioValidator from slp_drawio.tests.resources.test_resource_paths import wrong_mxgraphmodel_drawio, wrong_mxfile_drawio, \ From f0857a624300532a7671e2e146dce3d99404610a Mon Sep 17 00:00:00 2001 From: David Antolin Alvarez Date: Tue, 13 Feb 2024 14:52:08 +0100 Subject: [PATCH 3/5] [OPT-1105] Used json library to load tfplan because of performance reasons --- sl_util/sl_util/json_utils.py | 13 +++++++++---- slp_cft/slp_cft/load/cft_loader.py | 7 +++---- slp_tfplan/slp_tfplan/load/tfplan_loader.py | 4 ++-- .../tests/unit/load/test_tfplan_loader.py | 18 +++++++++--------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/sl_util/sl_util/json_utils.py b/sl_util/sl_util/json_utils.py index 654a4adc..e8e93c1d 100644 --- a/sl_util/sl_util/json_utils.py +++ b/sl_util/sl_util/json_utils.py @@ -1,5 +1,6 @@ import json import logging +from typing import Union import yaml @@ -9,14 +10,18 @@ logger = logging.getLogger(__name__) +def __yaml_data_as_str(data: Union[str, bytes]) -> str: + return data if isinstance(data, str) else read_byte_data(data) + + def get_otm_as_json(otm: OTM): logger.info("getting OTM contents as JSON") return json.dumps(otm.json(), indent=2) -def yaml_data_as_str(data) -> str: - return data if isinstance(data, str) else read_byte_data(data) +def read_yaml(data: bytes, loader=yaml.SafeLoader) -> dict: + return yaml.load(__yaml_data_as_str(data), Loader=loader) -def yaml_reader(data, loader=yaml.BaseLoader): - return yaml.load(yaml_data_as_str(data), Loader=loader) +def read_json(data: bytes) -> dict: + return json.loads(__yaml_data_as_str(data)) diff --git a/slp_cft/slp_cft/load/cft_loader.py b/slp_cft/slp_cft/load/cft_loader.py index faffddd9..138718b5 100644 --- a/slp_cft/slp_cft/load/cft_loader.py +++ b/slp_cft/slp_cft/load/cft_loader.py @@ -1,10 +1,9 @@ import logging from deepmerge import always_merger - from yaml import BaseLoader, ScalarNode -from sl_util.sl_util.json_utils import yaml_reader +from sl_util.sl_util.json_utils import read_yaml from slp_base.slp_base.errors import LoadingIacFileError from slp_base.slp_base.provider_loader import ProviderLoader @@ -33,7 +32,7 @@ class CloudformationLoader(ProviderLoader): def __init__(self, sources): self.sources = sources - self.yaml_reader = yaml_reader + self.yaml_reader = read_yaml self.cloudformation = None def load(self): @@ -57,7 +56,7 @@ def __merge_cft_data(self, cft_data): def __load_cft_data(self, source) -> dict: try: - logger.debug(f"Loading iac data and reading as string") + logger.debug("Loading iac data and reading as string") cft_data = self.yaml_reader(source, loader=get_loader()) diff --git a/slp_tfplan/slp_tfplan/load/tfplan_loader.py b/slp_tfplan/slp_tfplan/load/tfplan_loader.py index 06eeb114..4c64f3ec 100644 --- a/slp_tfplan/slp_tfplan/load/tfplan_loader.py +++ b/slp_tfplan/slp_tfplan/load/tfplan_loader.py @@ -4,14 +4,14 @@ from networkx import nx_agraph, DiGraph from sl_util.sl_util.file_utils import read_byte_data -from sl_util.sl_util.json_utils import yaml_reader +from sl_util.sl_util.json_utils import read_json from slp_base import ProviderLoader, LoadingIacFileError from slp_tfplan.slp_tfplan.load.tfplan_to_resource_dict import TfplanToResourceDict def load_tfplan(source: bytes) -> Dict: try: - return yaml_reader(source) + return read_json(source) except Exception: pass diff --git a/slp_tfplan/tests/unit/load/test_tfplan_loader.py b/slp_tfplan/tests/unit/load/test_tfplan_loader.py index ca0df787..47ff947b 100644 --- a/slp_tfplan/tests/unit/load/test_tfplan_loader.py +++ b/slp_tfplan/tests/unit/load/test_tfplan_loader.py @@ -36,7 +36,7 @@ def mock_load_graph(mocker, mocked_graph): class TestTFPlanLoader: @patch('slp_tfplan.slp_tfplan.load.tfplan_loader.load_tfgraph') - @patch('yaml.load') + @patch('json.loads') def test_load_tfplan_and_graph(self, yaml_mock, from_agraph_mock): # GIVEN a valid plain Terraform Plan file with no modules yaml_mock.side_effect = [build_tfplan(resources=generate_resources(2))] @@ -55,7 +55,7 @@ def test_load_tfplan_and_graph(self, yaml_mock, from_agraph_mock): # AND the TFGRAPH is also loaded assert tfplan_loader.get_tfgraph().graph['label'] == graph_label - @patch('yaml.load') + @patch('json.loads') def test_load_no_modules(self, yaml_mock): # GIVEN a valid plain Terraform Plan file with no modules yaml_mock.side_effect = [build_tfplan(resources=generate_resources(2))] @@ -79,7 +79,7 @@ def test_load_no_modules(self, yaml_mock): assert_resource_values(resource['resource_values']) - @patch('yaml.load') + @patch('json.loads') def test_load_only_modules(self, yaml_mock): # GIVEN a valid plain Terraform Plan file with only modules yaml_mock.side_effect = [build_tfplan( @@ -110,7 +110,7 @@ def test_load_only_modules(self, yaml_mock): resource_index += 1 - @patch('yaml.load') + @patch('json.loads') def test_load_nested_modules(self, yaml_mock): # GIVEN a valid plain Terraform Plan file with nested modules yaml_mock.side_effect = [build_tfplan( @@ -136,7 +136,7 @@ def test_load_nested_modules(self, yaml_mock): assert_resource_values(resource['resource_values']) - @patch('yaml.load') + @patch('json.loads') def test_load_complex_structure(self, yaml_mock): # GIVEN a valid plain Terraform Plan file with modules and root-level resources yaml_mock.side_effect = [build_tfplan( @@ -169,7 +169,7 @@ def test_load_complex_structure(self, yaml_mock): assert_resource_values(resource['resource_values']) - @patch('yaml.load') + @patch('json.loads') def test_load_resources_same_name(self, yaml_mock): # GIVEN a valid plain Terraform Plan file with only one module tfplan = build_tfplan( @@ -196,7 +196,7 @@ def test_load_resources_same_name(self, yaml_mock): assert resources[0]['resource_id'] == 'r1-addr' assert resources[0]['resource_name'] == 'cm1-addr.r1-name' - @patch('yaml.load') + @patch('json.loads') def test_load_modules_same_name(self, yaml_mock): # GIVEN a valid plain Terraform Plan file with only one module tfplan = build_tfplan( @@ -233,7 +233,7 @@ def test_load_modules_same_name(self, yaml_mock): assert resources[0]['resource_id'] == 'cm1-addr.r1-addr' assert resources[0]['resource_name'] == 'cm1-addr.r1-name' - @patch('yaml.load') + @patch('json.loads') def test_load_no_resources(self, yaml_mock): # GIVEN a valid Terraform Plan file with no resources yaml_mock.side_effect = [{'planned_values': {'root_module': {}}}] @@ -245,7 +245,7 @@ def test_load_no_resources(self, yaml_mock): # THEN TfplanLoader.terraform is an empty dictionary assert tfplan_loader.terraform == {} - @patch('yaml.load') + @patch('json.loads') def test_load_empty_tfplan(self, yaml_mock): # GIVEN an empty TFPLAN yaml_mock.side_effect = [{}] From 2ac9db8ae125f577a379cc86f6da5f12aa21fc55 Mon Sep 17 00:00:00 2001 From: David Antolin Alvarez Date: Wed, 14 Feb 2024 16:51:39 +0100 Subject: [PATCH 4/5] [OPT-1105] Fixed error in tfplan_to_resource_dict due to ints loaded as strings --- slp_tfplan/slp_tfplan/load/tfplan_to_resource_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slp_tfplan/slp_tfplan/load/tfplan_to_resource_dict.py b/slp_tfplan/slp_tfplan/load/tfplan_to_resource_dict.py index e1bba77a..568cb4b2 100644 --- a/slp_tfplan/slp_tfplan/load/tfplan_to_resource_dict.py +++ b/slp_tfplan/slp_tfplan/load/tfplan_to_resource_dict.py @@ -4,7 +4,7 @@ def is_not_cloned_resource(resource: Dict) -> bool: - return 'index' not in resource or resource['index'] == '0' or resource['index'] == 'zero' + return 'index' not in resource or resource['index'] == '0' or resource['index'] == 0 or resource['index'] == 'zero' def get_resource_id(resource: Dict) -> str: From 4a746b5f827fd467e47ca76e6a9791570d8298f0 Mon Sep 17 00:00:00 2001 From: David Antolin Alvarez Date: Mon, 19 Feb 2024 10:39:44 +0100 Subject: [PATCH 5/5] [OPT-1105] Max tfplan and tfgraph file sizes increased to 50MB --- slp_tfplan/slp_tfplan/validate/tfplan_validator.py | 4 ++-- slp_tfplan/tests/util/builders.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/slp_tfplan/slp_tfplan/validate/tfplan_validator.py b/slp_tfplan/slp_tfplan/validate/tfplan_validator.py index 42e30144..0a188419 100644 --- a/slp_tfplan/slp_tfplan/validate/tfplan_validator.py +++ b/slp_tfplan/slp_tfplan/validate/tfplan_validator.py @@ -13,8 +13,8 @@ logger = logging.getLogger(__name__) MIN_FILE_SIZE = 20 -MAX_TFPLAN_FILE_SIZE = 20 * 1024 * 1024 # 20MB -MAX_TFGRAPH_FILE_SIZE = 20 * 1024 * 1024 # 20MB +MAX_TFPLAN_FILE_SIZE = 50 * 1024 * 1024 # 50MB +MAX_TFGRAPH_FILE_SIZE = 50 * 1024 * 1024 # 50MB TFPLAN_MIME_TYPE = 'application/json' TFGRAPH_MIME_TYPE = 'text/plain' diff --git a/slp_tfplan/tests/util/builders.py b/slp_tfplan/tests/util/builders.py index dfc7f232..3b770933 100644 --- a/slp_tfplan/tests/util/builders.py +++ b/slp_tfplan/tests/util/builders.py @@ -18,8 +18,8 @@ TFPLAN_MINIMUM_STRUCTURE = {'planned_values': {'root_module': {}}} MIN_FILE_SIZE = 20 -MAX_TFPLAN_FILE_SIZE = 20 * 1024 * 1024 # 20MB -MAX_TFGRAPH_FILE_SIZE = 20 * 1024 * 1024 # 20MB +MAX_TFPLAN_FILE_SIZE = 50 * 1024 * 1024 # 50MB +MAX_TFGRAPH_FILE_SIZE = 50 * 1024 * 1024 # 50MB #######