Union data type (#51)

* Added the `union` data type to the schema to allow a key to be more than one specific type
* Reduced the constraints on the `install_requirements` in `setup.py`
* Minor fixes and improvements to the overall codebase

Author: ryan95f

changelog.md

@@ -41,3 +41,9 @@
 
 * Added support for Python 3.6+ by removing the `__future__` and `dataclass` imports from the codebase
 * Downgraded pylint in the developer dependencies to allow for Python 3.6 to run locally
+
+## v0.3.0 (29th January 2023)
+
+* Added the `union` data type to the schema to allow a key to be more than one specific type
+* Reduced the constraints on the `install_requirements` in `setup.py`
+* Minor fixes and improvements to the overall codebase

docs/schema_components.md

@@ -15,6 +15,7 @@ Below are the various components that can be used to construct a schema with Yam
   * [Regex Type](#regex-type)
   * [Ruleset Type](#ruleset-type)
   * [Enum Type](#enum-type)
+  * [Union Type](#union-type)
 
 ## Schema
 
@@ -362,7 +363,38 @@ ruleset <ruleset-name> {
 Or in a schema block:
 
 ```text
-Schema {
+schema {
     logLevel LogLevel
 }
 ```
+
+### Union Type
+
+The `union` type can be used when a key could be one or more types. For example, a key could be either an `int` or a `float`. The union type supports all available types including lists, rulesets, regex, enums, maps and all the other basic types. When defining a `union`, 2 or more types must be defined for it to be valid.
+
+Nested `unions` are *NOT* supported and will raise a parsing error. Unions can be nested in other data types such as a list, rulesets and maps which can then used within a `union`. However direct nesting of unions is not supported.
+
+__Note__: The union will always return a single violation if one or more data types are not met. It currently does not display the individual violations against each type.
+
+An example of a `union` type:
+
+```yaml
+items:
+    - name: item1
+      price: 10
+    - name: item2
+      price: 15.2
+```
+
+Then the following schema can be defined:
+
+```text
+ruleset Item {
+    name str
+    price union(int, float)
+}
+
+schema {
+    items list(Item)
+}
+```

setup.py

@@ -2,7 +2,7 @@
 
 import setuptools
 
-VERSION = '0.2.2'
+VERSION = '0.3.0'
 PACKAGE_NAME = 'yamlator'
 DESCRIPTION = 'Yamlator is a CLI tool that allows a YAML file to be validated using a lightweight schema language'  # nopep8
 
@@ -48,8 +48,8 @@ def create_long_description():
         'yamlator.cmd.outputs'
     ]),
     install_requires=[
-        'lark==1.0.0',
-        'PyYAML==6.0'
+        'lark>=1.0.0',
+        'PyYAML>=5.4.1'
     ],
     python_requires='>=3.6',
     classifiers=[

tests/files/invalid_files/invalid_syntax.ys

@@ -0,0 +1,8 @@
+ruleset Hello {
+    name str required true
+    no_data
+}
+
+schema {
+    hello Hello
+}

tests/files/invalid_files/nested_union.ys

@@ -0,0 +1,9 @@
+
+ruleset Config {
+    kind str
+    version union(str, int, union(int, str))
+}
+
+schema {
+    config Config optional
+}

tests/files/valid/valid.ys

@@ -5,7 +5,7 @@ enum Employee_department {
 }
 
 ruleset PersonAddress {
-    houseNumber int
+    houseNumber union(int, str)
     street str
     city str
     post_code str

tests/parser/test_parse_schema.py

@@ -19,10 +19,10 @@
 
 from yamlator.utils import load_schema
 from yamlator.parser import parse_schema
-from yamlator.utils import load_yaml_file
 from yamlator.parser import MalformedEnumNameError
 from yamlator.parser import MalformedRulesetNameError
 from yamlator.parser import MissingRulesError
+from yamlator.parser import SchemaParseError
 from yamlator.parser import SchemaSyntaxError
 
 
@@ -74,11 +74,16 @@ def test_parse_with_valid_content(self):
         (
             'with_ruleset_not_defined',
             './tests/files/invalid_files/missing_defined_ruleset.ys',
-            SchemaSyntaxError
+            SchemaParseError
+        ),
+        (
+            'union_with_nested_union',
+            './tests/files/invalid_files/nested_union.ys',
+            SchemaParseError
         ),
         (
-            'with_invalid_schema_syntax',
-            './tests/files/valid/valid.yaml',
+            'with_invalid_rule_syntax',
+            './tests/files/invalid_files/invalid_syntax.ys',
             SchemaSyntaxError
         )
     ])
@@ -87,9 +92,9 @@ def test_parse_syntax_errors(self, name: str, schema_file_path: str,
         # Unused by test case, however is required by the parameterized library
         del name
 
-        schema_content = load_yaml_file(schema_file_path)
+        schema_content = load_schema(schema_file_path)
         with self.assertRaises(exception_type):
-            parse_schema(str(schema_content))
+            parse_schema(schema_content)
 
 
 if __name__ == '__main__':

tests/parser/test_schema_transformer.py

@@ -34,6 +34,10 @@
        container type when the name of the given container is not found
     * `test_regex_type` tests transforming regex tokens into a Yamlator
        regex type
+    * `test_union_type` tests transforming a union token into a Union
+       rule type
+    * `test_union_type_with_nested_union` tests that a exception
+       is raised when a union contains a nested union
     * `test_type` tests the type transformer returns the token it was
        passed so the other transformer can successfully transform it
     * `test_schema_entry` tests that given a set of rule tokens within
@@ -57,6 +61,7 @@
 
 from parameterized import parameterized
 from yamlator.exceptions import ConstructNotFoundError
+from yamlator.exceptions import NestedUnionError
 
 from yamlator.parser import SchemaTransformer
 from yamlator.types import EnumItem, Rule, RuleType
@@ -231,6 +236,27 @@ def test_regex_type(self):
         self.assertEqual(expected_regex_str, rule_type.regex)
         self.assertEqual(rule_type.schema_type, SchemaTypes.REGEX)
 
+    def test_union_type(self):
+        tokens = [
+            RuleType(SchemaTypes.INT),
+            RuleType(SchemaTypes.STR),
+            RuleType(SchemaTypes.LIST, sub_type=RuleType(SchemaTypes.INT))
+        ]
+
+        union_type = self.transformer.union_type(tokens)
+        self.assertIsNotNone(union_type)
+        self.assertEqual(SchemaTypes.UNION, union_type.schema_type)
+
+    def test_union_type_with_nested_union(self):
+        tokens = [
+            RuleType(SchemaTypes.INT),
+            RuleType(SchemaTypes.STR),
+            RuleType(SchemaTypes.LIST, sub_type=RuleType(SchemaTypes.INT)),
+            RuleType(SchemaTypes.UNION)
+        ]
+        with self.assertRaises(NestedUnionError):
+            self.transformer.union_type(tokens)
+
     def test_type(self):
         type_token = self.transformer.type((self.name_token, ))
         self.assertEqual(self.name_token, type_token)

tests/test_types.py

@@ -0,0 +1,71 @@
+"""Test case for the Yamlator types module"""
+
+import unittest
+
+from parameterized import parameterized
+
+from yamlator.types import RuleType
+from yamlator.types import UnionRuleType
+from yamlator.types import SchemaTypes
+
+
+class TestRuleType(unittest.TestCase):
+    """Test cases for the RuleType objects"""
+
+    @parameterized.expand([
+        ('str_type', RuleType(SchemaTypes.STR), 'str'),
+        ('int_type', RuleType(SchemaTypes.INT), 'int'),
+        ('float_type', RuleType(SchemaTypes.FLOAT), 'float'),
+        ('int_map', RuleType(
+            SchemaTypes.MAP,
+            sub_type=RuleType(SchemaTypes.INT)), 'map(int)'),
+        ('map_of_int_maps', RuleType(
+            SchemaTypes.MAP,
+            sub_type=RuleType(
+                    SchemaTypes.MAP,
+                    sub_type=RuleType(SchemaTypes.INT)
+                )
+            ),
+            'map(map(int))'),
+        ('str_list', RuleType(
+            SchemaTypes.LIST,
+            sub_type=RuleType(SchemaTypes.INT)), 'list(int)'),
+        ('list_of_str_lists', RuleType(
+            SchemaTypes.LIST,
+            sub_type=RuleType(
+                SchemaTypes.LIST,
+                sub_type=RuleType(SchemaTypes.STR))
+            ),
+            'list(list(str))'),
+        ('enum_type', RuleType(
+            SchemaTypes.ENUM, lookup='Numbers'),
+            'Numbers'),
+        ('ruleset_type',
+            RuleType(SchemaTypes.RULESET, lookup='Details'),
+            'Details'),
+        ('any_type', RuleType(SchemaTypes.ANY), 'any'),
+        ('regex_type',
+            RuleType(SchemaTypes.REGEX, regex='^role'),
+            'Regex(^role)'),
+        ('bool_type', RuleType(SchemaTypes.BOOL), 'bool'),
+        ('union_type', UnionRuleType([
+            RuleType(SchemaTypes.INT),
+            RuleType(
+                SchemaTypes.LIST,
+                sub_type=RuleType(
+                    SchemaTypes.LIST,
+                    sub_type=RuleType(SchemaTypes.STR)
+                )
+            )
+        ]), 'union(int, list(list(str)))')
+    ])
+    def test_rule_type_string_representation(self, name: str, rtype: RuleType,
+                                             expected_string: str):
+        del name
+
+        actual_string = str(rtype)
+        self.assertEqual(expected_string, actual_string)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/validators/test_union_validator.py

@@ -0,0 +1,110 @@
+"""Test cases for the Union Validator"""
+
+import unittest
+import typing
+
+from parameterized import parameterized
+from .base import BaseValidatorTest
+
+from yamlator.types import Data
+from yamlator.types import RuleType
+from yamlator.types import UnionRuleType
+from yamlator.types import SchemaTypes
+from yamlator.violations import TypeViolation
+from yamlator.validators import UnionValidator
+from yamlator.validators.base_validator import Validator
+
+
+class DummyValidator(Validator):
+    """Dummy Validator to test the union validator
+
+    This validator will always return a Type violation
+
+    """
+    def validate(self, key: str, data: Data, parent: str, rtype: RuleType,
+                 is_required: bool = False) -> None:
+        # Not used by the DummyValidator
+        del data
+        del is_required
+        del rtype
+
+        message = 'Invalid type'
+        violation = TypeViolation(key, parent, message)
+        self._violations.append(violation)
+
+
+class TestUnionValidator(BaseValidatorTest):
+    """Test cases for the Union Validator"""
+
+    @parameterized.expand([
+        ('rule_not_union_type', RuleType(SchemaTypes.FLOAT), 1.23, 0),
+        ('union_rule_with_invalid_data', UnionRuleType([
+            RuleType(SchemaTypes.LIST, sub_type=RuleType(SchemaTypes.INT)),
+            RuleType(SchemaTypes.MAP, sub_type=RuleType(SchemaTypes.INT)),
+            RuleType(SchemaTypes.RULESET, lookup='test'),
+            RuleType(SchemaTypes.STR)
+        ]), 1.23, 1),
+        ('union_rule_with_valid_data', UnionRuleType([
+            RuleType(SchemaTypes.LIST, sub_type=RuleType(SchemaTypes.INT)),
+            RuleType(SchemaTypes.MAP, sub_type=RuleType(SchemaTypes.INT)),
+            RuleType(SchemaTypes.RULESET, lookup='test'),
+            RuleType(SchemaTypes.FLOAT)
+        ]), 1.23, 0),
+    ])
+    def test_union_validation(self, name: str,
+                              rtype: typing.Union[UnionRuleType, RuleType],
+                              data: Data,
+                              expected_violation_count: int):
+        # Unused by test case, however is required by the parameterized library
+        del name
+
+        validator = UnionValidator(self.violations)
+        self._set_sub_type_validators(validator)
+        validator.validate(self.key, data, self.parent, rtype)
+
+        actual_violation_count = len(self.violations)
+        self.assertEqual(expected_violation_count, actual_violation_count)
+
+    def _set_sub_type_validators(self, validator: UnionValidator):
+        dummy_validator = DummyValidator(self.violations)
+
+        validator.set_enum_validator(dummy_validator)
+        validator.set_list_validator(dummy_validator)
+        validator.set_map_validator(dummy_validator)
+        validator.set_regex_validator(dummy_validator)
+        validator.set_ruleset_validator(dummy_validator)
+
+    @parameterized.expand([
+        ('rule_not_union_type', RuleType(SchemaTypes.FLOAT), 1.23, 0),
+        ('union_rule_with_invalid_data', UnionRuleType([
+            RuleType(SchemaTypes.LIST, sub_type=RuleType(SchemaTypes.INT)),
+            RuleType(SchemaTypes.MAP, sub_type=RuleType(SchemaTypes.INT)),
+            RuleType(SchemaTypes.RULESET, lookup='test'),
+            RuleType(SchemaTypes.STR)
+        ]), 1.23, 0),
+        ('union_rule_with_valid_data', UnionRuleType([
+            RuleType(SchemaTypes.LIST, sub_type=RuleType(SchemaTypes.INT)),
+            RuleType(SchemaTypes.MAP, sub_type=RuleType(SchemaTypes.INT)),
+            RuleType(SchemaTypes.RULESET, lookup='test'),
+            RuleType(SchemaTypes.FLOAT)
+        ]), 1.23, 0),
+    ])
+    def test_union_validation_without_sub_validators(self, name: str,
+                                                     rtype: typing.Union[
+                                                        UnionRuleType,
+                                                        RuleType
+                                                     ],
+                                                     data: Data,
+                                                     expected_violation_count: int):  # nopep8 pylint: disable=C0301
+        # Unused by test case, however is required by the parameterized library
+        del name
+
+        validator = UnionValidator(self.violations)
+        validator.validate(self.key, data, self.parent, rtype)
+
+        actual_violation_count = len(self.violations)
+        self.assertEqual(expected_violation_count, actual_violation_count)
+
+
+if __name__ == '__main__':
+    unittest.main()