Merge pull request #607 from Clariteia/0.1.16

0.1.16

HISTORY.md

@@ -199,3 +199,11 @@ History
 --------------------
 
 * Fix bug from `PostgreSqlSnapshotReader` that returned already deleted aggregates when `Condition.TRUE` was passed.
+
+0.1.16 (2021-10-07)
+--------------------
+
+* Improve support for `Model` inheritance inside container classes (`list`, `dict`, `EntitySet`, etc.).
+* Add support for `set[T]` type.
+* Fix bug related with complex types and `PostgreSqlSnapshotQueryBuilder`.
+* Fix bug related with empty `dict` and `minos.saga.SagaContext`.

minos/common/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.15"
+__version__ = "0.1.16"
 
 from .configuration import (
     BROKER,

minos/common/model/serializers/avro_data_decoder.py

@@ -4,6 +4,8 @@
 
 import logging
 from collections.abc import (
+    Iterable,
+    Mapping,
     MutableSet,
 )
 from datetime import (
@@ -15,7 +17,6 @@
 )
 from typing import (
     Any,
-    Iterable,
     TypeVar,
     Union,
     get_args,
@@ -251,6 +252,9 @@ def _cast_composed_value(self, type_: type, data: Any) -> Any:
         if origin_type is list:
             return self._cast_list(data, type_)
 
+        if origin_type is set:
+            return self._cast_set(data, type_)
+
         if origin_type is dict:
             return self._cast_dict(data, type_)
 
@@ -261,14 +265,21 @@ def _cast_composed_value(self, type_: type, data: Any) -> Any:
 
     def _cast_list(self, data: list, type_values: Any) -> list[Any]:
         type_values = get_args(type_values)[0]
-        if not isinstance(data, list):
+        if not isinstance(data, Iterable):
             raise DataDecoderTypeException(list, data)
 
         return list(self._cast_iterable(data, type_values))
 
+    def _cast_set(self, data: set, type_values: Any) -> set[Any]:
+        type_values = get_args(type_values)[0]
+        if not isinstance(data, Iterable):
+            raise DataDecoderTypeException(set, data)
+
+        return set(self._cast_iterable(data, type_values))
+
     def _cast_dict(self, data: dict, type_: type) -> dict[str, Any]:
         type_keys, type_values = get_args(type_)
-        if not isinstance(data, dict):
+        if not isinstance(data, Mapping):
             raise DataDecoderTypeException(dict, data)
 
         if type_keys is not str:

minos/common/model/serializers/avro_data_encoder.py

@@ -68,7 +68,7 @@ def _to_avro_raw(self, value: Any) -> Any:
         if isinstance(value, UUID):
             return self._uuid_to_avro_raw(value)
 
-        if isinstance(value, list):
+        if isinstance(value, (list, set,)):
             return [self._to_avro_raw(v) for v in value]
 
         if isinstance(value, dict):

minos/common/model/serializers/avro_schema_decoder.py

@@ -37,6 +37,7 @@
     AVRO_MAP,
     AVRO_NULL,
     AVRO_RECORD,
+    AVRO_SET,
     AVRO_STRING,
     AVRO_TIME,
     AVRO_TIMEDELTA,
@@ -74,7 +75,7 @@ def _build_type_from_list(self, schema: list[Any]) -> type:
 
     def _build_type_from_dict(self, schema: dict) -> type:
         if "logicalType" in schema:
-            return self._build_logical_type(schema["logicalType"])
+            return self._build_logical_type(schema)
         elif schema["type"] == AVRO_ARRAY:
             return self._build_list_type(schema["items"])
         elif schema["type"] == AVRO_MAP:
@@ -84,8 +85,8 @@ def _build_type_from_dict(self, schema: dict) -> type:
         else:
             return self._build_type(schema["type"])
 
-    @staticmethod
-    def _build_logical_type(type_: str) -> type:
+    def _build_logical_type(self, schema: dict[str, Any]) -> type:
+        type_ = schema["logicalType"]
         if type_ == AVRO_DATE["logicalType"]:
             return date
         if type_ == AVRO_TIME["logicalType"]:
@@ -96,11 +97,16 @@ def _build_logical_type(type_: str) -> type:
             return timedelta
         if type_ == AVRO_UUID["logicalType"]:
             return UUID
+        if type_ == AVRO_SET["logicalType"]:
+            return self._build_set_type(schema["items"])
         raise MinosMalformedAttributeException(f"Given logical field type is not supported: {type_!r}")
 
-    def _build_list_type(self, items: Union[dict, str, Any] = None) -> type:
+    def _build_list_type(self, items: Any = None) -> type:
         return list[self._build_type(items)]
 
+    def _build_set_type(self, items: Any = None) -> type:
+        return set[self._build_type(items)]
+
     def _build_dict_type(self, values: Union[dict, str, Any] = None) -> type:
         return dict[str, self._build_type(values)]
 

minos/common/model/serializers/avro_schema_encoder.py

@@ -37,6 +37,7 @@
     AVRO_INT,
     AVRO_MAP,
     AVRO_NULL,
+    AVRO_SET,
     AVRO_STRING,
     AVRO_TIME,
     AVRO_TIMEDELTA,
@@ -150,6 +151,9 @@ def _build_composed_schema(self, type_: type) -> Any:
         if origin_type is list:
             return self._build_list_schema(type_)
 
+        if origin_type is set:
+            return self._build_set_schema(type_)
+
         if origin_type is dict:
             return self._build_dict_schema(type_)
 
@@ -158,6 +162,10 @@ def _build_composed_schema(self, type_: type) -> Any:
 
         raise ValueError(f"Given field type is not supported: {type_}")  # pragma: no cover
 
+    def _build_set_schema(self, type_: type) -> dict[str, Any]:
+        schema = self._build_list_schema(type_)
+        return schema | AVRO_SET
+
     def _build_list_schema(self, type_: type) -> dict[str, Any]:
         return {"type": AVRO_ARRAY, "items": self._build_schema(get_args(type_)[0])}
 

minos/common/model/serializers/constants.py

@@ -18,3 +18,4 @@
 AVRO_TIMESTAMP = {"type": AVRO_LONG, "logicalType": "timestamp-micros"}
 AVRO_TIMEDELTA = {"type": AVRO_LONG, "logicalType": "timedelta-micros"}
 AVRO_UUID = {"type": AVRO_STRING, "logicalType": "uuid"}
+AVRO_SET = {"type": AVRO_ARRAY, "logicalType": "set"}

minos/common/model/types/builders.py

@@ -79,7 +79,7 @@ def _build(self, value, type_: Optional[type]) -> type:
             return type(value)[self._build_from_iterable(value, b1)]
 
         if isinstance(value, dict):
-            b1, b2 = (Any, Any) if (type_ is None or len(get_args(type_)) != 2) else get_args(type_)
+            b1, b2 = (str, Any) if (type_ is None or len(get_args(type_)) != 2) else get_args(type_)
             return type(value)[
                 self._build_from_iterable(value.keys(), b1), self._build_from_iterable(value.values(), b2)
             ]

minos/common/model/types/comparators.py

@@ -81,6 +81,12 @@ def _compare(self, first: T, second: K) -> bool:
         if second is Any:
             return True
 
+        if get_origin(first) is Union and all(self._compare(f, second) for f in get_args(first)):
+            return True
+
+        if get_origin(second) is Union and any(self._compare(first, s) for s in get_args(second)):
+            return True
+
         if get_origin(first) is ModelRef:
             first = Union[(*get_args(first), UUID)]
 

minos/common/snapshot/pg/queries.py

@@ -120,12 +120,16 @@ def _build_condition_composed(self, condition: _ComposedCondition) -> Composable
         return SQL("({composed})").format(composed=operator.join(parts))
 
     def _build_condition_simple(self, condition: _SimpleCondition) -> Composable:
+        from ...model import (
+            AvroDataEncoder,
+        )
+
         field = condition.field
         # noinspection PyTypeChecker
         operator = _SIMPLE_MAPPER[type(condition)]
 
-        parameter = condition.parameter
-        if isinstance(parameter, (list, tuple, set)):
+        parameter = AvroDataEncoder(condition.parameter).build()
+        if isinstance(parameter, list):
             if not len(parameter):
                 return self._build_condition(_FALSE_CONDITION)
             parameter = tuple(parameter)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "minos_microservice_common"
-version = "0.1.15"
+version = "0.1.16"
 description = "Python Package with common Classes and Utilities used in Minos Microservices."
 readme = "README.md"
 repository = "https://github.com/clariteia/minos_microservice_common"

tests/test_common/test_model/test_serializers/test_avro_data_decoder.py

@@ -18,6 +18,7 @@
 
 from minos.common import (
     AvroDataDecoder,
+    DataDecoderException,
     DataDecoderMalformedTypeException,
     DataDecoderRequiredValueException,
     DataDecoderTypeException,
@@ -34,6 +35,7 @@
     Owner,
 )
 from tests.model_classes import (
+    Analytics,
     Base,
     GenericUser,
     User,
@@ -78,8 +80,8 @@ def test_any(self):
 
     def test_any_raises(self):
         decoder = AvroDataDecoder(Any)
-        with self.assertRaises(DataDecoderTypeException):
-            decoder.build({"one", "two"})
+        with self.assertRaises(DataDecoderException):
+            decoder.build(AvroDataDecoder)
 
     def test_none(self):
         decoder = AvroDataDecoder(type(None))
@@ -244,6 +246,11 @@ async def test_list_model_subaggregate_ref(self):
             observed = decoder.build(value)
             self.assertEqual(value, observed)
 
+    def test_list_empty(self):
+        decoder = AvroDataDecoder(list[int])
+        observed = decoder.build([])
+        self.assertEqual([], observed)
+
     def test_list_raises(self):
         decoder = AvroDataDecoder(list)
         with self.assertRaises(DataDecoderMalformedTypeException):
@@ -265,12 +272,32 @@ def test_list_any(self):
         decoder = AvroDataDecoder(list[Any])
         self.assertEqual([1, "hola", True], decoder.build([1, "hola", True]))
 
+    def test_set(self):
+        decoder = AvroDataDecoder(set[int])
+        self.assertEqual({1, 2, 3}, decoder.build([1, 2, 3]))
+
+    def test_set_raises(self):
+        decoder = AvroDataDecoder(set)
+        with self.assertRaises(DataDecoderMalformedTypeException):
+            decoder.build({1, 2, 3})
+
+        decoder = AvroDataDecoder(set[int])
+        with self.assertRaises(DataDecoderTypeException):
+            decoder.build(3)
+        with self.assertRaises(DataDecoderRequiredValueException):
+            decoder.build(None)
+
     def test_dict(self):
         decoder = AvroDataDecoder(dict[str, bool])
         value = {"foo": True, "bar": False}
         observed = decoder.build(value)
         self.assertEqual(value, observed)
 
+    def test_dict_empty(self):
+        decoder = AvroDataDecoder(dict[str, bool])
+        observed = decoder.build(dict())
+        self.assertEqual(dict(), observed)
+
     def test_dict_raises(self):
         decoder = AvroDataDecoder(dict[str, bool])
         with self.assertRaises(DataDecoderTypeException):
@@ -332,9 +359,9 @@ def test_model_optional(self):
         self.assertIsNone(observed)
 
     def test_unsupported(self):
-        decoder = AvroDataDecoder(set[int])
+        decoder = AvroDataDecoder(type[Any])
         with self.assertRaises(DataDecoderTypeException):
-            decoder.build({3})
+            decoder.build(AvroDataDecoder)
 
     def test_empty_raises(self):
         decoder = AvroDataDecoder(date)
@@ -377,6 +404,14 @@ def test_entity_set(self):
 
         self.assertEqual(entities, observed)
 
+    def test_container_inheritance(self):
+        Container = ModelType.build("Container", {"data": list[Base]})
+        raw = Container([User(1, "John"), Analytics(2, dict()), User(3, "John"), Analytics(4, dict())])
+        decoder = AvroDataDecoder(Container)
+        observed = decoder.build(raw)
+
+        self.assertEqual(raw, observed)
+
     def test_entity_set_empty(self):
         entities = EntitySet()
         decoder = AvroDataDecoder(EntitySet[FakeEntity])

tests/test_common/test_model/test_serializers/test_avro_data_encoder.py

@@ -58,6 +58,10 @@ def test_avro_data_list_model(self):
         expected = [{"id": 123, "username": None}, {"id": 456, "username": None}]
         self.assertEqual(expected, observed)
 
+    def test_avro_data_set(self):
+        observed = AvroDataEncoder({1, 2}).build()
+        self.assertEqual([1, 2], observed)
+
     def test_avro_data_dict(self):
         observed = AvroDataEncoder({"foo": 1, "bar": 2}).build()
         self.assertEqual({"bar": 2, "foo": 1}, observed)

tests/test_common/test_model/test_serializers/test_avro_schema_decoder.py

@@ -101,6 +101,12 @@ def test_plain_array(self):
         observed = AvroSchemaDecoder({"name": "example", "type": "array", "items": "string"}).build()
         self.assertEqual(expected, observed)
 
+    def test_set(self):
+        expected = set[str]
+        schema = {"name": "example", "type": "array", "items": "string", "logicalType": "set"}
+        observed = AvroSchemaDecoder(schema).build()
+        self.assertEqual(expected, observed)
+
     def test_plain_map(self):
         expected = dict[str, int]
         observed = AvroSchemaDecoder({"name": "example", "type": "map", "values": "int"}).build()

tests/test_common/test_model/test_serializers/test_avro_schema_encoder.py

@@ -92,6 +92,11 @@ def test_timedelta(self):
         observed = AvroSchemaEncoder(timedelta).build()
         self.assertEqual(expected, observed)
 
+    def test_set(self):
+        expected = {"type": "array", "items": "string", "logicalType": "set"}
+        observed = AvroSchemaEncoder(set[str]).build()
+        self.assertEqual(expected, observed)
+
     def test_dict(self):
         expected = {"type": "map", "values": "int"}
         observed = AvroSchemaEncoder(dict[str, int]).build()

tests/test_common/test_model/test_types/test_builders.py

@@ -37,7 +37,7 @@ def test_dict(self):
         self.assertEqual(dict[str, int], TypeHintBuilder({"one": 1, "two": 2}).build())
 
     def test_dict_empty(self):
-        self.assertEqual(dict[Any, Any], TypeHintBuilder(dict()).build())
+        self.assertEqual(dict[str, Any], TypeHintBuilder(dict()).build())
 
     def test_dict_empty_with_base(self):
         self.assertEqual(dict[str, float], TypeHintBuilder(dict(), dict[str, float]).build())

tests/test_common/test_protocol/test_avro/test_base.py

@@ -84,6 +84,17 @@ def test_timedelta(self):
         deserialized = MinosAvroProtocol.decode(serialized)
         self.assertEqual(data, deserialized)
 
+    def test_set(self):
+        schema = {
+            "type": "record",
+            "name": "tests.model_classes.ShoppingList",
+            "fields": [{"type": {"type": "array", "items": "string", "logicalType": "set"}, "name": "foo"}],
+        }
+        data = {"foo": ["one", "two"]}
+        serialized = MinosAvroProtocol.encode(data, schema)
+        deserialized = MinosAvroProtocol.decode(serialized)
+        self.assertEqual(data, deserialized)
+
 
 if __name__ == "__main__":
     unittest.main()