Improve perfomance for apply events

- upgrade to sqlalchemy 2
- cache event from_value
- add _gobid to confirms
- allow create DateTime from Date
- Minimize entity attribute setting in MODIFY

gobcore/events/import_events.py

@@ -174,21 +174,22 @@ class MODIFY(ImportEvent):
     name = "MODIFY"
     timestamp_field = "_date_modified"
 
+    skip = {"_entity_source_id", "_source_id", "_tid"}
+
     def apply_to(self, entity):
         # Set the hash
         entity._hash = self._data[hash_key]
 
         # Extract modifications from the data, before applying the event to the entity.
         modifications = self._data.pop(modifications_key)
-        attribute_set = self._extract_modifications(entity, modifications)
+        attribute_set = self._extract_modifications(modifications)
         self._data = {**self._data, **attribute_set}
 
         super().apply_to(entity)
 
-    def _extract_modifications(self, entity, modifications):
+    def _extract_modifications(self, modifications):
         """Extracts attributes to modify, and checks if old values are indeed present on entity.
 
-        :param entity: the instance to be modified -- unused argument!
         :param modifications: a collection of mutations of attributes to be interpretated
 
         :return: a dict with extracted and verified mutations
@@ -214,6 +215,19 @@ def create_event(cls, _tid, data, version):
 
         return super().create_event(_tid, mods, version)
 
+    def get_attribute_dict(self):
+        fields = self._model['all_fields']
+
+        def get_value(key, value):
+            type_info = fields[key]
+            gob_type = get_gob_type_from_info(type_info)
+            return gob_type.from_value(value, **get_kwargs_from_type_info(type_info)).to_db
+
+        return {
+            self.timestamp_field: self._metadata.timestamp,
+            **{k: v if v is None else get_value(k, v) for k, v in self._data.items() if k not in self.skip}
+        }
+
 
 class DELETE(ImportEvent):
     """
@@ -252,7 +266,7 @@ class CONFIRM(ImportEvent):
     @classmethod
     def create_event(cls, _tid, data, version):
         #  CONFIRM has no data, except reference to entity age
-        return super().create_event(_tid, cls.last_event(data), version)
+        return super().create_event(_tid, cls.last_event(data) | {"_gobid": data["_gobid"]}, version)
 
 
 class BULKCONFIRM(ImportEvent):

gobcore/typesystem/gob_types.py

@@ -21,6 +21,7 @@
 import numbers
 import re
 from abc import ABCMeta, abstractmethod
+from functools import lru_cache, cached_property
 from math import isnan
 from typing import Any, Optional
 
@@ -30,6 +31,9 @@
 from gobcore.model.metadata import FIELD
 
 
+cache = lru_cache(250_000, typed=True)
+
+
 def get_kwargs_from_type_info(type_info: dict[str, Any]) -> dict[str, Any]:
     """Return kwargs dictionary from GOB Model field type info."""
     # Collect special keys like 'precision'.
@@ -202,6 +206,7 @@ class Character(String):
     sql_type = sqlalchemy.CHAR
 
     @classmethod
+    @cache
     def from_value(cls, value, **kwargs) -> GOBType:
         """
         Returns GOBType as a String containing a single character value if input has a string representation with
@@ -286,6 +291,7 @@ def __init__(self, value: Optional[str], **kwargs) -> None:  # noqa: C901
         super().__init__(value)
 
     @classmethod
+    @cache
     def from_value(cls, value, **kwargs):
         """Create a Decimal GOB Type from value and kwargs.
 
@@ -346,6 +352,7 @@ def __init__(self, value):
         super().__init__(value)
 
     @classmethod
+    @cache
     def from_value(cls, value, **kwargs):
         """Create a Boolean GOB Type from value and kwargs.
 
@@ -405,6 +412,7 @@ class Date(String):
     internal_format = "%Y-%m-%d"
 
     @classmethod
+    @cache
     def from_value(cls, value, **kwargs):
         """ Create a Date GOB type as a string containing a date value in ISO 8601 format:
 
@@ -424,13 +432,13 @@ def from_value(cls, value, **kwargs):
 
         return cls(str(value)) if value is not None else cls(None)
 
-    @property
+    @cached_property
     def to_db(self):
         if self._string is None:
             return None
         return datetime.datetime.strptime(self._string, self.internal_format)
 
-    @property
+    @cached_property
     def to_value(self):
         if self._string is None:
             return None
@@ -448,6 +456,7 @@ def __init__(self, value):
         super().__init__(value)
 
     @classmethod
+    @cache
     def from_value(cls, value, **kwargs):
         input_format = kwargs['format'] if 'format' in kwargs else cls.internal_format
 
@@ -457,6 +466,8 @@ def from_value(cls, value, **kwargs):
                     if isinstance(value, str) and '.%f' in input_format and len(value) == len('YYYY-MM-DDTHH:MM:SS'):
                         # Add missing microseconds if needed
                         value += '.000000'
+                    elif isinstance(value, str) and len(value) == len('YYYY-MM-DD'):
+                        value += "T00:00:00.000000"
                     value = datetime.datetime.strptime(str(value), input_format)
                 # Transform to internal string format and work around issue: https://bugs.python.org/issue13305
                 value = f"{value.year:04d}-" + value.strftime("%m-%dT%H:%M:%S.%f")
@@ -465,13 +476,13 @@ def from_value(cls, value, **kwargs):
 
         return cls(str(value)) if value is not None else cls(None)
 
-    @property
+    @cached_property
     def to_db(self):
         if self._string is None:
             return None
         return datetime.datetime.strptime(self._string, self.internal_format)
 
-    @property
+    @cached_property
     def to_value(self):
         if self._string is None:
             return None

requirements.txt

@@ -20,7 +20,7 @@ python-keystoneclient~=5.1.0
 python-swiftclient==4.3.0
 requests~=2.31.0
 Shapely==1.8.5.post1
-SQLAlchemy~=1.4.49
+SQLAlchemy~=2.0.18
 urllib3~=1.26.16
 
 # Test requirements