Map default values to new model data during migration

Author: martastain

ayon_server/addons/addon.py

@@ -266,7 +266,9 @@ async def get_studio_overrides(
                 )
 
             try:
-                return target_addon.convert_settings_overrides(self.version, data)
+                return await target_addon.convert_settings_overrides(
+                    self.version, overrides=data
+                )
             except Exception:
                 log_traceback(f"Unable to migrate {self} settings to {as_version}")
                 return {}
@@ -305,7 +307,9 @@ async def get_project_overrides(
                 )
 
             try:
-                return target_addon.convert_settings_overrides(self.version, data)
+                return await target_addon.convert_settings_overrides(
+                    self.version, overrides=data
+                )
             except Exception:
                 log_traceback(f"Unable to migrate {self} settings to {as_version}")
                 return {}
@@ -346,7 +350,9 @@ async def get_project_site_overrides(
                 )
 
             try:
-                return target_addon.convert_settings_overrides(self.version, data)
+                return await target_addon.convert_settings_overrides(
+                    self.version, overrides=data
+                )
             except Exception:
                 log_traceback(f"Unable to migrate {self} settings to {as_version}")
                 return {}
@@ -487,7 +493,7 @@ async def get_default_settings(self) -> BaseSettingsModel | None:
             return None
         return model()
 
-    def convert_settings_overrides(
+    async def convert_settings_overrides(
         self,
         source_version: str,
         overrides: dict[str, Any],
@@ -507,7 +513,7 @@ def convert_settings_overrides(
         has been changed from `str` to `list[str]`, you may use:
 
         ```python
-        def convert_str_to_list_str(value: str | list[str]) -> list[str]:
+        await def convert_str_to_list_str(value: str | list[str]) -> list[str]:
             if isinstance(value, str):
                 return [value]
             elif isinstance(value, list):
@@ -527,5 +533,8 @@ def convert_str_to_list_str(value: str | list[str]) -> list[str]:
         model_class = self.get_settings_model()
         if model_class is None:
             return {}
-        result = migrate_settings(overrides, new_model_class=model_class)
+        defaults = await self.get_default_settings()
+        result = migrate_settings(
+            overrides, new_model_class=model_class, defaults=defaults.dict()
+        )
         return result.dict(exclude_unset=True, exclude_none=True, exclude_defaults=True)

ayon_server/settings/common.py

@@ -2,6 +2,7 @@
 import re
 from typing import Any, Callable, Type
 
+from nxtools import logging
 from pydantic import BaseModel, ValidationError, parse_obj_as
 
 from ayon_server.utils import json_dumps, json_loads
@@ -28,6 +29,7 @@ class Config:
 def migrate_settings(
     old_data: dict[str, Any],
     new_model_class: Type[BaseSettingsModel],
+    defaults: dict[str, Any],
     custom_conversions: dict[str, Callable[[Any], Any]] = {},
     parent_key: str = "",
 ) -> Any:
@@ -51,27 +53,36 @@ def migrate_settings(
                         migrate_settings(
                             old_value,
                             field_type.type_,
+                            defaults.get(field_name, []),
                             custom_conversions,
                             key_path,
                         )
                         for old_value in old_value
                     ]
-                else:
+                elif isinstance(old_value, dict):
                     new_instance_data[field_name] = migrate_settings(
                         old_value,
                         field_type.outer_type_,
+                        defaults.get(field_name, {}),
                         custom_conversions,
                         key_path,
                     )
+                else:
+                    logging.warning(
+                        f"Unsupported type for {key_path} model: {old_value}"
+                    )
             else:
                 try:
                     validated_value = parse_obj_as(field_type.outer_type_, old_value)
                     new_instance_data[field_name] = validated_value
                 except ValidationError:
+                    logging.warning(
+                        f"Failed to validate {field_name} with value {old_value}"
+                    )
                     # Skip incompatible fields
                     continue
         else:
-            # Skip fields not in the old data, allowing new model's defaults to apply
-            continue
+            if field_name in defaults:
+                new_instance_data[field_name] = defaults.get(field_name, None)
 
     return new_model_class.parse_obj(new_instance_data)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ayon-server"
-version = "1.0.5"
+version = "1.0.6"
 description = ""
 authors = ["Ynput <info@ynput.io>"]