fix: More Backup Restore Fixes (#2859)

* refactor normalized search migration to use dummy default

* changed group slug migration to use raw SQL

* updated comment

* added tests with anonymized backups (currently failing)

* typo

* fixed LDAP enum in test data

* fix for adding label settings across groups

* add migration data fixes

* fix shopping list label settings test

* re-run db init instead of just running alembic migration, to include fixes

* intentionally broke SQLAlchemy GUID handling

* safely convert between GUID types in different databases

* restore original test data after testing backup restores

* added missing group name update to migration

Author: michael-genson

alembic/versions/2023-02-14-20.45.41_5ab195a474eb_add_normalized_search_properties.py

@@ -7,12 +7,11 @@
 """
 import sqlalchemy as sa
 from sqlalchemy import orm, select
-from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 from text_unidecode import unidecode
 
 import mealie.db.migration_types
 from alembic import op
-
 from mealie.db.models._model_utils import GUID
 
 # revision identifiers, used by Alembic.
@@ -52,30 +51,46 @@ def do_data_migration():
     session = orm.Session(bind=bind)
 
     recipes = session.execute(select(RecipeModel)).scalars().all()
-    ingredients = session.execute(select(RecipeIngredient)).scalars().all()
     for recipe in recipes:
         if recipe.name is not None:
-            recipe.name_normalized = unidecode(recipe.name).lower().strip()
+            session.execute(
+                sa.text(
+                    f"UPDATE {RecipeModel.__tablename__} SET name_normalized=:name_normalized WHERE id=:id"
+                ).bindparams(name_normalized=unidecode(recipe.name).lower().strip(), id=recipe.id)
+            )
 
         if recipe.description is not None:
-            recipe.description_normalized = unidecode(recipe.description).lower().strip()
-        session.add(recipe)
+            session.execute(
+                sa.text(
+                    f"UPDATE {RecipeModel.__tablename__} SET description_normalized=:description_normalized WHERE id=:id"
+                ).bindparams(description_normalized=unidecode(recipe.description).lower().strip(), id=recipe.id)
+            )
 
+    ingredients = session.execute(select(RecipeIngredient)).scalars().all()
     for ingredient in ingredients:
         if ingredient.note is not None:
-            ingredient.note_normalized = unidecode(ingredient.note).lower().strip()
+            session.execute(
+                sa.text(
+                    f"UPDATE {RecipeIngredient.__tablename__} SET note_normalized=:note_normalized WHERE id=:id"
+                ).bindparams(note_normalized=unidecode(ingredient.note).lower().strip(), id=ingredient.id)
+            )
 
         if ingredient.original_text is not None:
-            ingredient.original_text_normalized = unidecode(ingredient.original_text).lower().strip()
-        session.add(ingredient)
+            session.execute(
+                sa.text(
+                    f"UPDATE {RecipeIngredient.__tablename__} SET original_text_normalized=:original_text_normalized WHERE id=:id"
+                ).bindparams(
+                    original_text_normalized=unidecode(ingredient.original_text).lower().strip(), id=ingredient.id
+                )
+            )
     session.commit()
 
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
 
-    # Set column to nullable first, since we do not have values here yet
-    op.add_column("recipes", sa.Column("name_normalized", sa.String(), nullable=True))
+    # Set column default first, since we do not have values here yet
+    op.add_column("recipes", sa.Column("name_normalized", sa.String(), nullable=False, server_default=""))
     op.add_column("recipes", sa.Column("description_normalized", sa.String(), nullable=True))
     op.drop_index("ix_recipes_description", table_name="recipes")
     op.drop_index("ix_recipes_name", table_name="recipes")
@@ -95,9 +110,9 @@ def upgrade():
         unique=False,
     )
     do_data_migration()
-    # Make recipes.name_normalized not nullable now that column should be filled for all rows
+    # Remove server default now that column should be filled for all rows
     with op.batch_alter_table("recipes", schema=None) as batch_op:
-        batch_op.alter_column("name_normalized", nullable=False, existing_type=sa.String())
+        batch_op.alter_column("name_normalized", existing_type=sa.String(), server_default=None)
     # ### end Alembic commands ###
 
 

alembic/versions/2023-02-21-22.03.19_b04a08da2108_added_shopping_list_label_settings.py

@@ -24,10 +24,10 @@
 
 def populate_shopping_lists_multi_purpose_labels(shopping_lists_multi_purpose_labels_table: sa.Table, session: Session):
     shopping_lists = session.query(ShoppingList).all()
-    labels = session.query(MultiPurposeLabel).all()
 
     shopping_lists_labels_data: list[dict] = []
     for shopping_list in shopping_lists:
+        labels = session.query(MultiPurposeLabel).filter(MultiPurposeLabel.group_id == ShoppingList.group_id).all()
         for i, label in enumerate(labels):
             shopping_lists_labels_data.append(
                 {"id": uuid4(), "shopping_list_id": shopping_list.id, "label_id": label.id, "position": i}

alembic/versions/2023-08-06-21.00.34_04ac51cbe9a4_added_group_slug.py

@@ -24,17 +24,22 @@ def populate_group_slugs(session: Session):
     seen_slugs: set[str] = set()
     for group in groups:
         original_name = group.name
+        new_name = original_name
         attempts = 0
         while True:
-            slug = slugify(group.name)
+            slug = slugify(new_name)
             if slug not in seen_slugs:
                 break
 
             attempts += 1
-            group.name = f"{original_name} ({attempts})"
+            new_name = f"{original_name} ({attempts})"
 
         seen_slugs.add(slug)
-        group.slug = slug
+        session.execute(
+            sa.text(f"UPDATE {Group.__tablename__} SET name=:name, slug=:slug WHERE id=:id").bindparams(
+                name=new_name, slug=slug, id=group.id
+            )
+        )
 
     session.commit()
 

dev/code-generation/utils/anonymize_backups.py

@@ -0,0 +1,74 @@
+import json
+import logging
+import random
+import string
+from datetime import datetime
+from uuid import UUID
+
+logger = logging.getLogger("anonymize_backups")
+
+
+def is_uuid4(value: str):
+    try:
+        UUID(value)
+        return True
+    except ValueError:
+        return False
+
+
+def is_iso_datetime(value: str):
+    try:
+        datetime.fromisoformat(value)
+        return True
+    except ValueError:
+        return False
+
+
+def random_string(length=10):
+    return "".join(random.choice(string.ascii_lowercase) for _ in range(length))
+
+
+def clean_value(value):
+    try:
+        match value:
+            # preserve non-strings
+            case int(value) | float(value):
+                return value
+            case None:
+                return value
+            # preserve UUIDs and datetimes
+            case str(value) if is_uuid4(value) or is_iso_datetime(value):
+                return value
+            # randomize strings
+            case str(value):
+                return random_string()
+            case _:
+                pass
+
+    except Exception as e:
+        logger.exception(e)
+
+    logger.error(f"Failed to anonymize value: {value}")
+    return value
+
+
+def walk_data_and_anonymize(data):
+    for k, v in data.items():
+        if isinstance(v, list):
+            for item in v:
+                walk_data_and_anonymize(item)
+        else:
+            # preserve alembic version number and enums
+            if k in ["auth_method", "version_num"]:
+                continue
+
+            data[k] = clean_value(v)
+
+
+def anonymize_database_json(input_filepath: str, output_filepath: str):
+    with open(input_filepath) as f:
+        data = json.load(f)
+
+    walk_data_and_anonymize(data)
+    with open(output_filepath, "w") as f:
+        json.dump(data, f)

mealie/db/fixes/fix_migration_data.py

@@ -0,0 +1,150 @@
+from uuid import uuid4
+
+from slugify import slugify
+from sqlalchemy.orm import Session
+
+from mealie.core import root_logger
+from mealie.db.models.group.group import Group
+from mealie.db.models.group.shopping_list import ShoppingList, ShoppingListMultiPurposeLabel
+from mealie.db.models.labels import MultiPurposeLabel
+from mealie.db.models.recipe.ingredient import IngredientFoodModel, IngredientUnitModel
+from mealie.db.models.recipe.recipe import RecipeModel
+
+logger = root_logger.get_logger("init_db")
+
+
+def fix_recipe_normalized_search_properties(session: Session):
+    recipes = session.query(RecipeModel).all()
+    recipes_fixed = False
+
+    for recipe in recipes:
+        add_to_session = False
+        if recipe.name and not recipe.name_normalized:
+            recipe.name_normalized = RecipeModel.normalize(recipe.name)
+            add_to_session = True
+        if recipe.description and not recipe.description_normalized:
+            recipe.description_normalized = RecipeModel.normalize(recipe.description)
+            add_to_session = True
+
+        for ingredient in recipe.recipe_ingredient:
+            if ingredient.note and not ingredient.note_normalized:
+                ingredient.note_normalized = RecipeModel.normalize(ingredient.note)
+                add_to_session = True
+            if ingredient.original_text and not ingredient.original_text_normalized:
+                ingredient.original_text = RecipeModel.normalize(ingredient.original_text_normalized)
+                add_to_session = True
+
+        if add_to_session:
+            recipes_fixed = True
+            session.add(recipe)
+
+    if recipes_fixed:
+        logger.info("Updating recipe normalized search properties")
+        session.commit()
+
+
+def fix_shopping_list_label_settings(session: Session):
+    shopping_lists = session.query(ShoppingList).all()
+    labels = session.query(MultiPurposeLabel).all()
+    label_settings_fixed = False
+
+    for shopping_list in shopping_lists:
+        labels_by_id = {label.id: label for label in labels if label.group_id == shopping_list.group_id}
+        for label_setting in shopping_list.label_settings:
+            if not labels_by_id.pop(label_setting.label_id, None):
+                # label setting is no longer valid, so delete it
+                session.delete(label_setting)
+                label_settings_fixed = True
+
+        if not labels_by_id:
+            # all labels are accounted for, so we don't need to add any
+            continue
+
+        label_settings_fixed = True
+        for i, label in enumerate(labels_by_id.values()):
+            new_label_setting = ShoppingListMultiPurposeLabel(
+                id=uuid4(),
+                shopping_list_id=shopping_list.id,
+                label_id=label.id,
+                position=i + len(shopping_list.label_settings),
+            )
+
+            session.add(new_label_setting)
+
+    if label_settings_fixed:
+        logger.info("Fixing shopping list label settings")
+        session.commit()
+
+
+def fix_group_slugs(session: Session):
+    groups = session.query(Group).all()
+    seen_slugs: set[str] = set()
+    groups_fixed = False
+
+    for group in groups:
+        if not group.slug:
+            original_name = group.name
+            new_name = original_name
+            attempts = 0
+            while True:
+                slug = slugify(group.name)
+                if slug not in seen_slugs:
+                    break
+
+                attempts += 1
+                new_name = f"{original_name} ({attempts})"
+
+            groups_fixed = True
+            group.name = new_name
+            group.slug = slug
+
+    if groups_fixed:
+        logger.info("Adding missing group slugs")
+        session.commit()
+
+
+def fix_normalized_unit_and_food_names(session: Session):
+    units = session.query(IngredientUnitModel).all()
+    units_fixed = False
+
+    for unit in units:
+        add_to_session = False
+        if unit.name and not unit.name_normalized:
+            unit.name_normalized = IngredientUnitModel.normalize(unit.name)
+            add_to_session = True
+        if unit.abbreviation and not unit.abbreviation_normalized:
+            unit.abbreviation_normalized = IngredientUnitModel.normalize(unit.abbreviation)
+            add_to_session = True
+
+        if add_to_session:
+            units_fixed = True
+            session.add(unit)
+
+    if units_fixed:
+        logger.info("Updating unit normalized search properties")
+        session.commit()
+
+    foods = session.query(IngredientFoodModel).all()
+    foods_fixed = False
+
+    for food in foods:
+        add_to_session = False
+        if food.name and not food.name_normalized:
+            food.name_normalized = IngredientFoodModel.normalize(food.name)
+            add_to_session = True
+
+        if add_to_session:
+            foods_fixed = True
+            session.add(food)
+
+    if foods_fixed:
+        logger.info("Updating food normalized search properties")
+        session.commit()
+
+
+def fix_migration_data(session: Session):
+    logger.info("Checking for migration data fixes")
+    fix_recipe_normalized_search_properties(session)
+    fix_shopping_list_label_settings(session)
+    fix_group_slugs(session)
+    fix_normalized_unit_and_food_names(session)

mealie/db/init_db.py

@@ -11,6 +11,7 @@
 from mealie.core.config import get_app_settings
 from mealie.db.db_setup import session_context
 from mealie.db.fixes.fix_group_with_no_name import fix_group_with_no_name
+from mealie.db.fixes.fix_migration_data import fix_migration_data
 from mealie.db.fixes.fix_slug_foods import fix_slug_food_names
 from mealie.repos.all_repositories import get_repositories
 from mealie.repos.repository_factory import AllRepositories
@@ -97,16 +98,16 @@ def main():
             session.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm;"))
 
         db = get_repositories(session)
+        safe_try(lambda: fix_migration_data(session))
+        safe_try(lambda: fix_slug_food_names(db))
+        safe_try(lambda: fix_group_with_no_name(session))
 
         if db.users.get_all():
             logger.debug("Database exists")
         else:
             logger.info("Database contains no users, initializing...")
             init_db(db)
 
-        safe_try(lambda: fix_slug_food_names(db))
-        safe_try(lambda: fix_group_with_no_name(session))
-
 
 if __name__ == "__main__":
     main()