diff --git a/.gitignore b/.gitignore
index c76b249726a..1259a442359 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@ docs/site/
frontend/dist/
dev/code-generation/generated/*
+dev/data/mealie.db-journal
dev/data/backups/*
dev/data/debug/*
dev/data/img/*
diff --git a/frontend/lang/messages/en-US.json b/frontend/lang/messages/en-US.json
index d52a11988b0..a4db6b14a1b 100644
--- a/frontend/lang/messages/en-US.json
+++ b/frontend/lang/messages/en-US.json
@@ -283,17 +283,34 @@
"meal-plan-settings": "Meal Plan Settings"
},
"migration": {
+ "migration-data-removed": "Migration data removed",
+ "new-migration": "New Migration",
+ "no-file-selected": "No File Selected",
+ "no-migration-data-available": "No Migration Data Available",
+ "previous-migrations": "Previous Migrations",
+ "recipe-migration": "Recipe Migration",
"chowdown": {
"description": "Migrate data from Chowdown",
+ "description-long": "Mealie natively supports the chowdown repository format. Download the code repository as a .zip file and upload it below.",
"title": "Chowdown"
},
- "migration-data-removed": "Migration data removed",
"nextcloud": {
"description": "Migrate data from a Nextcloud Cookbook instance",
+ "description-long": "Nextcloud recipes can be imported from a zip file that contains the data stored in Nextcloud. See the example folder structure below to ensure your recipes are able to be imported.",
"title": "Nextcloud Cookbook"
},
- "no-migration-data-available": "No Migration Data Available",
- "recipe-migration": "Recipe Migration"
+ "copymethat": {
+ "description-long": "Mealie can import recipes from Copy Me That. Export your recipes in HTML format, then upload the .zip below.",
+ "title": "Copy Me That Recipe Manager"
+ },
+ "paprika": {
+ "description-long": "Mealie can import recipes from the Paprika application. Export your recipes from paprika, rename the export extension to .zip and upload it below.",
+ "title": "Paprika Recipe Manager"
+ },
+ "mealie-pre-v1": {
+ "description-long": "Mealie can import recipes from the Mealie application from a pre v1.0 release. Export your recipes from your old instance, and upload the zip file below. Note that only recipes can be imported from the export.",
+ "title": "Mealie Pre v1.0"
+ }
},
"new-recipe": {
"bulk-add": "Bulk Add",
diff --git a/frontend/lib/api/types/group.ts b/frontend/lib/api/types/group.ts
index eaee22ec294..1f448dc0a1b 100644
--- a/frontend/lib/api/types/group.ts
+++ b/frontend/lib/api/types/group.ts
@@ -6,7 +6,7 @@
*/
export type WebhookType = "mealplan";
-export type SupportedMigrations = "nextcloud" | "chowdown" | "paprika" | "mealie_alpha";
+export type SupportedMigrations = "nextcloud" | "chowdown" | "copymethat" | "paprika" | "mealie_alpha";
export interface CreateGroupPreferences {
privateGroup?: boolean;
diff --git a/frontend/pages/group/migrations.vue b/frontend/pages/group/migrations.vue
index 2394ab5c0b1..224d5c01349 100644
--- a/frontend/pages/group/migrations.vue
+++ b/frontend/pages/group/migrations.vue
@@ -14,7 +14,7 @@
Mealie.
-
+
Choose Migration Type
@@ -39,7 +39,7 @@
:text-btn="false"
@uploaded="setFileObject"
/>
- {{ fileObject.name || "No file selected" }}
+ {{ fileObject.name || $i18n.tc('migration.no-file-selected') }}
@@ -58,7 +58,7 @@
-
+
@@ -74,13 +74,14 @@ import { SupportedMigrations } from "~/lib/api/types/group";
const MIGRATIONS = {
nextcloud: "nextcloud",
chowdown: "chowdown",
+ copymethat: "copymethat",
paprika: "paprika",
mealie: "mealie_alpha",
};
export default defineComponent({
setup() {
- const { $globals } = useContext();
+ const { $globals, i18n } = useContext();
const api = useUserApi();
@@ -95,26 +96,30 @@ export default defineComponent({
const items: MenuItem[] = [
{
- text: "Nextcloud",
+ text: i18n.tc("migration.nextcloud.title"),
value: MIGRATIONS.nextcloud,
},
{
- text: "Chowdown",
+ text: i18n.tc("migration.chowdown.title"),
value: MIGRATIONS.chowdown,
},
{
- text: "Paprika",
+ text: i18n.tc("migration.copymethat.title"),
+ value: MIGRATIONS.copymethat,
+ },
+ {
+ text: i18n.tc("migration.paprika.title"),
value: MIGRATIONS.paprika,
},
{
- text: "Mealie",
+ text: i18n.tc("migration.mealie-pre-v1.title"),
value: MIGRATIONS.mealie,
},
];
const _content = {
[MIGRATIONS.nextcloud]: {
- text: "Nextcloud recipes can be imported from a zip file that contains the data stored in Nextcloud. See the example folder structure below to ensure your recipes are able to be imported.",
+ text: i18n.tc("migration.nextcloud.description-long"),
tree: [
{
id: 1,
@@ -146,7 +151,7 @@ export default defineComponent({
],
},
[MIGRATIONS.chowdown]: {
- text: "Mealie natively supports the chowdown repository format. Download the code repository as a .zip file and upload it below",
+ text: i18n.tc("migration.chowdown.description-long"),
tree: [
{
id: 1,
@@ -177,12 +182,35 @@ export default defineComponent({
},
],
},
+ [MIGRATIONS.copymethat]: {
+ text: i18n.tc("migration.copymethat.description-long"),
+ tree: [
+ {
+ id: 1,
+ icon: $globals.icons.zip,
+ name: "Copy_Me_That_20230306.zip",
+ children: [
+ {
+ id: 2,
+ name: "images",
+ icon: $globals.icons.folderOutline,
+ children: [
+ { id: 3, name: "recipe_1_an5zy.jpg", icon: $globals.icons.fileImage },
+ { id: 4, name: "recipe_2_82el8.jpg", icon: $globals.icons.fileImage },
+ { id: 5, name: "recipe_3_j75qg.jpg", icon: $globals.icons.fileImage },
+ ],
+ },
+ { id: 6, name: "recipes.html", icon: $globals.icons.codeJson }
+ ]
+ }
+ ],
+ },
[MIGRATIONS.paprika]: {
- text: "Mealie can import recipes from the Paprika application. Export your recipes from paprika, rename the export extension to .zip and upload it below.",
+ text: i18n.tc("migration.paprika.description-long"),
tree: false,
},
[MIGRATIONS.mealie]: {
- text: "Mealie can import recipes from the Mealie application from a pre v1.0 release. Export your recipes from your old instance, and upload the zip file below. Note that only recipes can be imported from the export.",
+ text: i18n.tc("migration.mealie-pre-v1.description-long"),
tree: [
{
id: 1,
diff --git a/mealie/routes/groups/controller_group_reports.py b/mealie/routes/groups/controller_group_reports.py
index 18755d07274..40beacc0b2f 100644
--- a/mealie/routes/groups/controller_group_reports.py
+++ b/mealie/routes/groups/controller_group_reports.py
@@ -44,6 +44,6 @@ def get_one(self, item_id: UUID4):
def delete_one(self, item_id: UUID4):
try:
self.mixins.delete_one(item_id) # type: ignore
- return SuccessResponse.respond(self.t("report-deleted"))
+ return SuccessResponse.respond(self.t("group.report-deleted"))
except Exception as ex:
raise HTTPException(500, ErrorResponse.respond("Failed to delete report")) from ex
diff --git a/mealie/routes/groups/controller_migrations.py b/mealie/routes/groups/controller_migrations.py
index f7f717a42ec..76ebfea7526 100644
--- a/mealie/routes/groups/controller_migrations.py
+++ b/mealie/routes/groups/controller_migrations.py
@@ -12,6 +12,7 @@
from mealie.services.migrations import (
BaseMigrator,
ChowdownMigrator,
+ CopyMeThatMigrator,
MealieAlphaMigrator,
NextcloudMigrator,
PaprikaMigrator,
@@ -45,6 +46,7 @@ def start_data_migration(
table: dict[SupportedMigrations, type[BaseMigrator]] = {
SupportedMigrations.chowdown: ChowdownMigrator,
+ SupportedMigrations.copymethat: CopyMeThatMigrator,
SupportedMigrations.mealie_alpha: MealieAlphaMigrator,
SupportedMigrations.nextcloud: NextcloudMigrator,
SupportedMigrations.paprika: PaprikaMigrator,
diff --git a/mealie/schema/group/group_migration.py b/mealie/schema/group/group_migration.py
index b3a6573d9c4..6e02825142a 100644
--- a/mealie/schema/group/group_migration.py
+++ b/mealie/schema/group/group_migration.py
@@ -6,6 +6,7 @@
class SupportedMigrations(str, enum.Enum):
nextcloud = "nextcloud"
chowdown = "chowdown"
+ copymethat = "copymethat"
paprika = "paprika"
mealie_alpha = "mealie_alpha"
diff --git a/mealie/services/migrations/__init__.py b/mealie/services/migrations/__init__.py
index 7111958affd..4610ff7705c 100644
--- a/mealie/services/migrations/__init__.py
+++ b/mealie/services/migrations/__init__.py
@@ -1,4 +1,5 @@
from .chowdown import *
+from .copymethat import *
from .mealie_alpha import *
from .nextcloud import *
from .paprika import *
diff --git a/mealie/services/migrations/_migration_base.py b/mealie/services/migrations/_migration_base.py
index c62c9a735d4..1268a38fb7c 100644
--- a/mealie/services/migrations/_migration_base.py
+++ b/mealie/services/migrations/_migration_base.py
@@ -5,6 +5,7 @@
from pydantic import UUID4
from mealie.core import root_logger
+from mealie.core.exceptions import UnexpectedNone
from mealie.repos.all_repositories import AllRepositories
from mealie.schema.recipe import Recipe
from mealie.schema.recipe.recipe_settings import RecipeSettings
@@ -16,6 +17,7 @@
ReportSummary,
ReportSummaryStatus,
)
+from mealie.services.recipe.recipe_service import RecipeService
from mealie.services.scraper import cleaner
from .._base_service import BaseService
@@ -38,17 +40,27 @@ def __init__(
self.archive = archive
self.db = db
self.session = session
- self.user_id = user_id
- self.group_id = group_id
self.add_migration_tag = add_migration_tag
+ user = db.users.get_one(user_id)
+ if not user:
+ raise UnexpectedNone(f"Cannot find user {user_id}")
+
+ group = db.groups.get_one(group_id)
+ if not group:
+ raise UnexpectedNone(f"Cannot find group {group_id}")
+
+ self.user = user
+ self.group = group
+
self.name = "migration"
self.report_entries = []
self.logger = root_logger.get_logger()
- self.helpers = DatabaseMigrationHelpers(self.db, self.session, self.group_id, self.user_id)
+ self.helpers = DatabaseMigrationHelpers(self.db, self.session, self.group.id, self.user.id)
+ self.recipe_service = RecipeService(db, user, group)
super().__init__()
@@ -60,7 +72,7 @@ def _create_report(self, report_name: str) -> None:
name=report_name,
category=ReportCategory.migration,
status=ReportSummaryStatus.in_progress,
- group_id=self.group_id,
+ group_id=self.group.id,
)
self.report = self.db.group_reports.create(report_to_save)
@@ -117,25 +129,23 @@ def import_recipes_to_database(self, validated_recipes: list[Recipe]) -> list[tu
return_vars: list[tuple[str, UUID4, bool]] = []
- group = self.db.groups.get_one(self.group_id)
-
- if not group or not group.preferences:
+ if not self.group.preferences:
raise ValueError("Group preferences not found")
default_settings = RecipeSettings(
- public=group.preferences.recipe_public,
- show_nutrition=group.preferences.recipe_show_nutrition,
- show_assets=group.preferences.recipe_show_assets,
- landscape_view=group.preferences.recipe_landscape_view,
- disable_comments=group.preferences.recipe_disable_comments,
- disable_amount=group.preferences.recipe_disable_amount,
+ public=self.group.preferences.recipe_public,
+ show_nutrition=self.group.preferences.recipe_show_nutrition,
+ show_assets=self.group.preferences.recipe_show_assets,
+ landscape_view=self.group.preferences.recipe_landscape_view,
+ disable_comments=self.group.preferences.recipe_disable_comments,
+ disable_amount=self.group.preferences.recipe_disable_amount,
)
for recipe in validated_recipes:
recipe.settings = default_settings
- recipe.user_id = self.user_id
- recipe.group_id = self.group_id
+ recipe.user_id = self.user.id
+ recipe.group_id = self.group.id
if recipe.tags:
recipe.tags = self.helpers.get_or_set_tags(x.name for x in recipe.tags)
@@ -151,7 +161,7 @@ def import_recipes_to_database(self, validated_recipes: list[Recipe]) -> list[tu
exception: str | Exception = ""
status = False
try:
- recipe = self.db.recipes.create(recipe)
+ recipe = self.recipe_service.create_one(recipe)
status = True
except Exception as inst:
diff --git a/mealie/services/migrations/copymethat.py b/mealie/services/migrations/copymethat.py
new file mode 100644
index 00000000000..d3b14f28cda
--- /dev/null
+++ b/mealie/services/migrations/copymethat.py
@@ -0,0 +1,123 @@
+import tempfile
+import zipfile
+from datetime import datetime
+from pathlib import Path
+
+from bs4 import BeautifulSoup
+
+from mealie.schema.reports.reports import ReportEntryCreate
+
+from ._migration_base import BaseMigrator
+from .utils.migration_alias import MigrationAlias
+from .utils.migration_helpers import import_image
+
+
+def parse_recipe_tags(tags: list) -> list[str]:
+ """Parses the list of recipe tags and removes invalid ones"""
+
+ updated_tags: list[str] = []
+ for tag in tags:
+ if not tag or not isinstance(tag, str):
+ continue
+
+ if "Tags:" in tag:
+ continue
+
+ updated_tags.append(tag)
+
+ return updated_tags
+
+
+class CopyMeThatMigrator(BaseMigrator):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ self.name = "copymethat"
+
+ self.key_aliases = [
+ MigrationAlias(key="last_made", alias="made_this", func=lambda x: datetime.now()),
+ MigrationAlias(key="notes", alias="recipeNotes"),
+ MigrationAlias(key="orgURL", alias="original_link"),
+ MigrationAlias(key="rating", alias="ratingValue"),
+ MigrationAlias(key="recipeIngredient", alias="recipeIngredients"),
+ MigrationAlias(key="recipeYield", alias="servings", func=lambda x: x.replace(":", ": ")),
+ ]
+
+ def _process_recipe_document(self, source_dir: Path, soup: BeautifulSoup) -> dict:
+ """Reads a single recipe's HTML and converts it to a dictionary"""
+
+ recipe_dict: dict = {}
+ recipe_tags: list[str] = []
+ for tag in soup.find_all():
+ # the recipe image tag has no id, so we parse it directly
+ if tag.name == "img" and "recipeImage" in tag.get("class", []):
+ if image_path := tag.get("src"):
+ recipe_dict["image"] = str(source_dir.joinpath(image_path))
+
+ continue
+
+ # tags (internally named categories) are not in a list, and don't have ids
+ if tag.name == "span" and "recipeCategory" in tag.get("class", []):
+ recipe_tag = tag.get_text(strip=True)
+ if "Tags:" not in recipe_tag:
+ recipe_tags.append(recipe_tag)
+
+ continue
+
+ # add only elements with an id to the recipe dictionary
+ if not (tag_id := tag.get("id")):
+ continue
+
+ # for lists, store the list items as an array (e.g. for recipe instructions)
+ if tag.name in ["ul", "ol"]:
+ recipe_dict[tag_id] = [item.get_text(strip=True) for item in tag.find_all("li", recursive=False)]
+ continue
+
+ # for all other tags, write the text directly to the recipe data
+ recipe_dict[tag_id] = tag.get_text(strip=True)
+
+ if recipe_tags:
+ recipe_dict["tags"] = recipe_tags
+
+ return recipe_dict
+
+ def _migrate(self) -> None:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ with zipfile.ZipFile(self.archive) as zip_file:
+ zip_file.extractall(tmpdir)
+
+ source_dir = Path(tmpdir)
+
+ recipes_as_dicts: list[dict] = []
+ for recipes_data_file in source_dir.glob("*.html"):
+ with open(recipes_data_file, encoding="utf-8") as f:
+ soup = BeautifulSoup(f, "lxml")
+ for recipe_data in soup.find_all("div", class_="recipe"):
+ try:
+ recipes_as_dicts.append(self._process_recipe_document(source_dir, recipe_data))
+
+ # since recipes are stored in one large file, we keep going on error
+ except Exception as e:
+ self.report_entries.append(
+ ReportEntryCreate(
+ report_id=self.report_id,
+ success=False,
+ message="Failed to parse recipe",
+ exception=f"{type(e).__name__}: {e}",
+ )
+ )
+
+ recipes = [self.clean_recipe_dictionary(x) for x in recipes_as_dicts]
+ results = self.import_recipes_to_database(recipes)
+ recipe_lookup = {r.slug: r for r in recipes}
+ for slug, recipe_id, status in results:
+ if status:
+ try:
+ r = recipe_lookup.get(slug)
+ if not r or not r.image:
+ continue
+
+ except StopIteration:
+ continue
+
+ import_image(r.image, recipe_id)
diff --git a/mealie/services/migrations/utils/migration_helpers.py b/mealie/services/migrations/utils/migration_helpers.py
index 20723f5b629..ed1123d1f35 100644
--- a/mealie/services/migrations/utils/migration_helpers.py
+++ b/mealie/services/migrations/utils/migration_helpers.py
@@ -81,10 +81,17 @@ def glob_walker(directory: Path, glob_str: str, return_parent=True) -> list[Path
return matches
-def import_image(src: Path, recipe_id: UUID4):
+def import_image(src: str | Path, recipe_id: UUID4):
"""Read the successful migrations attribute and for each import the image
appropriately into the image directory. Minification is done in mass
after the migration occurs.
"""
+
+ if isinstance(src, str):
+ src = Path(src)
+
+ if not src.exists():
+ return
+
data_service = RecipeDataService(recipe_id=recipe_id)
data_service.write_image(src, src.suffix)
diff --git a/mealie/services/scraper/cleaner.py b/mealie/services/scraper/cleaner.py
index 920927c6614..762abd98783 100644
--- a/mealie/services/scraper/cleaner.py
+++ b/mealie/services/scraper/cleaner.py
@@ -49,7 +49,9 @@ def clean(recipe_data: dict, url=None) -> dict:
recipe_data["recipeInstructions"] = clean_instructions(recipe_data.get("recipeInstructions", []))
recipe_data["image"] = clean_image(recipe_data.get("image"))[0]
recipe_data["slug"] = slugify(recipe_data.get("name", ""))
- recipe_data["orgURL"] = url
+ recipe_data["orgURL"] = url or recipe_data.get("orgURL")
+ recipe_data["notes"] = clean_notes(recipe_data.get("notes"))
+ recipe_data["rating"] = clean_int(recipe_data.get("rating"))
return recipe_data
@@ -255,6 +257,48 @@ def clean_ingredients(ingredients: list | str | None, default: list | None = Non
raise TypeError(f"Unexpected type for ingredients: {type(ingredients)}, {ingredients}")
+def clean_int(val: str | int | None, min: int | None = None, max: int | None = None):
+ if val is None or isinstance(val, int):
+ return val
+
+ filtered_val = "".join(c for c in val if c.isnumeric())
+ if not filtered_val:
+ return None
+
+ val = int(filtered_val)
+ if min is None or max is None:
+ return val
+
+ if not (min <= val <= max):
+ return None
+
+ return val
+
+
+def clean_notes(notes: typing.Any) -> list[dict] | None:
+ if not isinstance(notes, list):
+ return None
+
+ parsed_notes: list[dict] = []
+ for note in notes:
+ if not isinstance(note, (str, dict)):
+ continue
+
+ if isinstance(note, dict):
+ if "text" not in note:
+ continue
+
+ if "title" not in note:
+ note["title"] = ""
+
+ parsed_notes.append(note)
+ continue
+
+ parsed_notes.append({"title": "", "text": note})
+
+ return parsed_notes
+
+
def clean_yield(yld: str | list[str] | None) -> str:
"""
yield_amount attemps to parse out the yield amount from a recipe.
diff --git a/poetry.lock b/poetry.lock
index eaa1c1e88a4..ca24343382a 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -3050,4 +3050,4 @@ pgsql = ["psycopg2-binary"]
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
-content-hash = "c88c209ebd76e8f697a1d1af54496b900d4e16a087ca97be7cb288da039526ce"
+content-hash = "2e8a98d35c22f3afceefbf22d2b2e23d3471eb5af88e3a907ec848370e92dd14"
diff --git a/pyproject.toml b/pyproject.toml
index 1cde68b5f0f..a0e6a0faf94 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,6 +41,7 @@ recipe-scrapers = "^14.26.0"
requests = "^2.25.1"
tzdata = "^2022.7"
uvicorn = {extras = ["standard"], version = "^0.20.0"}
+beautifulsoup4 = "^4.11.2"
[tool.poetry.group.dev.dependencies]
black = "^23.1.0"
diff --git a/tests/data/__init__.py b/tests/data/__init__.py
index abdd46163d1..30be70f5ca5 100644
--- a/tests/data/__init__.py
+++ b/tests/data/__init__.py
@@ -8,6 +8,8 @@
migrations_chowdown = CWD / "migrations/chowdown.zip"
+migrations_copymethat = CWD / "migrations/copymethat.zip"
+
migrations_mealie = CWD / "migrations/mealie.zip"
migrations_nextcloud = CWD / "migrations/nextcloud.zip"
diff --git a/tests/data/migrations/copymethat.zip b/tests/data/migrations/copymethat.zip
new file mode 100644
index 00000000000..49ab75b1e80
Binary files /dev/null and b/tests/data/migrations/copymethat.zip differ
diff --git a/tests/integration_tests/recipe_migration_tests/test_recipe_migrations.py b/tests/integration_tests/recipe_migration_tests/test_recipe_migrations.py
index e2f25916db4..6c5306c660e 100644
--- a/tests/integration_tests/recipe_migration_tests/test_recipe_migrations.py
+++ b/tests/integration_tests/recipe_migration_tests/test_recipe_migrations.py
@@ -7,6 +7,7 @@
from mealie.schema.group.group_migration import SupportedMigrations
from tests import data as test_data
from tests.utils import api_routes
+from tests.utils.assertion_helpers import assert_derserialize
from tests.utils.fixture_schemas import TestUser
@@ -20,6 +21,7 @@ class MigrationTestData:
MigrationTestData(typ=SupportedMigrations.nextcloud, archive=test_data.migrations_nextcloud),
MigrationTestData(typ=SupportedMigrations.paprika, archive=test_data.migrations_paprika),
MigrationTestData(typ=SupportedMigrations.chowdown, archive=test_data.migrations_chowdown),
+ MigrationTestData(typ=SupportedMigrations.copymethat, archive=test_data.migrations_copymethat),
MigrationTestData(typ=SupportedMigrations.mealie_alpha, archive=test_data.migrations_mealie),
]
@@ -27,6 +29,7 @@ class MigrationTestData:
"nextcloud_archive",
"paprika_archive",
"chowdown_archive",
+ "copymethat_archive",
"mealie_alpha_archive",
]
@@ -56,3 +59,15 @@ def test_recipe_migration(api_client: TestClient, unique_user: TestUser, mig: Mi
for item in response.json()["entries"]:
assert item["success"]
+
+ # Validate Create Event
+ params = {"orderBy": "created_at", "orderDirection": "desc"}
+ response = api_client.get(api_routes.recipes, params=params, headers=unique_user.token)
+ query_data = assert_derserialize(response)
+ assert len(query_data["items"])
+ slug = query_data["items"][0]["slug"]
+
+ response = api_client.get(api_routes.recipes_slug_timeline_events(slug), headers=unique_user.token)
+ query_data = assert_derserialize(response)
+ events = query_data["items"]
+ assert len(events)