From f4b62bc539abca74fa48484dfd04c77d43b444d3 Mon Sep 17 00:00:00 2001 From: brokeh Date: Thu, 7 Aug 2025 22:12:06 +1000 Subject: [PATCH] Fix import from Paprika not importing some images This fixes 4 issues that can arise when importing recipes from Paprika: 1. A silent failure to import the image due to an empty (0 bytes) or incomplete JPEG image. There was nothing in the logs, just no image attached to the imported recipe. The problem is that the temp file contents may not have been flushed to disk before copying the temp file to the working directory as `original.jpeg`. This was the main problem I faced when importing from Paprika and meant that none of the images would come across in the import. 2. The below error log when importing a recipe that has no image. ``` Failed to download image for slow-cooker-beef-ragu: slow-cooker-beef-ragu ``` The error log doesn't cause any issues with the import, it's just not particularly helpful and did make me look into it breifly when trying to figure out why the other images failed to import. The error message has been replaced by an info message saying ``` Recipe 'Slow-Cooker Beef Ragu' has no image ``` 3. Failure to import the image for a recipe when there's another recipe with the same name, either in the same import or pre-existing. The failure occurs with the error log ``` Failed to download image for chicken-pot-pie-2: chicken-pot-pie-2 ``` The way it was handling images did not take into account that the imported recipe might end up with a slug that is not the same as the slugified name. 4. Silently failing to import images due to unrecognised file contents when converting to WEBP. For my issue, the unrecognised files were all due to issue 1 making them all empty, but this could also just be for other reasons such as a corrupt file or unrecognised format. Now when an image being imported can't be processed, the below error will be logged ``` Failed to download image for chicken-and-leek-casserole: cannot identify image file '/app/data/recipes/4598f8cb-fabd-4e1e-bca7-a357d4e09f34/images/original.jpeg' ``` This applies to migrations from all supported formats, not just Paprika. --- mealie/services/migrations/_migration_base.py | 8 +++++ mealie/services/migrations/chowdown.py | 4 +-- mealie/services/migrations/nextcloud.py | 10 ++---- mealie/services/migrations/paprika.py | 33 +++++++------------ mealie/services/migrations/recipekeeper.py | 6 ++-- mealie/services/migrations/tandoor.py | 3 +- .../migrations/utils/migration_helpers.py | 7 ++-- 7 files changed, 30 insertions(+), 41 deletions(-) diff --git a/mealie/services/migrations/_migration_base.py b/mealie/services/migrations/_migration_base.py index 0c13867c3..0a953fd74 100644 --- a/mealie/services/migrations/_migration_base.py +++ b/mealie/services/migrations/_migration_base.py @@ -1,6 +1,7 @@ import contextlib from pathlib import Path +from PIL import UnidentifiedImageError from pydantic import UUID4 from mealie.core import root_logger @@ -24,6 +25,7 @@ from mealie.services.scraper import cleaner from .._base_service import BaseService from .utils.database_helpers import DatabaseMigrationHelpers from .utils.migration_alias import MigrationAlias +from .utils.migration_helpers import import_image class BaseMigrator(BaseService): @@ -269,3 +271,9 @@ class BaseMigrator(BaseService): recipe = cleaner.clean(recipe_dict, self.translator, url=recipe_dict.get("org_url", None)) return recipe + + def import_image(self, slug: str, src: str | Path, recipe_id: UUID4): + try: + import_image(src, recipe_id) + except UnidentifiedImageError as e: + self.logger.error(f"Failed to import image for {slug}: {e}") diff --git a/mealie/services/migrations/chowdown.py b/mealie/services/migrations/chowdown.py index 9b8bd1de3..da6c565db 100644 --- a/mealie/services/migrations/chowdown.py +++ b/mealie/services/migrations/chowdown.py @@ -4,7 +4,7 @@ from pathlib import Path from ._migration_base import BaseMigrator from .utils.migration_alias import MigrationAlias -from .utils.migration_helpers import MigrationReaders, import_image, split_by_comma +from .utils.migration_helpers import MigrationReaders, split_by_comma class ChowdownMigrator(BaseMigrator): @@ -64,4 +64,4 @@ class ChowdownMigrator(BaseMigrator): except StopIteration: continue if cd_image: - import_image(cd_image, recipe_id) + self.import_image(slug, cd_image, recipe_id) diff --git a/mealie/services/migrations/nextcloud.py b/mealie/services/migrations/nextcloud.py index c8b897dbe..d2a2a5888 100644 --- a/mealie/services/migrations/nextcloud.py +++ b/mealie/services/migrations/nextcloud.py @@ -9,13 +9,7 @@ from mealie.schema.reports.reports import ReportEntryCreate from ._migration_base import BaseMigrator from .utils.migration_alias import MigrationAlias -from .utils.migration_helpers import ( - MigrationReaders, - glob_walker, - import_image, - parse_iso8601_duration, - split_by_comma, -) +from .utils.migration_helpers import MigrationReaders, glob_walker, parse_iso8601_duration, split_by_comma @dataclass @@ -103,4 +97,4 @@ class NextcloudMigrator(BaseMigrator): if status: nc_dir = nextcloud_dirs[slug] if nc_dir.image: - import_image(nc_dir.image, recipe_id) + self.import_image(slug, nc_dir.image, recipe_id) diff --git a/mealie/services/migrations/paprika.py b/mealie/services/migrations/paprika.py index 3f92339af..8696dbc18 100644 --- a/mealie/services/migrations/paprika.py +++ b/mealie/services/migrations/paprika.py @@ -7,13 +7,10 @@ import zipfile from gzip import GzipFile from pathlib import Path -from slugify import slugify - from mealie.schema.recipe import RecipeNote from ._migration_base import BaseMigrator from .utils.migration_alias import MigrationAlias -from .utils.migration_helpers import import_image def paprika_recipes(file: Path): @@ -67,32 +64,26 @@ class PaprikaMigrator(BaseMigrator): ] def _migrate(self) -> None: - recipe_image_urls = {} + recipes = [r for r in paprika_recipes(self.archive) if "name" in r] + recipe_models = [self.clean_recipe_dictionary(r) for r in recipes] + results = self.import_recipes_to_database(recipe_models) - recipes = [] - for recipe in paprika_recipes(self.archive): - if "name" not in recipe: + for (slug, recipe_id, status), recipe in zip(results, recipes, strict=True): + if not status: continue - recipe_model = self.clean_recipe_dictionary(recipe) - - if "photo_data" in recipe: - recipe_image_urls[slugify(recipe["name"])] = recipe["photo_data"] - - recipes.append(recipe_model) - - results = self.import_recipes_to_database(recipes) - - for slug, recipe_id, status in results: - if not status: + image_data = recipe.get("photo_data") + if image_data is None: + self.logger.info(f"Recipe '{recipe['name']}' has no image") continue try: # Images are stored as base64 encoded strings, so we need to decode them before importing. - image = io.BytesIO(base64.b64decode(recipe_image_urls[slug])) + image = io.BytesIO(base64.b64decode(image_data)) with tempfile.NamedTemporaryFile(suffix=".jpeg") as temp_file: temp_file.write(image.read()) + temp_file.flush() path = Path(temp_file.name) - import_image(path, recipe_id) + self.import_image(slug, path, recipe_id) except Exception as e: - self.logger.error(f"Failed to download image for {slug}: {e}") + self.logger.error(f"Failed to import image for {slug}: {e}") diff --git a/mealie/services/migrations/recipekeeper.py b/mealie/services/migrations/recipekeeper.py index ae463e644..c469a3a0f 100644 --- a/mealie/services/migrations/recipekeeper.py +++ b/mealie/services/migrations/recipekeeper.py @@ -8,7 +8,7 @@ from mealie.services.scraper import cleaner from ._migration_base import BaseMigrator from .utils.migration_alias import MigrationAlias -from .utils.migration_helpers import import_image, parse_iso8601_duration +from .utils.migration_helpers import parse_iso8601_duration def clean_instructions(instructions: list[str]) -> list[str]: @@ -98,7 +98,7 @@ class RecipeKeeperMigrator(BaseMigrator): recipes = [self.clean_recipe_dictionary(x) for x in recipes_as_dicts] results = self.import_recipes_to_database(recipes) - for (_, recipe_id, status), recipe in zip(results, recipes, strict=False): + for (slug, recipe_id, status), recipe in zip(results, recipes, strict=False): if status: try: if not recipe or not recipe.image: @@ -107,4 +107,4 @@ class RecipeKeeperMigrator(BaseMigrator): except StopIteration: continue - import_image(recipe.image, recipe_id) + self.import_image(slug, recipe.image, recipe_id) diff --git a/mealie/services/migrations/tandoor.py b/mealie/services/migrations/tandoor.py index ef78698ab..70432e33f 100644 --- a/mealie/services/migrations/tandoor.py +++ b/mealie/services/migrations/tandoor.py @@ -10,7 +10,6 @@ from mealie.schema.reports.reports import ReportEntryCreate from ._migration_base import BaseMigrator from .utils.migration_alias import MigrationAlias -from .utils.migration_helpers import import_image def _build_ingredient_from_ingredient_data(ingredient_data: dict[str, Any], title: str | None = None) -> dict[str, Any]: @@ -150,4 +149,4 @@ class TandoorMigrator(BaseMigrator): except StopIteration: continue - import_image(r.image, recipe_id) + self.import_image(slug, r.image, recipe_id) diff --git a/mealie/services/migrations/utils/migration_helpers.py b/mealie/services/migrations/utils/migration_helpers.py index 293714955..a7f928212 100644 --- a/mealie/services/migrations/utils/migration_helpers.py +++ b/mealie/services/migrations/utils/migration_helpers.py @@ -104,6 +104,7 @@ def import_image(src: str | Path, recipe_id: UUID4): """Read the successful migrations attribute and for each import the image appropriately into the image directory. Minification is done in mass after the migration occurs. + May raise an UnidentifiedImageError if the file is not a recognised format. """ if isinstance(src, str): @@ -113,11 +114,7 @@ def import_image(src: str | Path, recipe_id: UUID4): return data_service = RecipeDataService(recipe_id=recipe_id) - - try: - data_service.write_image(src, src.suffix) - except UnidentifiedImageError: - return + data_service.write_image(src, src.suffix) async def scrape_image(image_url: str, recipe_id: UUID4):