mirror of
https://github.com/hay-kot/mealie.git
synced 2025-08-20 13:33:39 -07:00
Fix import from Paprika not importing some images
This fixes 4 issues that can arise when importing recipes from Paprika: 1. A silent failure to import the image due to an empty (0 bytes) or incomplete JPEG image. There was nothing in the logs, just no image attached to the imported recipe. The problem is that the temp file contents may not have been flushed to disk before copying the temp file to the working directory as `original.jpeg`. This was the main problem I faced when importing from Paprika and meant that none of the images would come across in the import. 2. The below error log when importing a recipe that has no image. ``` Failed to download image for slow-cooker-beef-ragu: slow-cooker-beef-ragu ``` The error log doesn't cause any issues with the import, it's just not particularly helpful and did make me look into it breifly when trying to figure out why the other images failed to import. The error message has been replaced by an info message saying ``` Recipe 'Slow-Cooker Beef Ragu' has no image ``` 3. Failure to import the image for a recipe when there's another recipe with the same name, either in the same import or pre-existing. The failure occurs with the error log ``` Failed to download image for chicken-pot-pie-2: chicken-pot-pie-2 ``` The way it was handling images did not take into account that the imported recipe might end up with a slug that is not the same as the slugified name. 4. Silently failing to import images due to unrecognised file contents when converting to WEBP. For my issue, the unrecognised files were all due to issue 1 making them all empty, but this could also just be for other reasons such as a corrupt file or unrecognised format. Now when an image being imported can't be processed, the below error will be logged ``` Failed to download image for chicken-and-leek-casserole: cannot identify image file '/app/data/recipes/4598f8cb-fabd-4e1e-bca7-a357d4e09f34/images/original.jpeg' ``` This applies to migrations from all supported formats, not just Paprika.
This commit is contained in:
parent
c4fdcec85f
commit
f4b62bc539
7 changed files with 30 additions and 41 deletions
|
@ -1,6 +1,7 @@
|
|||
import contextlib
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import UnidentifiedImageError
|
||||
from pydantic import UUID4
|
||||
|
||||
from mealie.core import root_logger
|
||||
|
@ -24,6 +25,7 @@ from mealie.services.scraper import cleaner
|
|||
from .._base_service import BaseService
|
||||
from .utils.database_helpers import DatabaseMigrationHelpers
|
||||
from .utils.migration_alias import MigrationAlias
|
||||
from .utils.migration_helpers import import_image
|
||||
|
||||
|
||||
class BaseMigrator(BaseService):
|
||||
|
@ -269,3 +271,9 @@ class BaseMigrator(BaseService):
|
|||
|
||||
recipe = cleaner.clean(recipe_dict, self.translator, url=recipe_dict.get("org_url", None))
|
||||
return recipe
|
||||
|
||||
def import_image(self, slug: str, src: str | Path, recipe_id: UUID4):
|
||||
try:
|
||||
import_image(src, recipe_id)
|
||||
except UnidentifiedImageError as e:
|
||||
self.logger.error(f"Failed to import image for {slug}: {e}")
|
||||
|
|
|
@ -4,7 +4,7 @@ from pathlib import Path
|
|||
|
||||
from ._migration_base import BaseMigrator
|
||||
from .utils.migration_alias import MigrationAlias
|
||||
from .utils.migration_helpers import MigrationReaders, import_image, split_by_comma
|
||||
from .utils.migration_helpers import MigrationReaders, split_by_comma
|
||||
|
||||
|
||||
class ChowdownMigrator(BaseMigrator):
|
||||
|
@ -64,4 +64,4 @@ class ChowdownMigrator(BaseMigrator):
|
|||
except StopIteration:
|
||||
continue
|
||||
if cd_image:
|
||||
import_image(cd_image, recipe_id)
|
||||
self.import_image(slug, cd_image, recipe_id)
|
||||
|
|
|
@ -9,13 +9,7 @@ from mealie.schema.reports.reports import ReportEntryCreate
|
|||
|
||||
from ._migration_base import BaseMigrator
|
||||
from .utils.migration_alias import MigrationAlias
|
||||
from .utils.migration_helpers import (
|
||||
MigrationReaders,
|
||||
glob_walker,
|
||||
import_image,
|
||||
parse_iso8601_duration,
|
||||
split_by_comma,
|
||||
)
|
||||
from .utils.migration_helpers import MigrationReaders, glob_walker, parse_iso8601_duration, split_by_comma
|
||||
|
||||
|
||||
@dataclass
|
||||
|
@ -103,4 +97,4 @@ class NextcloudMigrator(BaseMigrator):
|
|||
if status:
|
||||
nc_dir = nextcloud_dirs[slug]
|
||||
if nc_dir.image:
|
||||
import_image(nc_dir.image, recipe_id)
|
||||
self.import_image(slug, nc_dir.image, recipe_id)
|
||||
|
|
|
@ -7,13 +7,10 @@ import zipfile
|
|||
from gzip import GzipFile
|
||||
from pathlib import Path
|
||||
|
||||
from slugify import slugify
|
||||
|
||||
from mealie.schema.recipe import RecipeNote
|
||||
|
||||
from ._migration_base import BaseMigrator
|
||||
from .utils.migration_alias import MigrationAlias
|
||||
from .utils.migration_helpers import import_image
|
||||
|
||||
|
||||
def paprika_recipes(file: Path):
|
||||
|
@ -67,32 +64,26 @@ class PaprikaMigrator(BaseMigrator):
|
|||
]
|
||||
|
||||
def _migrate(self) -> None:
|
||||
recipe_image_urls = {}
|
||||
recipes = [r for r in paprika_recipes(self.archive) if "name" in r]
|
||||
recipe_models = [self.clean_recipe_dictionary(r) for r in recipes]
|
||||
results = self.import_recipes_to_database(recipe_models)
|
||||
|
||||
recipes = []
|
||||
for recipe in paprika_recipes(self.archive):
|
||||
if "name" not in recipe:
|
||||
for (slug, recipe_id, status), recipe in zip(results, recipes, strict=True):
|
||||
if not status:
|
||||
continue
|
||||
|
||||
recipe_model = self.clean_recipe_dictionary(recipe)
|
||||
|
||||
if "photo_data" in recipe:
|
||||
recipe_image_urls[slugify(recipe["name"])] = recipe["photo_data"]
|
||||
|
||||
recipes.append(recipe_model)
|
||||
|
||||
results = self.import_recipes_to_database(recipes)
|
||||
|
||||
for slug, recipe_id, status in results:
|
||||
if not status:
|
||||
image_data = recipe.get("photo_data")
|
||||
if image_data is None:
|
||||
self.logger.info(f"Recipe '{recipe['name']}' has no image")
|
||||
continue
|
||||
|
||||
try:
|
||||
# Images are stored as base64 encoded strings, so we need to decode them before importing.
|
||||
image = io.BytesIO(base64.b64decode(recipe_image_urls[slug]))
|
||||
image = io.BytesIO(base64.b64decode(image_data))
|
||||
with tempfile.NamedTemporaryFile(suffix=".jpeg") as temp_file:
|
||||
temp_file.write(image.read())
|
||||
temp_file.flush()
|
||||
path = Path(temp_file.name)
|
||||
import_image(path, recipe_id)
|
||||
self.import_image(slug, path, recipe_id)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to download image for {slug}: {e}")
|
||||
self.logger.error(f"Failed to import image for {slug}: {e}")
|
||||
|
|
|
@ -8,7 +8,7 @@ from mealie.services.scraper import cleaner
|
|||
|
||||
from ._migration_base import BaseMigrator
|
||||
from .utils.migration_alias import MigrationAlias
|
||||
from .utils.migration_helpers import import_image, parse_iso8601_duration
|
||||
from .utils.migration_helpers import parse_iso8601_duration
|
||||
|
||||
|
||||
def clean_instructions(instructions: list[str]) -> list[str]:
|
||||
|
@ -98,7 +98,7 @@ class RecipeKeeperMigrator(BaseMigrator):
|
|||
|
||||
recipes = [self.clean_recipe_dictionary(x) for x in recipes_as_dicts]
|
||||
results = self.import_recipes_to_database(recipes)
|
||||
for (_, recipe_id, status), recipe in zip(results, recipes, strict=False):
|
||||
for (slug, recipe_id, status), recipe in zip(results, recipes, strict=False):
|
||||
if status:
|
||||
try:
|
||||
if not recipe or not recipe.image:
|
||||
|
@ -107,4 +107,4 @@ class RecipeKeeperMigrator(BaseMigrator):
|
|||
except StopIteration:
|
||||
continue
|
||||
|
||||
import_image(recipe.image, recipe_id)
|
||||
self.import_image(slug, recipe.image, recipe_id)
|
||||
|
|
|
@ -10,7 +10,6 @@ from mealie.schema.reports.reports import ReportEntryCreate
|
|||
|
||||
from ._migration_base import BaseMigrator
|
||||
from .utils.migration_alias import MigrationAlias
|
||||
from .utils.migration_helpers import import_image
|
||||
|
||||
|
||||
def _build_ingredient_from_ingredient_data(ingredient_data: dict[str, Any], title: str | None = None) -> dict[str, Any]:
|
||||
|
@ -150,4 +149,4 @@ class TandoorMigrator(BaseMigrator):
|
|||
except StopIteration:
|
||||
continue
|
||||
|
||||
import_image(r.image, recipe_id)
|
||||
self.import_image(slug, r.image, recipe_id)
|
||||
|
|
|
@ -104,6 +104,7 @@ def import_image(src: str | Path, recipe_id: UUID4):
|
|||
"""Read the successful migrations attribute and for each import the image
|
||||
appropriately into the image directory. Minification is done in mass
|
||||
after the migration occurs.
|
||||
May raise an UnidentifiedImageError if the file is not a recognised format.
|
||||
"""
|
||||
|
||||
if isinstance(src, str):
|
||||
|
@ -113,11 +114,7 @@ def import_image(src: str | Path, recipe_id: UUID4):
|
|||
return
|
||||
|
||||
data_service = RecipeDataService(recipe_id=recipe_id)
|
||||
|
||||
try:
|
||||
data_service.write_image(src, src.suffix)
|
||||
except UnidentifiedImageError:
|
||||
return
|
||||
data_service.write_image(src, src.suffix)
|
||||
|
||||
|
||||
async def scrape_image(image_url: str, recipe_id: UUID4):
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue