feat: Update food seeding logic to use new format, now with removed CrowdIn limits? (#5514)
Some checks are pending
CodeQL / Analyze (push) Waiting to run
Docker Nightly Production / Backend Server Tests (push) Waiting to run
Docker Nightly Production / Frontend Tests (push) Waiting to run
Docker Nightly Production / Build Package (push) Waiting to run
Docker Nightly Production / Build Tagged Release (push) Blocked by required conditions
Docker Nightly Production / Notify Discord (push) Blocked by required conditions
Release Drafter / ✏️ Draft release (push) Waiting to run

Co-authored-by: Michael Genson <71845777+michael-genson@users.noreply.github.com>
Co-authored-by: Hayden <64056131+hay-kot@users.noreply.github.com>
This commit is contained in:
Cameronwyatt 2025-07-04 13:44:21 -04:00 committed by Hayden
parent abc37f258d
commit e794c6b525
No known key found for this signature in database
GPG key ID: 17CF79474E257545
47 changed files with 46784 additions and 29051 deletions

View file

@ -0,0 +1,75 @@
import glob
import json
import pathlib
def get_seed_locale_names() -> set[str]:
"""Find all locales in the seed/resources/ folder
Returns:
A set of every file name where there's both a seed label and seed food file
"""
LABELS_PATH = "/workspaces/mealie/mealie/repos/seed/resources/labels/locales/"
FOODS_PATH = "/workspaces/mealie/mealie/repos/seed/resources/foods/locales/"
label_locales = glob.glob("*.json", root_dir=LABELS_PATH)
foods_locales = glob.glob("*.json", root_dir=FOODS_PATH)
# ensure that a locale has both a label and a food seed file
return set(label_locales).intersection(foods_locales)
def get_labels_from_file(locale: str) -> list[str]:
"""Query a locale to get all of the labels so that they can be added to the new foods seed format
Returns:
All of the labels found within the seed file for a given locale
"""
locale_path = pathlib.Path("/workspaces/mealie/mealie/repos/seed/resources/labels/locales/" + locale)
label_names = [label["name"] for label in json.loads(locale_path.read_text(encoding="utf-8"))]
return label_names
def transform_foods(locale: str):
"""
Convert the current food seed file for a locale into a new format which maps each food to a label
Existing format of foods seed file is a dictionary where each key is a food name and the values are a dictionary
of attributes such as name and plural_name
New format maps each food to a label. The top-level dictionary has each key as a label e.g. "Fruits".
Each label key as a value that is a dictionary with an element called "foods"
"Foods" is a dictionary of each food for that label, with a key of the english food name e.g. "baking-soda"
and a value of attributes, including the translated name of the item e.g. "bicarbonate of soda" for en-GB.
"""
locale_path = pathlib.Path("/workspaces/mealie/mealie/repos/seed/resources/foods/locales/" + locale)
with open(locale_path, encoding="utf-8") as infile:
data = json.load(infile)
first_value = next(iter(data.values()))
if isinstance(first_value, dict) and "foods" in first_value:
# Locale is already in the new format, skipping transformation
return
transformed_data = {"": {"foods": dict(data.items())}}
# Seeding for labels now pulls from the foods file and parses the labels from there (as top-level keys),
# thus we need to add all of the existing labels to the new food seed file and give them an empty foods dictionary
label_names = get_labels_from_file(locale)
for label in label_names:
transformed_data[label] = {"foods": {}}
with open(locale_path, "w", encoding="utf-8") as outfile:
json.dump(transformed_data, outfile, indent=4, ensure_ascii=False)
def main():
for locale in get_seed_locale_names():
transform_foods(locale)
if __name__ == "__main__":
main()

View file

@ -1052,8 +1052,8 @@
"foods": {
"merge-dialog-text": "Combining the selected foods will merge the source food and target food into a single food. The source food will be deleted and all of the references to the source food will be updated to point to the target food.",
"merge-food-example": "Merging {food1} into {food2}",
"seed-dialog-text": "Seed the database with foods based on your local language. This will create 200+ common foods that can be used to organize your database. Foods are translated via a community effort.",
"seed-dialog-warning": "You have already have some items in your database. This action will not reconcile duplicates, you will have to manage them manually.",
"seed-dialog-text": "Seed the database with foods based on your local language. This will create ~2700 common foods that can be used to organize your database. Foods are translated via a community effort.",
"seed-dialog-warning": "You already have some items in your database. A new item will not be added if an item with the same name already exists.",
"combine-food": "Combine Food",
"source-food": "Source Food",
"target-food": "Target Food",

File diff suppressed because it is too large Load diff

View file

@ -1,692 +1,62 @@
{
"acorn-squash": {
"name": "acorn squash"
},
"alfalfa-sprouts": {
"name": "alfalfa sprouts"
},
"anchovies": {
"name": "anchovies"
},
"apples": {
"name": "apple",
"plural_name": "apples"
},
"artichoke": {
"name": "artichoke"
},
"arugula": {
"name": "arugula"
},
"asparagus": {
"name": "asparagus"
},
"avocado": {
"name": "avocado",
"plural_name": "avocado"
},
"bacon": {
"name": "bacon"
},
"baking-powder": {
"name": "baking powder"
},
"baking-soda": {
"name": "baking soda"
},
"baking-sugar": {
"name": "baking sugar"
},
"bar-sugar": {
"name": "bar sugar"
},
"basil": {
"name": "basil"
},
"beans": {
"name": "beans"
},
"bell-peppers": {
"name": "bell peppers",
"plural_name": "bell peppers"
},
"blackberries": {
"name": "blackberries"
},
"bok-choy": {
"name": "bok choy"
},
"brassicas": {
"name": "brassicas"
},
"bread": {
"name": "bread"
},
"breadfruit": {
"name": "breadfruit"
},
"broccoflower": {
"name": "broccoflower"
},
"broccoli": {
"name": "broccoli"
},
"broccoli-rabe": {
"name": "broccoli rabe"
},
"broccolini": {
"name": "broccolini"
},
"brown-sugar": {
"name": "brown sugar"
},
"brussels-sprouts": {
"name": "brussels sprouts"
},
"butter": {
"name": "butter"
},
"butternut-pumpkin": {
"name": "butternut pumpkin"
},
"butternut-squash": {
"name": "butternut squash"
},
"cabbage": {
"name": "cabbage",
"plural_name": "cabbages"
},
"cactus-edible": {
"name": "cactus, edible"
},
"calabrese": {
"name": "calabrese"
},
"cane-sugar": {
"name": "cane sugar"
},
"cannabis": {
"name": "cannabis"
},
"capsicum": {
"name": "capsicum"
},
"caraway": {
"name": "caraway"
},
"carrot": {
"name": "carrot",
"plural_name": "carrots"
},
"caster-sugar": {
"name": "caster sugar"
},
"castor-sugar": {
"name": "castor sugar"
},
"catfish": {
"name": "catfish"
},
"cauliflower": {
"name": "cauliflower",
"plural_name": "cauliflowers"
},
"cayenne-pepper": {
"name": "cayenne pepper"
},
"celeriac": {
"name": "celery root"
},
"celery": {
"name": "celery"
},
"cereal-grains": {
"name": "cereal grains"
},
"chard": {
"name": "chard"
},
"cheese": {
"name": "cheese"
},
"chicory": {
"name": "chicory"
},
"chilli-peppers": {
"name": "chilli pepper",
"plural_name": "chilli peppers"
},
"chinese-leaves": {
"name": "chinese leaves"
},
"chives": {
"name": "chives"
},
"chocolate": {
"name": "chocolate"
},
"cilantro": {
"name": "cilantro"
},
"cinnamon": {
"name": "cinnamon"
},
"clarified-butter": {
"name": "clarified butter"
},
"coconut": {
"name": "coconut",
"plural_name": "coconuts"
},
"coconut-milk": {
"name": "coconut milk"
},
"cod": {
"name": "cod"
},
"coffee": {
"name": "coffee"
},
"collard-greens": {
"name": "collard greens"
},
"confectioners-sugar": {
"name": "confectioners' sugar"
},
"coriander": {
"name": "coriander"
},
"corn": {
"name": "corn",
"plural_name": "corns"
},
"corn-syrup": {
"name": "corn syrup"
},
"cottonseed-oil": {
"name": "cottonseed oil"
},
"courgette": {
"name": "courgette"
},
"cream-of-tartar": {
"name": "cream of tartar"
},
"cucumber": {
"name": "cucumber",
"plural_name": "cucumbers"
},
"cumin": {
"name": "cumin"
},
"daikon": {
"name": "daikon",
"plural_name": "daikons"
},
"dairy-products-and-dairy-substitutes": {
"name": "dairy products and dairy substitutes"
},
"dandelion": {
"name": "dandelion"
},
"demerara-sugar": {
"name": "demerara sugar"
},
"dough": {
"name": "dough"
},
"edible-cactus": {
"name": "edible cactus"
},
"eggplant": {
"name": "eggplant",
"plural_name": "eggplants"
},
"eggs": {
"name": "egg",
"plural_name": "eggs"
},
"endive": {
"name": "endive",
"plural_name": "endives"
},
"fats": {
"name": "fats"
},
"fava-beans": {
"name": "fava beans"
},
"fiddlehead": {
"name": "fiddlehead"
},
"fiddlehead-fern": {
"name": "fiddlehead fern",
"plural_name": "fiddlehead ferns"
},
"fish": {
"name": "fish"
},
"five-spice-powder": {
"name": "five spice powder"
},
"flour": {
"name": "flour"
},
"frisee": {
"name": "frisee"
},
"fructose": {
"name": "fructose"
},
"fruit": {
"name": "fruit"
},
"fruit-sugar": {
"name": "fruit sugar"
},
"ful": {
"name": "ful"
},
"garam-masala": {
"name": "garam masala"
},
"garlic": {
"name": "garlic",
"plural_name": "garlics"
},
"gem-squash": {
"name": "gem squash"
},
"ghee": {
"name": "ghee"
},
"giblets": {
"name": "giblets"
},
"ginger": {
"name": "ginger"
},
"grains": {
"name": "grains"
},
"granulated-sugar": {
"name": "granulated sugar"
},
"grape-seed-oil": {
"name": "grape seed oil"
},
"green-onion": {
"name": "green onion",
"plural_name": "green onions"
},
"heart-of-palm": {
"name": "heart of palm",
"plural_name": "heart of palms"
},
"hemp": {
"name": "قنب والعياذ بالله"
},
"herbs": {
"name": "herbs"
},
"honey": {
"name": "honey"
},
"isomalt": {
"name": "isomalt"
},
"jackfruit": {
"name": "jackfruit",
"plural_name": "jackfruits"
},
"jaggery": {
"name": "jaggery"
},
"jams": {
"name": "jams"
},
"jellies": {
"name": "jellies"
},
"jerusalem-artichoke": {
"name": "jerusalem artichoke"
},
"jicama": {
"name": "jicama"
},
"kale": {
"name": "kale"
},
"kohlrabi": {
"name": "kohlrabi"
},
"kumara": {
"name": "kumara"
},
"leavening-agents": {
"name": "leavening agents"
},
"leek": {
"name": "leek",
"plural_name": "leeks"
},
"legumes": {
"name": "legumes"
},
"lemongrass": {
"name": "lemongrass"
},
"lentils": {
"name": "lentils"
},
"lettuce": {
"name": "lettuce"
},
"liver": {
"name": "liver",
"plural_name": "livers"
},
"maize": {
"name": "maize"
},
"maple-syrup": {
"name": "maple syrup"
},
"meat": {
"name": "meat"
},
"milk": {
"name": "milk"
},
"mortadella": {
"name": "mortadella"
},
"mushroom": {
"name": "mushroom",
"plural_name": "mushrooms"
},
"mussels": {
"name": "mussels"
},
"nanaimo-bar-mix": {
"name": "nanaimo bar mix"
},
"nori": {
"name": "nori"
},
"nutmeg": {
"name": "nutmeg"
},
"nutritional-yeast-flakes": {
"name": "nutritional yeast flakes"
},
"nuts": {
"name": "nuts"
},
"octopuses": {
"name": "octopus",
"plural_name": "octopuses"
},
"oils": {
"name": "oils"
},
"okra": {
"name": "okra"
},
"olive": {
"name": "olive"
},
"olive-oil": {
"name": "olive oil"
},
"onion": {
"name": "onion"
},
"onion-family": {
"name": "onion family"
},
"orange-blossom-water": {
"name": "orange blossom water"
},
"oranges": {
"name": "orange",
"plural_name": "oranges"
},
"oregano": {
"name": "oregano"
},
"oysters": {
"name": "oysters"
},
"panch-puran": {
"name": "panch puran"
},
"paprika": {
"name": "paprika"
},
"parsley": {
"name": "parsley"
},
"parsnip": {
"name": "parsnip",
"plural_name": "parsnips"
},
"pear": {
"name": "pear",
"plural_name": "pears"
},
"peas": {
"name": "peas"
},
"pepper": {
"name": "pepper",
"plural_name": "peppers"
},
"pineapple": {
"name": "pineapple",
"plural_name": "pineapples"
},
"plantain": {
"name": "plantain",
"plural_name": "plantains"
},
"poppy-seeds": {
"name": "poppy seeds"
},
"potato": {
"name": "potato",
"plural_name": "potatoes"
},
"poultry": {
"name": "poultry"
},
"powdered-sugar": {
"name": "powdered sugar"
},
"pumpkin": {
"name": "pumpkin",
"plural_name": "pumpkins"
},
"pumpkin-seeds": {
"name": "pumpkin seeds"
},
"radish": {
"name": "radish",
"plural_name": "radishes"
},
"raw-sugar": {
"name": "raw sugar"
},
"refined-sugar": {
"name": "refined sugar"
},
"rice": {
"name": "rice"
},
"rice-flour": {
"name": "rice flour"
},
"rock-sugar": {
"name": "rock sugar"
},
"rum": {
"name": "rum"
},
"salmon": {
"name": "salmon"
},
"salt": {
"name": "salt"
},
"salt-cod": {
"name": "salt cod"
},
"scallion": {
"name": "scallion",
"plural_name": "scallions"
},
"seafood": {
"name": "seafood"
},
"seeds": {
"name": "seeds"
},
"sesame-seeds": {
"name": "sesame seeds"
},
"shallot": {
"name": "shallot",
"plural_name": "shallots"
},
"skate": {
"name": "skate"
},
"soda": {
"name": "soda"
},
"soda-baking": {
"name": "soda, baking"
},
"soybean": {
"name": "soybean"
},
"spaghetti-squash": {
"name": "spaghetti squash",
"plural_name": "spaghetti squashes"
},
"speck": {
"name": "speck"
},
"spices": {
"name": "spices"
},
"spinach": {
"name": "spinach"
},
"spring-onion": {
"name": "spring onion",
"plural_name": "spring onions"
},
"squash": {
"name": "squash",
"plural_name": "squashes"
},
"squash-family": {
"name": "squash family"
},
"stockfish": {
"name": "stockfish"
},
"sugar": {
"name": "sugar"
},
"sunchoke": {
"name": "sunchoke",
"plural_name": "sunchokes"
},
"sunflower-seeds": {
"name": "sunflower seeds"
},
"superfine-sugar": {
"name": "superfine sugar"
},
"sweet-potato": {
"name": "sweet potato",
"plural_name": "sweet potatoes"
},
"sweetcorn": {
"name": "sweetcorn",
"plural_name": "sweetcorns"
},
"sweeteners": {
"name": "sweeteners"
},
"tahini": {
"name": "tahini"
},
"taro": {
"name": "taro",
"plural_name": "taroes"
},
"teff": {
"name": "teff"
},
"tomato": {
"name": "tomato",
"plural_name": "tomatoes"
},
"trout": {
"name": "trout"
},
"tubers": {
"name": "tuber",
"plural_name": "tubers"
},
"tuna": {
"name": "tuna"
},
"turbanado-sugar": {
"name": "turbanado sugar"
},
"turnip": {
"name": "turnip",
"plural_name": "turnips"
},
"unrefined-sugar": {
"name": "unrefined sugar"
},
"vanilla": {
"name": "vanilla"
},
"vegetables": {
"name": "vegetables"
},
"watercress": {
"name": "watercress"
},
"watermelon": {
"name": "watermelon",
"plural_name": "watermelons"
},
"white-mushroom": {
"name": "white mushroom",
"plural_name": "white mushrooms"
},
"white-sugar": {
"name": "white sugar"
},
"xanthan-gum": {
"name": "xanthan gum"
},
"yam": {
"name": "yam",
"plural_name": "yams"
},
"yeast": {
"name": "yeast"
},
"zucchini": {
"name": "zucchini",
"plural_name": "zucchinis"
}
}
"": {
"foods": {}
},
"خَضْراوات وفواكه": {
"foods": {}
},
"الحبوب": {
"foods": {}
},
"آلفواكه": {
"foods": {}
},
"الخضراوات": {
"foods": {}
},
"اللحوم": {
"foods": {}
},
"المأكولات البحرية": {
"foods": {}
},
"المشروبات": {
"foods": {}
},
"المخبوزات": {
"foods": {}
},
"المعلبات": {
"foods": {}
},
"الباهرات": {
"foods": {}
},
"الحَلْوَيَات": {
"foods": {}
},
"منتجات الألبان": {
"foods": {}
},
"الأطعمة المجمدة": {
"foods": {}
},
"الأغذية الصحية": {
"foods": {}
},
"المنزل": {
"foods": {}
},
"منتجات اللحوم": {
"foods": {}
},
"الوجبات الخفيفة": {
"foods": {}
},
"التوابل": {
"foods": {}
},
"أخرى": {
"foods": {}
}
}

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -3,12 +3,17 @@ import pathlib
from collections.abc import Generator
from functools import cached_property
from mealie.schema.labels import MultiPurposeLabelSave
from mealie.schema.recipe.recipe_ingredient import SaveIngredientFood, SaveIngredientUnit
from mealie.schema.labels import MultiPurposeLabelOut, MultiPurposeLabelSave
from mealie.schema.recipe.recipe_ingredient import (
IngredientFood,
IngredientUnit,
SaveIngredientFood,
SaveIngredientUnit,
)
from mealie.services.group_services.labels_service import MultiPurposeLabelService
from ._abstract_seeder import AbstractSeeder
from .resources import foods, labels, units
from .resources import foods, units
class MultiPurposeLabelSeeder(AbstractSeeder):
@ -17,20 +22,24 @@ class MultiPurposeLabelSeeder(AbstractSeeder):
return MultiPurposeLabelService(self.repos)
def get_file(self, locale: str | None = None) -> pathlib.Path:
locale_path = self.resources / "labels" / "locales" / f"{locale}.json"
return locale_path if locale_path.exists() else labels.en_US
# Get the labels from the foods seed file now
locale_path = self.resources / "foods" / "locales" / f"{locale}.json"
return locale_path if locale_path.exists() else foods.en_US
def get_all_labels(self) -> list[MultiPurposeLabelOut]:
return self.repos.group_multi_purpose_labels.get_all()
def load_data(self, locale: str | None = None) -> Generator[MultiPurposeLabelSave, None, None]:
file = self.get_file(locale)
seen_label_names = set()
for label in json.loads(file.read_text(encoding="utf-8")):
if label["name"] in seen_label_names:
continue
seen_label_names.add(label["name"])
current_label_names = {label.name for label in self.get_all_labels()}
# load from the foods locale file and remove any empty strings
seed_label_names = set(filter(None, json.loads(file.read_text(encoding="utf-8")).keys())) # type: set[str]
# only seed new labels
to_seed_labels = seed_label_names - current_label_names
for label in to_seed_labels:
yield MultiPurposeLabelSave(
name=label["name"],
name=label,
group_id=self.repos.group_id,
)
@ -48,10 +57,13 @@ class IngredientUnitsSeeder(AbstractSeeder):
locale_path = self.resources / "units" / "locales" / f"{locale}.json"
return locale_path if locale_path.exists() else units.en_US
def get_all_units(self) -> list[IngredientUnit]:
return self.repos.ingredient_units.get_all()
def load_data(self, locale: str | None = None) -> Generator[SaveIngredientUnit, None, None]:
file = self.get_file(locale)
seen_unit_names = set()
seen_unit_names = {unit.name for unit in self.get_all_units()}
for unit in json.loads(file.read_text(encoding="utf-8")).values():
if unit["name"] in seen_unit_names:
continue
@ -80,21 +92,32 @@ class IngredientFoodsSeeder(AbstractSeeder):
locale_path = self.resources / "foods" / "locales" / f"{locale}.json"
return locale_path if locale_path.exists() else foods.en_US
def get_label(self, value: str) -> MultiPurposeLabelOut | None:
return self.repos.group_multi_purpose_labels.get_one(value, "name")
def get_all_foods(self) -> list[IngredientFood]:
return self.repos.ingredient_foods.get_all()
def load_data(self, locale: str | None = None) -> Generator[SaveIngredientFood, None, None]:
file = self.get_file(locale)
seed_foods_names = set()
for food in json.loads(file.read_text(encoding="utf-8")).values():
if food["name"] in seed_foods_names:
continue
# get all current unique foods
seen_foods_names = {food.name for food in self.get_all_foods()}
for label, values in json.loads(file.read_text(encoding="utf-8")).items():
label_out = self.get_label(label)
seed_foods_names.add(food["name"])
yield SaveIngredientFood(
group_id=self.repos.group_id,
name=food["name"],
plural_name=food.get("plural_name"),
description="",
)
for food_name, attributes in values["foods"].items():
if food_name in seen_foods_names:
continue
seen_foods_names.add(food_name)
yield SaveIngredientFood(
group_id=self.repos.group_id,
name=attributes["name"],
plural_name=attributes.get("plural_name"),
description="", # description expected to be empty string by UnitFoodBase class
label_id=label_out.id if label_out and label_out.id else None,
)
def seed(self, locale: str | None = None) -> None:
self.logger.info("Seeding Ingredient Foods")

View file

@ -12,7 +12,7 @@ def test_seed_invalid_locale(api_client: TestClient, unique_user: TestUser):
def test_seed_foods(api_client: TestClient, unique_user: TestUser):
CREATED_FOODS = 214
CREATED_FOODS = 2687
database = unique_user.repos
# Check that the foods was created
@ -44,7 +44,7 @@ def test_seed_units(api_client: TestClient, unique_user: TestUser):
def test_seed_labels(api_client: TestClient, unique_user: TestUser):
CREATED_LABELS = 21
CREATED_LABELS = 32
database = unique_user.repos
# Check that the foods was created

View file

@ -99,7 +99,7 @@ def test_new_label_creates_list_labels_in_all_households(
def test_seed_label_creates_list_labels(api_client: TestClient, unique_user: TestUser):
CREATED_LABELS = 21
CREATED_LABELS = 32
database = unique_user.repos
# create a list with some labels