From feb4540180c27ecd58e3e1186078153778b9a930 Mon Sep 17 00:00:00 2001 From: Roelof Rietbroek Date: Mon, 1 Jan 2024 21:04:53 +0000 Subject: [PATCH 01/34] Add option to download ingredients as JSONL from Open Food Facts (requires no mongo) --- .../commands/import-off-products.py | 73 ++++++++++++++++--- wger/utils/models.py | 2 +- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.py index 6d82b8a1c..817561562 100644 --- a/wger/nutrition/management/commands/import-off-products.py +++ b/wger/nutrition/management/commands/import-off-products.py @@ -15,6 +15,7 @@ # Standard Library import enum import logging +import os from collections import Counter # Django @@ -58,7 +59,7 @@ def add_arguments(self, parser): type=str, help='Script mode, "insert" or "update". Insert will insert the ingredients as new ' 'entries in the database, while update will try to update them if they are ' - 'already present. Deault: insert' + 'already present. Default: insert' ) parser.add_argument( '--completeness', @@ -70,8 +71,17 @@ def add_arguments(self, parser): 'completeness score that ranges from 0 to 1.1. Default: 0.7' ) - def handle(self, **options): - + parser.add_argument( + '--jsonl', + action='store_true', + default=False, + dest='usejsonl', + help='Use the JSONL dump of the Open Food Facts database.' + '(this option does not require mongo)' + ) + + def products_mongo(self,filterdict): + """returns a mongo iterator with filtered prodcuts""" try: # Third Party from pymongo import MongoClient @@ -79,6 +89,45 @@ def handle(self, **options): self.stdout.write('Please install pymongo, `pip install pymongo`') return + client = MongoClient('mongodb://off:off-wger@127.0.0.1', port=27017) + db = client.admin + return db.products.find(filterdict) + + def products_jsonl(self,languages,completeness): + import json + import requests + from gzip import GzipFile + off_url='https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz' + download_dir=os.path.expanduser('~/.cache/off_cache') + os.makedirs(download_dir,exist_ok=True) + gzipdb=os.path.join(download_dir,os.path.basename(off_url)) + if os.path.exists(gzipdb): + self.stdout.write(f'Already downloaded {gzipdb}, skipping download') + else: + self.stdout.write(f'downloading {gzipdb}... (this may take a while)') + req=requests.get(off_url,stream=True) + with open(gzipdb,'wb') as fid: + for chunk in req.iter_content(chunk_size=50*1024): + fid.write(chunk) + with GzipFile(gzipdb,'rb') as gzid: + for line in gzid: + try: + product=json.loads(line) + if product['completeness'] < completeness: + continue + if not product['lang'] in languages: + continue + yield product + except: + self.stdout.write(f' Error parsing and/or filtering json record, skipping') + continue + + + + + def handle(self, **options): + + if options['mode'] == 'insert': self.mode = Mode.INSERT @@ -92,25 +141,25 @@ def handle(self, **options): self.stdout.write(f' - {self.mode}') self.stdout.write('') - client = MongoClient('mongodb://off:off-wger@127.0.0.1', port=27017) - db = client.admin languages = {l.short_name: l.pk for l in Language.objects.all()} bulk_update_bucket = [] counter = Counter() - - for product in db.products.find( - { + if options['usejsonl']: + products=self.products_jsonl(languages=list(languages.keys()),completeness=self.completeness) + else: + filterdict={ 'lang': { "$in": list(languages.keys()) }, 'completeness': { "$gt": self.completeness } - } - ): + } + products=self.products_mongo(filterdict) + for product in products: try: ingredient_data = extract_info_from_off(product, languages[product['lang']]) except KeyError as e: @@ -149,7 +198,7 @@ def handle(self, **options): self.stdout.write( '--> Error while saving the product bucket. Saving individually' ) - self.stdout.write(e) + self.stdout.write(str(e)) # Try saving the ingredients individually as most will be correct for ingredient in bulk_update_bucket: @@ -159,7 +208,7 @@ def handle(self, **options): # ¯\_(ツ)_/¯ except Exception as e: self.stdout.write('--> Error while saving the product individually') - self.stdout.write(e) + self.stdout.write(str(e)) counter['new'] += self.bulk_size bulk_update_bucket = [] diff --git a/wger/utils/models.py b/wger/utils/models.py index 1c1bcf83c..72dcce863 100644 --- a/wger/utils/models.py +++ b/wger/utils/models.py @@ -59,7 +59,7 @@ class Meta: blank=True, ) - license_author = models.CharField( + license_author = models.TextField( verbose_name=_('Author(s)'), max_length=600, blank=True, From fff7c86a8f80ba351199224a45204c72a5cdbbdb Mon Sep 17 00:00:00 2001 From: Roelof Rietbroek Date: Mon, 1 Jan 2024 21:27:54 +0000 Subject: [PATCH 02/34] update authors --- AUTHORS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 75326e5ec..8ba094a66 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -75,7 +75,7 @@ Developers * Gabriel Liss - https://github.com/gabeliss * Alexandra Rhodes - https://github.com/arhodes130 * Jayanth Bontha - https://github.com/JayanthBontha - +* Roelof Rietbroek https://github.com/strawpants Translators ----------- From 193e6cad27fa54b255a0d1caab6282e077f5ebc1 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Sun, 21 Apr 2024 17:37:58 +0200 Subject: [PATCH 03/34] Start working on import script for USDA products --- .../commands/import-usda-products.py | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 wger/nutrition/management/commands/import-usda-products.py diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py new file mode 100644 index 000000000..d59b11ea0 --- /dev/null +++ b/wger/nutrition/management/commands/import-usda-products.py @@ -0,0 +1,151 @@ +# This file is part of wger Workout Manager. +# +# wger Workout Manager is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# wger Workout Manager is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License + +# Standard Library +import enum +import json +import logging +import os +import tempfile +from json import JSONDecodeError +from zipfile import ZipFile + +import requests + +# Django +from django.core.management.base import BaseCommand + +logger = logging.getLogger(__name__) + + +# Mode for this script. When using 'insert', the script will bulk-insert the new +# ingredients, which is very efficient. Importing the whole database will require +# barely a minute. When using 'update', existing ingredients will be updated, which +# requires two queries per product and is needed when there are already existing +# entries in the local ingredient table. +class Mode(enum.Enum): + INSERT = enum.auto() + UPDATE = enum.auto() + + +class Command(BaseCommand): + """ + Import an Open Food facts Dump + """ + + mode = Mode.UPDATE + + def add_arguments(self, parser): + parser.add_argument( + '--set-mode', + action='store', + default=10, + dest='mode', + type=str, + help='Script mode, "insert" or "update". Insert will insert the ingredients as new ' + 'entries in the database, while update will try to update them if they are ' + 'already present. Deault: insert', + ) + + def handle(self, **options): + if options['mode'] == 'insert': + self.mode = Mode.INSERT + + self.stdout.write('Importing entries from USDA') + self.stdout.write(f' - {self.mode}') + self.stdout.write('') + + usda_url = 'https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_foundation_food_json_2024-04-18.zip' + + with tempfile.TemporaryDirectory() as folder: + folder = '/Users/roland/Entwicklung/wger/server/extras/usda' + + print(f'{folder=}') + zip_file = os.path.join(folder, 'usda.zip') + if os.path.exists(zip_file): + self.stdout.write(f'Already downloaded {zip_file}, skipping download') + else: + self.stdout.write(f'downloading {zip_file}... (this may take a while)') + req = requests.get(usda_url, stream=True) + with open(zip_file, 'wb') as fid: + for chunk in req.iter_content(chunk_size=50 * 1024): + fid.write(chunk) + + self.stdout.write('download successful') + + with ZipFile(zip_file, 'r') as zip_ref: + file_list = zip_ref.namelist() + if not file_list: + raise Exception("No files found in the ZIP archive") + + first_file = file_list[0] + self.stdout.write(f'Extracting {first_file=}') + extracted_file_path = zip_ref.extract(first_file, path=folder) + + with open(extracted_file_path, "r") as extracted_file: + for line in extracted_file: + self.process_product(line.strip().strip(',')) + + def process_product(self, json_data): + try: + data = json.loads(json_data) + except JSONDecodeError as e: + # print(e) + # print(json_data) + # print('---------------') + return + + name = data['description'] + fdc_id = data['fdcId'] + + if not data.get('foodNutrients'): + return + + proteins = None + carbs = None + fats = None + energy = None + for d in data['foodNutrients']: + + if not d.get("nutrient"): + return + + nutrient = d.get("nutrient") + nutrient_id = nutrient.get("id") + + match nutrient_id: + case 1003: + proteins = float(d.get("amount")) + + case 1004: + carbs = float(d.get("amount")) + + case 1005: + fats = float(d.get("amount")) + + case 2048: + energy = float(d.get("amount")) + + if not all([proteins, carbs, fats, energy]): + return + + self.stdout.write(f' - {fdc_id}') + self.stdout.write(f' - {name}') + self.stdout.write(f' - {proteins=}') + self.stdout.write(f' - {carbs=}') + self.stdout.write(f' - {fats=}') + self.stdout.write(f' - {energy=}') + + self.stdout.write('') + return From 42d2addf93ab86b884b2e391dd58836b8d02098e Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Sat, 18 May 2024 16:21:48 +0200 Subject: [PATCH 04/34] Fix path for OFF mongo dump --- extras/open-food-facts/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extras/open-food-facts/docker-compose.yml b/extras/open-food-facts/docker-compose.yml index 555d08a16..e77e8ba83 100644 --- a/extras/open-food-facts/docker-compose.yml +++ b/extras/open-food-facts/docker-compose.yml @@ -4,7 +4,7 @@ services: ports: - "27017:27017" volumes: - - $PWD/dump:/dump + - $PWD/dump/dump:/dump environment: MONGO_INITDB_ROOT_USERNAME: off MONGO_INITDB_ROOT_PASSWORD: off-wger From ebd9534382ba7ed79899c279eb6cf1db1f19c870 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Sat, 18 May 2024 16:25:16 +0200 Subject: [PATCH 05/34] Add remote_id to ingredients. This allows to properly update them during imports. Currently, we were using the code field for OFF products but e.g. USDA products have their own id --- wger/nutrition/api/serializers.py | 1 + wger/nutrition/dataclasses.py | 44 ++++++ wger/nutrition/fixtures/test-ingredients.json | 14 ++ .../commands/import-off-products.py | 129 ++---------------- .../nutrition/management/commands/products.py | 117 ++++++++++++++++ .../migrations/0021_add_remote_id.py | 37 +++++ wger/nutrition/models/ingredient.py | 18 +-- wger/nutrition/off.py | 41 +----- wger/nutrition/tests/test_ingredient.py | 12 +- wger/nutrition/tests/test_off.py | 7 +- wger/utils/constants.py | 5 - 11 files changed, 243 insertions(+), 182 deletions(-) create mode 100644 wger/nutrition/dataclasses.py create mode 100644 wger/nutrition/management/commands/products.py create mode 100644 wger/nutrition/migrations/0021_add_remote_id.py diff --git a/wger/nutrition/api/serializers.py b/wger/nutrition/api/serializers.py index ad5e0a551..98e5755bb 100644 --- a/wger/nutrition/api/serializers.py +++ b/wger/nutrition/api/serializers.py @@ -114,6 +114,7 @@ class Meta: fields = [ 'id', 'uuid', + 'remote_id', 'code', 'name', 'created', diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.py new file mode 100644 index 000000000..b5c443c4e --- /dev/null +++ b/wger/nutrition/dataclasses.py @@ -0,0 +1,44 @@ +# This file is part of wger Workout Manager. +# +# wger Workout Manager is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# wger Workout Manager is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License + +from dataclasses import dataclass, asdict +from typing import Optional + + +@dataclass +class IngredientData: + name: str + remote_id: str + language_id: int + energy: float + protein: float + carbohydrates: float + carbohydrates_sugar: float + fat: float + fat_saturated: float + fibres: Optional[float] + sodium: Optional[float] + code: str + source_name: str + source_url: str + common_name: str + brand: str + status: str + license_id: int + license_author: str + license_title: str + license_object_url: str + + def dict(self): + return asdict(self) diff --git a/wger/nutrition/fixtures/test-ingredients.json b/wger/nutrition/fixtures/test-ingredients.json index d4152ebef..da61a7814 100644 --- a/wger/nutrition/fixtures/test-ingredients.json +++ b/wger/nutrition/fixtures/test-ingredients.json @@ -4,6 +4,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "7908c204-907f-4b1e-ad4e-f482e9769ade", + "remote_id": "1234567890987654321", "status": "2", "license_author": "wger test", "last_update": "2013-02-12T01:00:00+01:00", @@ -31,6 +32,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "44dc5966-73a2-4df7-8b15-f6d37a8990d9", + "remote_id": "1234567890", "status": "2", "license_author": "wger test", "name": "Ingredient, test, 2, organic, raw", @@ -58,6 +60,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "f28c8a68-ecda-42db-a9c4-781ee2aee472", + "remote_id": "1223334444", "status": "2", "license_author": "wger test", "last_update": "2013-03-22T18:32:15+01:00", @@ -85,6 +88,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "fdd662aa-696c-46c6-acae-b192ccc25f11", + "remote_id": "0987654321", "status": "2", "license_author": "wger test", "last_update": "2013-02-12T12:17:45+01:00", @@ -112,6 +116,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "293957df-b5bb-4ca6-b883-2234185c5372", + "remote_id": "555555555", "status": "2", "license_author": "wger test", "last_update": "2013-02-12T14:14:00+01:00", @@ -139,6 +144,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "dfc5c622-027b-4f17-8141-dadd1ce7e3f1", + "remote_id": "1112222333333334444444", "status": "2", "license_author": "wger test", "last_update": "2013-02-12T01:00:00+01:00", @@ -166,6 +172,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "42a41805-2caf-4c7e-9ff3-decbfee5f695", + "remote_id": "11222222333", "status": "1", "license_author": "test", "last_update": "2013-02-12T01:00:00+01:00", @@ -193,6 +200,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "db9e0502-2a54-472d-97e2-3761d1b286ee", + "remote_id": "9988877766655", "status": "2", "license_author": "test", "last_update": "2013-02-12T01:00:00+01:00", @@ -220,6 +228,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "7164629a-9f58-450b-8316-ad7954564171", + "remote_id": "123454321", "status": "2", "license_author": "test", "last_update": "2013-02-12T01:00:00+01:00", @@ -247,6 +256,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "9d53cf0e-5100-480f-a320-2de51c70603b", + "remote_id": "0009999888777", "status": "2", "license_author": "test", "last_update": "2013-02-12T01:00:00+01:00", @@ -274,6 +284,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "0187406a-7290-4199-a4dc-ccceb8cb55ea", + "remote_id": "7787878787878787", "status": "2", "license_author": "test", "last_update": "2013-02-12T01:00:00+01:00", @@ -301,6 +312,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "16ade127-00b0-4bcf-906e-dd9730f5ade1", + "remote_id": "29292929292929", "status": "2", "license_author": "test", "last_update": "2013-02-12T01:00:00+01:00", @@ -328,6 +340,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "59a73b2c-ead1-4001-b6a0-355c9cb366a7", + "remote_id": "19191919191", "status": "2", "license_author": "test", "last_update": "2013-02-12T01:00:00+01:00", @@ -355,6 +368,7 @@ "model": "nutrition.ingredient", "fields": { "uuid": "a18665d0-24e3-4255-88aa-92f2831d568a", + "remote_id": "4444444444444", "status": "2", "license_author": "test", "last_update": "2013-02-12T01:00:00+01:00", diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.py index 06397f26d..17857211f 100644 --- a/wger/nutrition/management/commands/import-off-products.py +++ b/wger/nutrition/management/commands/import-off-products.py @@ -13,64 +13,28 @@ # You should have received a copy of the GNU Affero General Public License # Standard Library -import enum import logging from collections import Counter # Django -from django.core.management.base import BaseCommand +from wger.nutrition.management.commands.products import ImportProductCommand, Mode # wger from wger.core.models import Language -from wger.nutrition.models import Ingredient from wger.nutrition.off import extract_info_from_off - logger = logging.getLogger(__name__) -# Mode for this script. When using 'insert', the script will bulk-insert the new -# ingredients, which is very efficient. Importing the whole database will require -# barely a minute. When using 'update', existing ingredients will be updated, which -# requires two queries per product and is needed when there are already existing -# entries in the local ingredient table. -class Mode(enum.Enum): - INSERT = enum.auto() - UPDATE = enum.auto() - - -class Command(BaseCommand): +class Command(ImportProductCommand): """ Import an Open Food facts Dump """ - mode = Mode.UPDATE - bulk_size = 500 completeness = 0.7 help = 'Import an Open Food Facts dump. Please consult extras/docker/open-food-facts' - def add_arguments(self, parser): - parser.add_argument( - '--set-mode', - action='store', - default=10, - dest='mode', - type=str, - help='Script mode, "insert" or "update". Insert will insert the ingredients as new ' - 'entries in the database, while update will try to update them if they are ' - 'already present. Deault: insert', - ) - parser.add_argument( - '--completeness', - action='store', - default=0.7, - dest='completeness', - type=float, - help='Completeness threshold for importing the products. Products in OFF have ' - 'completeness score that ranges from 0 to 1.1. Default: 0.7', - ) - def handle(self, **options): try: # Third Party @@ -79,16 +43,12 @@ def handle(self, **options): self.stdout.write('Please install pymongo, `pip install pymongo`') return + self.counter = Counter() + if options['mode'] == 'insert': self.mode = Mode.INSERT - if options['completeness'] < 0 or options['completeness'] > 1.1: - self.stdout.write('Completeness must be between 0 and 1.1') - return - self.completeness = options['completeness'] - self.stdout.write('Importing entries from Open Food Facts') - self.stdout.write(f' - Completeness threshold: {self.completeness}') self.stdout.write(f' - {self.mode}') self.stdout.write('') @@ -97,88 +57,17 @@ def handle(self, **options): languages = {l.short_name: l.pk for l in Language.objects.all()} - bulk_update_bucket = [] - counter = Counter() - - for product in db.products.find( - {'lang': {'$in': list(languages.keys())}, 'completeness': {'$gt': self.completeness}} - ): + for product in db.products.find({'lang': {'$in': list(languages.keys())}}): try: ingredient_data = extract_info_from_off(product, languages[product['lang']]) except KeyError as e: # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}') - # self.stdout.write( - # '***********************************************************************************************') - # self.stdout.write( - # '***********************************************************************************************') - # self.stdout.write( - # '***********************************************************************************************') + # self.stdout.write(repr(e)) # pprint(product) - # self.stdout.write(f'--> Product: {product}') - counter['skipped'] += 1 - continue - - # Some products have no name or name is too long, skipping - if not ingredient_data.name: - # self.stdout.write('--> Ingredient has no name field') - counter['skipped'] += 1 - continue - - if not ingredient_data.common_name: - # self.stdout.write('--> Ingredient has no common name field') - counter['skipped'] += 1 + self.counter['skipped'] += 1 continue - # - # Add entries as new products - if self.mode == Mode.INSERT: - bulk_update_bucket.append(Ingredient(**ingredient_data.dict())) - if len(bulk_update_bucket) > self.bulk_size: - try: - Ingredient.objects.bulk_create(bulk_update_bucket) - self.stdout.write('***** Bulk adding products *****') - except Exception as e: - self.stdout.write( - '--> Error while saving the product bucket. Saving individually' - ) - self.stdout.write(e) - - # Try saving the ingredients individually as most will be correct - for ingredient in bulk_update_bucket: - try: - ingredient.save() - - # ¯\_(ツ)_/¯ - except Exception as e: - self.stdout.write('--> Error while saving the product individually') - self.stdout.write(e) - - counter['new'] += self.bulk_size - bulk_update_bucket = [] - - # Update existing entries - else: - try: - # Update an existing product (look-up key is the code) or create a new - # one. While this might not be the most efficient query (there will always - # be a SELECT first), it's ok because this script is run very rarely. - obj, created = Ingredient.objects.update_or_create( - code=ingredient_data.code, - defaults=ingredient_data.dict(), - ) - - if created: - counter['new'] += 1 - # self.stdout.write('-> added to the database') - else: - counter['edited'] += 1 - # self.stdout.write('-> updated') - - except Exception as e: - self.stdout.write('--> Error while performing update_or_create') - self.stdout.write(str(e)) - counter['error'] += 1 - continue + self.handle_data(ingredient_data) self.stdout.write(self.style.SUCCESS('Finished!')) - self.stdout.write(self.style.SUCCESS(str(counter))) + self.stdout.write(self.style.SUCCESS(str(self.counter))) diff --git a/wger/nutrition/management/commands/products.py b/wger/nutrition/management/commands/products.py new file mode 100644 index 000000000..c4fdecb71 --- /dev/null +++ b/wger/nutrition/management/commands/products.py @@ -0,0 +1,117 @@ +# This file is part of wger Workout Manager. +# +# wger Workout Manager is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# wger Workout Manager is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License + +# Standard Library +import enum +import logging +from collections import Counter + +# Django +from django.core.management.base import BaseCommand + +# wger +from wger.nutrition.dataclasses import IngredientData +from wger.nutrition.models import Ingredient + +logger = logging.getLogger(__name__) + + +# Mode for this script. When using 'insert', the script will bulk-insert the new +# ingredients, which is very efficient. Importing the whole database will require +# barely a minute. When using 'update', existing ingredients will be updated, which +# requires two queries per product and is needed when there are already existing +# entries in the local ingredient table. +class Mode(enum.Enum): + INSERT = enum.auto() + UPDATE = enum.auto() + + +class ImportProductCommand(BaseCommand): + """ + Import an Open Food facts Dump + """ + + mode = Mode.UPDATE + bulk_update_bucket: list[Ingredient] = [] + bulk_size = 500 + counter: Counter + + help = "Don't run this command directly. Use either import-off-products or import-usda-products" + + def add_arguments(self, parser): + parser.add_argument( + '--set-mode', + action='store', + default='update', + dest='mode', + type=str, + help='Script mode, "insert" or "update". Insert will insert the ingredients as new ' + 'entries in the database, while update will try to update them if they are ' + 'already present. Deault: insert', + ) + + def handle(self, **options): + raise NotImplementedError('Do not run this command on its own!') + + def handle_data(self, ingredient_data: IngredientData): + + # + # Add entries as new products + if self.mode == Mode.INSERT: + self.bulk_update_bucket.append(Ingredient(**ingredient_data.dict())) + if len(self.bulk_update_bucket) > self.bulk_size: + try: + Ingredient.objects.bulk_create(self.bulk_update_bucket) + self.stdout.write('***** Bulk adding products *****') + except Exception as e: + self.stdout.write( + '--> Error while saving the product bucket. Saving individually' + ) + self.stdout.write(e) + + # Try saving the ingredients individually as most will be correct + for ingredient in self.bulk_update_bucket: + try: + ingredient.save() + + # ¯\_(ツ)_/¯ + except Exception as e: + self.stdout.write('--> Error while saving the product individually') + self.stdout.write(e) + + self.counter['new'] += self.bulk_size + self.bulk_update_bucket = [] + + # Update existing entries + else: + try: + # Update an existing product (look-up key is the code) or create a new + # one. While this might not be the most efficient query (there will always + # be a SELECT first), it's ok because this script is run very rarely. + obj, created = Ingredient.objects.update_or_create( + remote_id=ingredient_data.remote_id, + defaults=ingredient_data.dict(), + ) + + if created: + self.counter['new'] += 1 + # self.stdout.write('-> added to the database') + else: + self.counter['edited'] += 1 + # self.stdout.write('-> updated') + + except Exception as e: + self.stdout.write('--> Error while performing update_or_create') + self.stdout.write(repr(e)) + self.counter['error'] += 1 diff --git a/wger/nutrition/migrations/0021_add_remote_id.py b/wger/nutrition/migrations/0021_add_remote_id.py new file mode 100644 index 000000000..6feefbc22 --- /dev/null +++ b/wger/nutrition/migrations/0021_add_remote_id.py @@ -0,0 +1,37 @@ +# Generated by Django 4.2.6 on 2024-05-18 07:31 + +from django.db import migrations, models + +from wger.nutrition.models import Source + + +def set_external_id(apps, schema_editor): + """Set remote_id, used by imported ingredients (OFF, etc.)""" + + Ingredient = apps.get_model('nutrition', 'Ingredient') + for ingredient in Ingredient.objects.filter(source_name=Source.OPEN_FOOD_FACTS.value): + ingredient.remote_id = ingredient.code + ingredient.save() + + +class Migration(migrations.Migration): + dependencies = [ + ('nutrition', '0020_full_text_search'), + ] + + operations = [ + migrations.RenameIndex( + model_name='ingredient', + new_name='nutrition_i_name_8f538f_gin', + old_name='nutrition_i_search__f274b7_gin', + ), + migrations.AddField( + model_name='ingredient', + name='remote_id', + field=models.CharField(blank=True, db_index=True, max_length=200, null=True), + ), + migrations.RunPython( + set_external_id, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.py index 252bbf085..4cf48975a 100644 --- a/wger/nutrition/models/ingredient.py +++ b/wger/nutrition/models/ingredient.py @@ -24,7 +24,6 @@ from django.conf import settings from django.contrib.auth.models import User from django.contrib.postgres.indexes import GinIndex -from django.contrib.postgres.search import SearchVectorField from django.contrib.sites.models import Site from django.core import mail from django.core.cache import cache @@ -53,10 +52,7 @@ ) from wger.nutrition.models.sources import Source from wger.utils.cache import cache_mapper -from wger.utils.constants import ( - OFF_SEARCH_PRODUCT_FOUND, - TWOPLACES, -) +from wger.utils.constants import TWOPLACES from wger.utils.language import load_language from wger.utils.managers import SubmissionManager from wger.utils.models import ( @@ -64,11 +60,9 @@ AbstractSubmissionModel, ) from wger.utils.requests import wger_user_agent - # Local from .ingredient_category import IngredientCategory - logger = logging.getLogger(__name__) @@ -197,7 +191,15 @@ class Ingredient(AbstractSubmissionModel, AbstractLicenseModel, models.Model): blank=True, db_index=True, ) - """Internal ID of the source database, e.g. a barcode or similar""" + """The product's barcode""" + + remote_id = models.CharField( + max_length=200, + null=True, + blank=True, + db_index=True, + ) + """ID of the product in the external source database. Used for updated during imports.""" source_name = models.CharField( max_length=200, diff --git a/wger/nutrition/off.py b/wger/nutrition/off.py index 9e99f141c..e7225d076 100644 --- a/wger/nutrition/off.py +++ b/wger/nutrition/off.py @@ -14,19 +14,14 @@ # along with this program. If not, see . # Standard Library -from dataclasses import ( - asdict, - dataclass, -) -from typing import Optional # wger from wger.nutrition.consts import KJ_PER_KCAL +from wger.nutrition.dataclasses import IngredientData from wger.nutrition.models import Source from wger.utils.constants import ODBL_LICENSE_ID from wger.utils.models import AbstractSubmissionModel - OFF_REQUIRED_TOP_LEVEL = [ 'product_name', 'code', @@ -39,34 +34,7 @@ ] -@dataclass -class IngredientData: - name: str - language_id: int - energy: float - protein: float - carbohydrates: float - carbohydrates_sugar: float - fat: float - fat_saturated: float - fibres: Optional[float] - sodium: Optional[float] - code: str - source_name: str - source_url: str - common_name: str - brand: str - status: str - license_id: int - license_author: str - license_title: str - license_object_url: str - - def dict(self): - return asdict(self) - - -def extract_info_from_off(product_data, language: int): +def extract_info_from_off(product_data: dict, language: int): if not all(req in product_data for req in OFF_REQUIRED_TOP_LEVEL): raise KeyError('Missing required top-level key') @@ -95,12 +63,12 @@ def extract_info_from_off(product_data, language: int): code = product_data['code'] protein = product_data['nutriments']['proteins_100g'] carbs = product_data['nutriments']['carbohydrates_100g'] - sugars = product_data['nutriments'].get('sugars_100g', 0) fat = product_data['nutriments']['fat_100g'] - saturated = product_data['nutriments'].get('saturated-fat_100g', 0) # these are optional + saturated = product_data['nutriments'].get('saturated-fat_100g', None) sodium = product_data['nutriments'].get('sodium_100g', None) + sugars = product_data['nutriments'].get('sugars_100g', None) fibre = product_data['nutriments'].get('fiber_100g', None) brand = product_data.get('brands', None) @@ -111,6 +79,7 @@ def extract_info_from_off(product_data, language: int): object_url = f'https://world.openfoodfacts.org/product/{code}/' return IngredientData( + remote_id=code, name=name, language_id=language, energy=energy, diff --git a/wger/nutrition/tests/test_ingredient.py b/wger/nutrition/tests/test_ingredient.py index 51afabd4d..5d8c8a549 100644 --- a/wger/nutrition/tests/test_ingredient.py +++ b/wger/nutrition/tests/test_ingredient.py @@ -41,11 +41,7 @@ Ingredient, Meal, ) -from wger.utils.constants import ( - NUTRITION_TAB, - OFF_SEARCH_PRODUCT_FOUND, - OFF_SEARCH_PRODUCT_NOT_FOUND, -) +from wger.utils.constants import NUTRITION_TAB class IngredientRepresentationTestCase(WgerTestCase): @@ -463,7 +459,6 @@ class IngredientModelTestCase(WgerTestCase): def setUp(self): super().setUp() self.off_response = { - 'status': OFF_SEARCH_PRODUCT_FOUND, 'product': { 'code': '1234', 'lang': 'de', @@ -484,10 +479,7 @@ def setUp(self): }, } - self.off_response_no_results = { - 'status': OFF_SEARCH_PRODUCT_NOT_FOUND, - 'status_verbose': 'product not found', - } + self.off_response_no_results = None @patch('openfoodfacts.api.ProductResource.get') def test_fetch_from_off_success(self, mock_api): diff --git a/wger/nutrition/tests/test_off.py b/wger/nutrition/tests/test_off.py index de652c26e..de9ed19d5 100644 --- a/wger/nutrition/tests/test_off.py +++ b/wger/nutrition/tests/test_off.py @@ -18,9 +18,9 @@ # wger from wger.nutrition.off import ( - IngredientData, extract_info_from_off, ) +from wger.nutrition.dataclasses import IngredientData from wger.utils.constants import ODBL_LICENSE_ID from wger.utils.models import AbstractSubmissionModel @@ -60,6 +60,7 @@ def test_regular_response(self): result = extract_info_from_off(self.off_data1, 1) data = IngredientData( name='Foo with chocolate', + remote_id='1234', language_id=1, energy=120, protein=10, @@ -112,5 +113,5 @@ def test_no_sugar_or_saturated_fat(self): del self.off_data1['nutriments']['saturated-fat_100g'] result = extract_info_from_off(self.off_data1, 1) - self.assertEqual(result.carbohydrates_sugar, 0) - self.assertEqual(result.fat_saturated, 0) + self.assertEqual(result.carbohydrates_sugar, None) + self.assertEqual(result.fat_saturated, None) diff --git a/wger/utils/constants.py b/wger/utils/constants.py index 144be1f2d..d327a380c 100644 --- a/wger/utils/constants.py +++ b/wger/utils/constants.py @@ -15,7 +15,6 @@ # Standard Library from decimal import Decimal - # Navigation WORKOUT_TAB = 'workout' EXERCISE_TAB = 'exercises' @@ -66,9 +65,5 @@ DOWNLOAD_INGREDIENT_NONE, ) -# OFF Api -OFF_SEARCH_PRODUCT_FOUND = 1 -OFF_SEARCH_PRODUCT_NOT_FOUND = 0 - # API API_MAX_ITEMS = 999 From 741a3c7ebbdc2cc5dbf01ff4139c399690c13680 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Sat, 18 May 2024 16:25:53 +0200 Subject: [PATCH 06/34] If no ingredients were found the result is now None --- wger/nutrition/models/ingredient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.py index 4cf48975a..d39e74d0d 100644 --- a/wger/nutrition/models/ingredient.py +++ b/wger/nutrition/models/ingredient.py @@ -475,7 +475,8 @@ def fetch_ingredient_from_off(cls, code: str): except JSONDecodeError as e: logger.info(f'Got JSONDecodeError from OFF: {e}') return None - if result['status'] != OFF_SEARCH_PRODUCT_FOUND: + + if not result: logger.info('Product not found') return None product = result['product'] From d5a796eb3b8664d7df0c2b22886ef995fb1295a3 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Sun, 19 May 2024 12:40:54 +0200 Subject: [PATCH 07/34] Make USDA food import actually import the products --- wger/nutrition/dataclasses.py | 8 +- wger/nutrition/management/__init__.py | 15 ++ .../commands/import-off-products.py | 20 +- .../commands/import-usda-products.py | 175 +++++++----------- .../management/commands/sync-ingredients.py | 9 - .../management/{commands => }/products.py | 10 +- ...remote_id_increase_author_field_length.py} | 22 +++ wger/nutrition/models/ingredient.py | 2 + wger/nutrition/models/sources.py | 1 + wger/nutrition/off.py | 5 +- wger/nutrition/tests/test_off.py | 4 +- wger/nutrition/usda.py | 103 +++++++++++ wger/utils/constants.py | 2 + wger/utils/models.py | 2 +- 14 files changed, 235 insertions(+), 143 deletions(-) create mode 100644 wger/nutrition/management/__init__.py rename wger/nutrition/management/{commands => }/products.py (93%) rename wger/nutrition/migrations/{0021_add_remote_id.py => 0021_add_remote_id_increase_author_field_length.py} (59%) create mode 100644 wger/nutrition/usda.py diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.py index b5c443c4e..c135f30cf 100644 --- a/wger/nutrition/dataclasses.py +++ b/wger/nutrition/dataclasses.py @@ -12,7 +12,11 @@ # # You should have received a copy of the GNU Affero General Public License -from dataclasses import dataclass, asdict +# Standard Library +from dataclasses import ( + asdict, + dataclass, +) from typing import Optional @@ -29,7 +33,7 @@ class IngredientData: fat_saturated: float fibres: Optional[float] sodium: Optional[float] - code: str + code: Optional[str] source_name: str source_url: str common_name: str diff --git a/wger/nutrition/management/__init__.py b/wger/nutrition/management/__init__.py new file mode 100644 index 000000000..1292caeab --- /dev/null +++ b/wger/nutrition/management/__init__.py @@ -0,0 +1,15 @@ +# This file is part of wger Workout Manager . +# Copyright (C) 2013 - 2021 wger Team +# +# wger Workout Manager is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# wger Workout Manager is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.py index 17857211f..c40d9486c 100644 --- a/wger/nutrition/management/commands/import-off-products.py +++ b/wger/nutrition/management/commands/import-off-products.py @@ -14,15 +14,16 @@ # Standard Library import logging -from collections import Counter - -# Django -from wger.nutrition.management.commands.products import ImportProductCommand, Mode # wger from wger.core.models import Language +from wger.nutrition.management.products import ( + ImportProductCommand, + Mode, +) from wger.nutrition.off import extract_info_from_off + logger = logging.getLogger(__name__) @@ -31,8 +32,6 @@ class Command(ImportProductCommand): Import an Open Food facts Dump """ - completeness = 0.7 - help = 'Import an Open Food Facts dump. Please consult extras/docker/open-food-facts' def handle(self, **options): @@ -43,8 +42,6 @@ def handle(self, **options): self.stdout.write('Please install pymongo, `pip install pymongo`') return - self.counter = Counter() - if options['mode'] == 'insert': self.mode = Mode.INSERT @@ -55,7 +52,7 @@ def handle(self, **options): client = MongoClient('mongodb://off:off-wger@127.0.0.1', port=27017) db = client.admin - languages = {l.short_name: l.pk for l in Language.objects.all()} + languages = {lang.short_name: lang.pk for lang in Language.objects.all()} for product in db.products.find({'lang': {'$in': list(languages.keys())}}): try: @@ -65,9 +62,8 @@ def handle(self, **options): # self.stdout.write(repr(e)) # pprint(product) self.counter['skipped'] += 1 - continue - - self.handle_data(ingredient_data) + else: + self.handle_data(ingredient_data) self.stdout.write(self.style.SUCCESS('Finished!')) self.stdout.write(self.style.SUCCESS(str(self.counter))) diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py index d59b11ea0..ae0ad9272 100644 --- a/wger/nutrition/management/commands/import-usda-products.py +++ b/wger/nutrition/management/commands/import-usda-products.py @@ -13,7 +13,6 @@ # You should have received a copy of the GNU Affero General Public License # Standard Library -import enum import json import logging import os @@ -21,131 +20,85 @@ from json import JSONDecodeError from zipfile import ZipFile +# Third Party import requests -# Django -from django.core.management.base import BaseCommand - -logger = logging.getLogger(__name__) +# wger +from wger.core.models import Language +from wger.nutrition.management.products import ( + ImportProductCommand, + Mode, +) +from wger.nutrition.usda import extract_info_from_usda +from wger.utils.constants import ENGLISH_SHORT_NAME -# Mode for this script. When using 'insert', the script will bulk-insert the new -# ingredients, which is very efficient. Importing the whole database will require -# barely a minute. When using 'update', existing ingredients will be updated, which -# requires two queries per product and is needed when there are already existing -# entries in the local ingredient table. -class Mode(enum.Enum): - INSERT = enum.auto() - UPDATE = enum.auto() +logger = logging.getLogger(__name__) -class Command(BaseCommand): +class Command(ImportProductCommand): """ Import an Open Food facts Dump """ - mode = Mode.UPDATE - - def add_arguments(self, parser): - parser.add_argument( - '--set-mode', - action='store', - default=10, - dest='mode', - type=str, - help='Script mode, "insert" or "update". Insert will insert the ingredients as new ' - 'entries in the database, while update will try to update them if they are ' - 'already present. Deault: insert', - ) - def handle(self, **options): if options['mode'] == 'insert': self.mode = Mode.INSERT + usda_url = 'https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_foundation_food_json_2024-04-18.zip' + folder = '/Users/roland/Entwicklung/wger/server/extras/usda' + self.stdout.write('Importing entries from USDA') self.stdout.write(f' - {self.mode}') + self.stdout.write(f' - {folder=}') self.stdout.write('') - usda_url = 'https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_foundation_food_json_2024-04-18.zip' - - with tempfile.TemporaryDirectory() as folder: - folder = '/Users/roland/Entwicklung/wger/server/extras/usda' - - print(f'{folder=}') - zip_file = os.path.join(folder, 'usda.zip') - if os.path.exists(zip_file): - self.stdout.write(f'Already downloaded {zip_file}, skipping download') - else: - self.stdout.write(f'downloading {zip_file}... (this may take a while)') - req = requests.get(usda_url, stream=True) - with open(zip_file, 'wb') as fid: - for chunk in req.iter_content(chunk_size=50 * 1024): - fid.write(chunk) - - self.stdout.write('download successful') - - with ZipFile(zip_file, 'r') as zip_ref: - file_list = zip_ref.namelist() - if not file_list: - raise Exception("No files found in the ZIP archive") - - first_file = file_list[0] - self.stdout.write(f'Extracting {first_file=}') - extracted_file_path = zip_ref.extract(first_file, path=folder) - - with open(extracted_file_path, "r") as extracted_file: - for line in extracted_file: - self.process_product(line.strip().strip(',')) - - def process_product(self, json_data): - try: - data = json.loads(json_data) - except JSONDecodeError as e: - # print(e) - # print(json_data) - # print('---------------') - return - - name = data['description'] - fdc_id = data['fdcId'] - - if not data.get('foodNutrients'): - return - - proteins = None - carbs = None - fats = None - energy = None - for d in data['foodNutrients']: - - if not d.get("nutrient"): - return - - nutrient = d.get("nutrient") - nutrient_id = nutrient.get("id") - - match nutrient_id: - case 1003: - proteins = float(d.get("amount")) - - case 1004: - carbs = float(d.get("amount")) - - case 1005: - fats = float(d.get("amount")) - - case 2048: - energy = float(d.get("amount")) - - if not all([proteins, carbs, fats, energy]): - return - - self.stdout.write(f' - {fdc_id}') - self.stdout.write(f' - {name}') - self.stdout.write(f' - {proteins=}') - self.stdout.write(f' - {carbs=}') - self.stdout.write(f' - {fats=}') - self.stdout.write(f' - {energy=}') - - self.stdout.write('') - return + english = Language.objects.get(short_name=ENGLISH_SHORT_NAME) + + zip_file = os.path.join(folder, 'usda.zip') + if os.path.exists(zip_file): + self.stdout.write(f'File already downloaded {zip_file}, not downloading it again') + else: + self.stdout.write(f'Downloading {zip_file}... (this may take a while)') + req = requests.get(usda_url, stream=True) + with open(zip_file, 'wb') as fid: + for chunk in req.iter_content(chunk_size=50 * 1024): + fid.write(chunk) + + self.stdout.write('download successful') + + with ZipFile(zip_file, 'r') as zip_ref: + file_list = zip_ref.namelist() + if not file_list: + raise Exception('No files found in the ZIP archive') + + first_file = file_list[0] + self.stdout.write(f'Extracting {first_file=}') + extracted_file_path = zip_ref.extract(first_file, path=folder) + + # Since the file is almost JSONL, just process each line individually + with open(extracted_file_path, 'r') as extracted_file: + for line in extracted_file.readlines(): + # Skip the first and last lines in the file + if 'FoundationFoods' in line: + continue + + if line.strip() == '}': + continue + + try: + json_data = json.loads(line.strip().strip(',')) + except JSONDecodeError as e: + self.stdout.write(f'--> Error while decoding JSON: {e}') + continue + + try: + ingredient_data = extract_info_from_usda(json_data, english.pk) + except KeyError as e: + self.stdout.write(f'--> KeyError while extracting info from USDA: {e}') + self.counter['skipped'] += 1 + else: + self.handle_data(ingredient_data) + + self.stdout.write(self.style.SUCCESS('Finished!')) + self.stdout.write(self.style.SUCCESS(str(self.counter))) diff --git a/wger/nutrition/management/commands/sync-ingredients.py b/wger/nutrition/management/commands/sync-ingredients.py index 4f4a92fa8..5e12aaff1 100644 --- a/wger/nutrition/management/commands/sync-ingredients.py +++ b/wger/nutrition/management/commands/sync-ingredients.py @@ -22,15 +22,6 @@ from django.core.validators import URLValidator # wger -from wger.exercises.sync import ( - handle_deleted_entries, - sync_categories, - sync_equipment, - sync_exercises, - sync_languages, - sync_licenses, - sync_muscles, -) from wger.nutrition.sync import sync_ingredients diff --git a/wger/nutrition/management/commands/products.py b/wger/nutrition/management/products.py similarity index 93% rename from wger/nutrition/management/commands/products.py rename to wger/nutrition/management/products.py index c4fdecb71..aead23dd9 100644 --- a/wger/nutrition/management/commands/products.py +++ b/wger/nutrition/management/products.py @@ -24,6 +24,7 @@ from wger.nutrition.dataclasses import IngredientData from wger.nutrition.models import Ingredient + logger = logging.getLogger(__name__) @@ -49,6 +50,10 @@ class ImportProductCommand(BaseCommand): help = "Don't run this command directly. Use either import-off-products or import-usda-products" + def __init__(self, stdout=None, stderr=None, no_color=False, force_color=False): + super().__init__(stdout, stderr, no_color, force_color) + self.counter = Counter() + def add_arguments(self, parser): parser.add_argument( '--set-mode', @@ -57,15 +62,14 @@ def add_arguments(self, parser): dest='mode', type=str, help='Script mode, "insert" or "update". Insert will insert the ingredients as new ' - 'entries in the database, while update will try to update them if they are ' - 'already present. Deault: insert', + 'entries in the database, while update will try to update them if they are ' + 'already present. Default: update', ) def handle(self, **options): raise NotImplementedError('Do not run this command on its own!') def handle_data(self, ingredient_data: IngredientData): - # # Add entries as new products if self.mode == Mode.INSERT: diff --git a/wger/nutrition/migrations/0021_add_remote_id.py b/wger/nutrition/migrations/0021_add_remote_id_increase_author_field_length.py similarity index 59% rename from wger/nutrition/migrations/0021_add_remote_id.py rename to wger/nutrition/migrations/0021_add_remote_id_increase_author_field_length.py index 6feefbc22..9d11dfcce 100644 --- a/wger/nutrition/migrations/0021_add_remote_id.py +++ b/wger/nutrition/migrations/0021_add_remote_id_increase_author_field_length.py @@ -34,4 +34,26 @@ class Migration(migrations.Migration): set_external_id, reverse_code=migrations.RunPython.noop, ), + migrations.AlterField( + model_name='image', + name='license_author', + field=models.CharField( + blank=True, + help_text='If you are not the author, enter the name or source here.', + max_length=2500, + null=True, + verbose_name='Author(s)', + ), + ), + migrations.AlterField( + model_name='ingredient', + name='license_author', + field=models.CharField( + blank=True, + help_text='If you are not the author, enter the name or source here.', + max_length=2500, + null=True, + verbose_name='Author(s)', + ), + ), ] diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.py index d39e74d0d..35d66ddde 100644 --- a/wger/nutrition/models/ingredient.py +++ b/wger/nutrition/models/ingredient.py @@ -60,9 +60,11 @@ AbstractSubmissionModel, ) from wger.utils.requests import wger_user_agent + # Local from .ingredient_category import IngredientCategory + logger = logging.getLogger(__name__) diff --git a/wger/nutrition/models/sources.py b/wger/nutrition/models/sources.py index 2d276966e..478d3e63f 100644 --- a/wger/nutrition/models/sources.py +++ b/wger/nutrition/models/sources.py @@ -20,3 +20,4 @@ class Source(Enum): WGER = 'wger' OPEN_FOOD_FACTS = 'Open Food Facts' + USDA = 'USDA' diff --git a/wger/nutrition/off.py b/wger/nutrition/off.py index e7225d076..9423fa216 100644 --- a/wger/nutrition/off.py +++ b/wger/nutrition/off.py @@ -22,6 +22,7 @@ from wger.utils.constants import ODBL_LICENSE_ID from wger.utils.models import AbstractSubmissionModel + OFF_REQUIRED_TOP_LEVEL = [ 'product_name', 'code', @@ -34,7 +35,7 @@ ] -def extract_info_from_off(product_data: dict, language: int): +def extract_info_from_off(product_data: dict, language: int) -> IngredientData: if not all(req in product_data for req in OFF_REQUIRED_TOP_LEVEL): raise KeyError('Missing required top-level key') @@ -70,7 +71,7 @@ def extract_info_from_off(product_data: dict, language: int): sodium = product_data['nutriments'].get('sodium_100g', None) sugars = product_data['nutriments'].get('sugars_100g', None) fibre = product_data['nutriments'].get('fiber_100g', None) - brand = product_data.get('brands', None) + brand = product_data.get('brands', '') # License and author info source_name = Source.OPEN_FOOD_FACTS.value diff --git a/wger/nutrition/tests/test_off.py b/wger/nutrition/tests/test_off.py index de9ed19d5..5583b2306 100644 --- a/wger/nutrition/tests/test_off.py +++ b/wger/nutrition/tests/test_off.py @@ -17,10 +17,8 @@ from django.test import SimpleTestCase # wger -from wger.nutrition.off import ( - extract_info_from_off, -) from wger.nutrition.dataclasses import IngredientData +from wger.nutrition.off import extract_info_from_off from wger.utils.constants import ODBL_LICENSE_ID from wger.utils.models import AbstractSubmissionModel diff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.py new file mode 100644 index 000000000..cb5c93e86 --- /dev/null +++ b/wger/nutrition/usda.py @@ -0,0 +1,103 @@ +# This file is part of wger Workout Manager . +# +# wger Workout Manager is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# wger Workout Manager is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# wger +from wger.nutrition.dataclasses import IngredientData +from wger.nutrition.models import Source +from wger.utils.constants import CC_0_LICENSE_ID +from wger.utils.models import AbstractSubmissionModel + + +def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: + if not product_data.get('foodNutrients'): + raise KeyError('Missing key "foodNutrients"') + + energy = 0 + protein = 0 + carbs = 0 + fats = 0 + sugars = 0 + saturated = 0 + fat = 0 + + sodium = None + fibre = None + + for d in product_data['foodNutrients']: + if not d.get('nutrient'): + raise KeyError('Missing key "nutrient"') + + nutrient = d.get('nutrient') + nutrient_id = nutrient.get('id') + + match nutrient_id: + case 1003: + protein = float(d.get('amount')) + + case 1004: + carbs = float(d.get('amount')) + + case 1005: + fats = float(d.get('amount')) + + case 2048: + energy = float(d.get('amount')) + + case 1093: + sodium = float(d.get('amount')) + + if not all([protein, carbs, fats, energy]): + raise KeyError('Could not extract all nutrition information') + + name = product_data['description'] + if len(name) > 200: + name = name[:200] + + code = None + remote_id = product_data['fdcId'] + brand = '' + + # License and author info + source_name = Source.USDA.value + source_url = f'https://fdc.nal.usda.gov/' + author = ( + 'U.S. Department of Agriculture, Agricultural Research Service, ' + 'Beltsville Human Nutrition Research Center. FoodData Central.' + ) + object_url = f'https://api.nal.usda.gov/fdc/v1/food/{remote_id}' + + return IngredientData( + code=code, + remote_id=remote_id, + name=name, + common_name=name, + language_id=language, + energy=energy, + protein=protein, + carbohydrates=carbs, + carbohydrates_sugar=sugars, + fat=fat, + fat_saturated=saturated, + fibres=fibre, + sodium=sodium, + source_name=source_name, + source_url=source_url, + brand=brand, + status=AbstractSubmissionModel.STATUS_ACCEPTED, + license_id=CC_0_LICENSE_ID, + license_author=author, + license_title=name, + license_object_url=object_url, + ) diff --git a/wger/utils/constants.py b/wger/utils/constants.py index d327a380c..1a24c0a13 100644 --- a/wger/utils/constants.py +++ b/wger/utils/constants.py @@ -15,6 +15,7 @@ # Standard Library from decimal import Decimal + # Navigation WORKOUT_TAB = 'workout' EXERCISE_TAB = 'exercises' @@ -50,6 +51,7 @@ # Important license IDs CC_BY_SA_4_ID = 2 CC_BY_SA_3_LICENSE_ID = 1 +CC_0_LICENSE_ID = 3 ODBL_LICENSE_ID = 5 # Default/fallback language diff --git a/wger/utils/models.py b/wger/utils/models.py index bfd80c8af..5e7fe89bc 100644 --- a/wger/utils/models.py +++ b/wger/utils/models.py @@ -61,7 +61,7 @@ class Meta: license_author = models.CharField( verbose_name=_('Author(s)'), - max_length=600, + max_length=2500, blank=True, null=True, help_text=_('If you are not the author, enter the name or source here.'), From 1a0ab64681c6fa489e729535b33414b258559c78 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Mon, 20 May 2024 14:11:54 +0200 Subject: [PATCH 08/34] Some polishing, add tests for the USDA extraction function --- wger/nutrition/dataclasses.py | 4 +- .../commands/import-usda-products.py | 9 +- wger/nutrition/management/products.py | 1 + wger/nutrition/tests/test_usda.py | 282 ++++++++++++++++++ wger/nutrition/usda.py | 71 +++-- wger/utils/requests.py | 2 +- 6 files changed, 338 insertions(+), 31 deletions(-) create mode 100644 wger/nutrition/tests/test_usda.py diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.py index c135f30cf..b190d5c6f 100644 --- a/wger/nutrition/dataclasses.py +++ b/wger/nutrition/dataclasses.py @@ -28,9 +28,9 @@ class IngredientData: energy: float protein: float carbohydrates: float - carbohydrates_sugar: float + carbohydrates_sugar: Optional[float] fat: float - fat_saturated: float + fat_saturated: Optional[float] fibres: Optional[float] sodium: Optional[float] code: Optional[str] diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py index ae0ad9272..d91dbc930 100644 --- a/wger/nutrition/management/commands/import-usda-products.py +++ b/wger/nutrition/management/commands/import-usda-products.py @@ -31,6 +31,7 @@ ) from wger.nutrition.usda import extract_info_from_usda from wger.utils.constants import ENGLISH_SHORT_NAME +from wger.utils.requests import wger_headers logger = logging.getLogger(__name__) @@ -45,7 +46,9 @@ def handle(self, **options): if options['mode'] == 'insert': self.mode = Mode.INSERT - usda_url = 'https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_foundation_food_json_2024-04-18.zip' + current_file = 'FoodData_Central_foundation_food_json_2024-04-18.zip' + + usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{current_file}' folder = '/Users/roland/Entwicklung/wger/server/extras/usda' self.stdout.write('Importing entries from USDA') @@ -55,12 +58,12 @@ def handle(self, **options): english = Language.objects.get(short_name=ENGLISH_SHORT_NAME) - zip_file = os.path.join(folder, 'usda.zip') + zip_file = os.path.join(folder, current_file) if os.path.exists(zip_file): self.stdout.write(f'File already downloaded {zip_file}, not downloading it again') else: self.stdout.write(f'Downloading {zip_file}... (this may take a while)') - req = requests.get(usda_url, stream=True) + req = requests.get(usda_url, stream=True, headers=wger_headers()) with open(zip_file, 'wb') as fid: for chunk in req.iter_content(chunk_size=50 * 1024): fid.write(chunk) diff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.py index aead23dd9..50965e549 100644 --- a/wger/nutrition/management/products.py +++ b/wger/nutrition/management/products.py @@ -118,4 +118,5 @@ def handle_data(self, ingredient_data: IngredientData): except Exception as e: self.stdout.write('--> Error while performing update_or_create') self.stdout.write(repr(e)) + # self.stdout.write(repr(ingredient_data)) self.counter['error'] += 1 diff --git a/wger/nutrition/tests/test_usda.py b/wger/nutrition/tests/test_usda.py new file mode 100644 index 000000000..eac565e46 --- /dev/null +++ b/wger/nutrition/tests/test_usda.py @@ -0,0 +1,282 @@ +# This file is part of wger Workout Manager. +# +# wger Workout Manager is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# wger Workout Manager is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Workout Manager. If not, see . + +# Django +from django.test import SimpleTestCase + +# wger +from wger.nutrition.dataclasses import IngredientData +from wger.nutrition.usda import ( + convert_to_g_per_100g, + extract_info_from_usda, +) +from wger.utils.constants import CC_0_LICENSE_ID +from wger.utils.models import AbstractSubmissionModel + + +class ExtractInfoFromUSDATestCase(SimpleTestCase): + """ + Test the extract_info_from_usda function + """ + + usda_data1 = {} + + def setUp(self): + self.usda_data1 = { + 'foodClass': 'FinalFood', + 'description': 'Foo with chocolate', + 'foodNutrients': [ + { + 'type': 'FoodNutrient', + 'id': 2259514, + 'nutrient': { + 'id': 1170, + 'number': '410', + 'name': 'Pantothenic acid', + 'rank': 6700, + 'unitName': 'mg', + }, + 'dataPoints': 2, + 'foodNutrientDerivation': { + 'code': 'A', + 'description': 'Analytical', + 'foodNutrientSource': { + 'id': 1, + 'code': '1', + 'description': 'Analytical or derived from analytical', + }, + }, + 'max': 1.64, + 'min': 1.53, + 'median': 1.58, + 'amount': 1.58, + }, + { + 'type': 'FoodNutrient', + 'id': 2259524, + 'nutrient': { + 'id': 1004, + 'number': '204', + 'name': 'Total lipid (fat)', + 'rank': 800, + 'unitName': 'g', + }, + 'dataPoints': 6, + 'foodNutrientDerivation': { + 'code': 'A', + 'description': 'Analytical', + 'foodNutrientSource': { + 'id': 1, + 'code': '1', + 'description': 'Analytical or derived from analytical', + }, + }, + 'max': 3.99, + 'min': 2.17, + 'median': 3.26, + 'amount': 3.24, + }, + { + 'type': 'FoodNutrient', + 'id': 2259525, + 'nutrient': { + 'id': 1005, + 'number': '205', + 'name': 'Carbohydrate, by difference', + 'rank': 1110, + 'unitName': 'g', + }, + 'foodNutrientDerivation': { + 'code': 'NC', + 'description': 'Calculated', + 'foodNutrientSource': { + 'id': 2, + 'code': '4', + 'description': 'Calculated or imputed', + }, + }, + 'amount': 0.000, + }, + { + 'type': 'FoodNutrient', + 'id': 2259526, + 'nutrient': { + 'id': 1008, + 'number': '208', + 'name': 'Energy', + 'rank': 300, + 'unitName': 'kcal', + }, + 'foodNutrientDerivation': { + 'code': 'NC', + 'description': 'Calculated', + 'foodNutrientSource': { + 'id': 2, + 'code': '4', + 'description': 'Calculated or imputed', + }, + }, + 'amount': 166, + }, + { + 'type': 'FoodNutrient', + 'id': 2259565, + 'nutrient': { + 'id': 1003, + 'number': '203', + 'name': 'Protein', + 'rank': 600, + 'unitName': 'g', + }, + 'foodNutrientDerivation': { + 'code': 'NC', + 'description': 'Calculated', + 'foodNutrientSource': { + 'id': 2, + 'code': '4', + 'description': 'Calculated or imputed', + }, + }, + 'max': 32.9, + 'min': 31.3, + 'median': 32.1, + 'amount': 32.1, + }, + ], + 'foodAttributes': [], + 'nutrientConversionFactors': [ + { + 'type': '.CalorieConversionFactor', + 'proteinValue': 4.27, + 'fatValue': 9.02, + 'carbohydrateValue': 3.87, + }, + {'type': '.ProteinConversionFactor', 'value': 6.25}, + ], + 'isHistoricalReference': False, + 'ndbNumber': 5746, + 'foodPortions': [ + { + 'id': 121343, + 'value': 1.0, + 'measureUnit': {'id': 1043, 'name': 'piece', 'abbreviation': 'piece'}, + 'gramWeight': 174.0, + 'sequenceNumber': 1, + 'minYearAcquired': 2012, + 'amount': 1.0, + } + ], + 'publicationDate': '4/1/2019', + 'inputFoods': [ + { + 'id': 11213, + 'foodDescription': 'Lorem ipsum', + 'inputFood': { + 'foodClass': 'Composite', + 'description': '......', + 'publicationDate': '4/1/2019', + 'foodCategory': { + 'id': 5, + 'code': '0500', + 'description': 'Poultry Products', + }, + 'fdcId': 331904, + 'dataType': 'Sample', + }, + }, + ], + 'foodCategory': {'description': 'Poultry Products'}, + 'fdcId': 1234567, + 'dataType': 'Foundation', + } + + def test_regular_response(self): + """ + Test that the function can read the regular case + """ + + result = extract_info_from_usda(self.usda_data1, 1) + data = IngredientData( + name='Foo with chocolate', + remote_id='1234567', + language_id=1, + energy=166.0, + protein=32.1, + carbohydrates=3.24, + carbohydrates_sugar=None, + fat=0.0, + fat_saturated=None, + fibres=None, + sodium=None, + code=None, + source_name='USDA', + source_url='https://fdc.nal.usda.gov/', + common_name='Foo with chocolate', + brand='', + status=AbstractSubmissionModel.STATUS_ACCEPTED, + license_id=CC_0_LICENSE_ID, + license_author='U.S. Department of Agriculture, Agricultural Research Service, ' + 'Beltsville Human Nutrition Research Center. FoodData Central.', + license_title='Foo with chocolate', + license_object_url='https://fdc.nal.usda.gov/fdc-app.html#/food-details/1234567/nutrients', + ) + + self.assertEqual(result, data) + + def test_no_energy(self): + """ + No energy available + """ + del self.usda_data1['foodNutrients'][3] + self.assertRaises(KeyError, extract_info_from_usda, self.usda_data1, 1) + + def test_no_nutrients(self): + """ + No nutrients available + """ + del self.usda_data1['foodNutrients'] + self.assertRaises(KeyError, extract_info_from_usda, self.usda_data1, 1) + + def test_converting_grams(self): + """ + Convert from grams (nothing changes) + """ + + entry = {'nutrient': {'unitName': 'g'}, 'amount': '5.0'} + self.assertEqual(convert_to_g_per_100g(entry), 5.0) + + def test_converting_milligrams(self): + """ + Convert from milligrams + """ + + entry = {'nutrient': {'unitName': 'mg'}, 'amount': '5000'} + self.assertEqual(convert_to_g_per_100g(entry), 5.0) + + def test_converting_unknown_unit(self): + """ + Convert from unknown unit + """ + + entry = {'nutrient': {'unitName': 'kg'}, 'amount': '5.0'} + self.assertRaises(ValueError, convert_to_g_per_100g, entry) + + def test_converting_invalid_amount(self): + """ + Convert from invalid amount + """ + + entry = {'nutrient': {'unitName': 'g'}, 'amount': 'invalid'} + self.assertRaises(ValueError, convert_to_g_per_100g, entry) diff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.py index cb5c93e86..ee67832ab 100644 --- a/wger/nutrition/usda.py +++ b/wger/nutrition/usda.py @@ -20,53 +20,74 @@ from wger.utils.models import AbstractSubmissionModel +def convert_to_g_per_100g(entry: dict) -> float: + """ + Convert a nutrient entry to grams per 100g + """ + + nutrient = entry['nutrient'] + amount = float(entry['amount']) + + if nutrient['unitName'] == 'g': + return amount + + elif nutrient['unitName'] == 'mg': + return amount / 1000 + + else: + raise ValueError(f'Unknown unit: {nutrient["unitName"]}') + + def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: if not product_data.get('foodNutrients'): raise KeyError('Missing key "foodNutrients"') - energy = 0 - protein = 0 - carbs = 0 - fats = 0 - sugars = 0 - saturated = 0 - fat = 0 + energy = None + protein = None + carbs = None + sugars = None + fats = None + fats_saturated = None sodium = None fibre = None - for d in product_data['foodNutrients']: - if not d.get('nutrient'): + for nutrient in product_data['foodNutrients']: + if not nutrient.get('nutrient'): raise KeyError('Missing key "nutrient"') - nutrient = d.get('nutrient') - nutrient_id = nutrient.get('id') - + nutrient_id = nutrient['nutrient']['id'] match nutrient_id: + case 1008: + energy = float(nutrient.get('amount')) + case 1003: - protein = float(d.get('amount')) + protein = convert_to_g_per_100g(nutrient) case 1004: - carbs = float(d.get('amount')) + carbs = convert_to_g_per_100g(nutrient) case 1005: - fats = float(d.get('amount')) - - case 2048: - energy = float(d.get('amount')) + fats = convert_to_g_per_100g(nutrient) + # These are optional case 1093: - sodium = float(d.get('amount')) + sodium = convert_to_g_per_100g(nutrient) + + case 1079: + fibre = convert_to_g_per_100g(nutrient) - if not all([protein, carbs, fats, energy]): - raise KeyError('Could not extract all nutrition information') + macros = [energy, protein, carbs, fats] + for value in macros: + if value is None: + raise KeyError(f'Could not extract all basic macros: {macros=}') name = product_data['description'] if len(name) > 200: name = name[:200] code = None - remote_id = product_data['fdcId'] + remote_id = str(product_data['fdcId']) brand = '' # License and author info @@ -76,7 +97,7 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: 'U.S. Department of Agriculture, Agricultural Research Service, ' 'Beltsville Human Nutrition Research Center. FoodData Central.' ) - object_url = f'https://api.nal.usda.gov/fdc/v1/food/{remote_id}' + object_url = f'https://fdc.nal.usda.gov/fdc-app.html#/food-details/{remote_id}/nutrients' return IngredientData( code=code, @@ -88,8 +109,8 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: protein=protein, carbohydrates=carbs, carbohydrates_sugar=sugars, - fat=fat, - fat_saturated=saturated, + fat=fats, + fat_saturated=fats_saturated, fibres=fibre, sodium=sodium, source_name=source_name, diff --git a/wger/utils/requests.py b/wger/utils/requests.py index d59020e63..aec1fd6f6 100644 --- a/wger/utils/requests.py +++ b/wger/utils/requests.py @@ -24,7 +24,7 @@ def wger_user_agent(): def wger_headers(): - return {'User-agent': wger_user_agent()} + return {'User-Agent': wger_user_agent()} def get_all_paginated(url: str, headers=None): From 248f28bafd7fd4195a284a4286ae8358ffe27328 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Mon, 20 May 2024 15:42:07 +0200 Subject: [PATCH 09/34] Better handling branded products --- .../commands/import-usda-products.py | 27 +++++++++++-------- wger/nutrition/tests/test_usda.py | 12 ++++----- wger/nutrition/usda.py | 23 ++++++++-------- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py index d91dbc930..de623380a 100644 --- a/wger/nutrition/management/commands/import-usda-products.py +++ b/wger/nutrition/management/commands/import-usda-products.py @@ -33,9 +33,13 @@ from wger.utils.constants import ENGLISH_SHORT_NAME from wger.utils.requests import wger_headers - logger = logging.getLogger(__name__) +# Check https://fdc.nal.usda.gov/download-datasets.html for current file names +# current_file = 'FoodData_Central_foundation_food_json_2024-04-18.zip' +current_file = 'FoodData_Central_branded_food_json_2024-04-18.zip' +download_folder = '/Users/roland/Entwicklung/wger/server/extras/usda' + class Command(ImportProductCommand): """ @@ -46,19 +50,18 @@ def handle(self, **options): if options['mode'] == 'insert': self.mode = Mode.INSERT - current_file = 'FoodData_Central_foundation_food_json_2024-04-18.zip' - usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{current_file}' - folder = '/Users/roland/Entwicklung/wger/server/extras/usda' self.stdout.write('Importing entries from USDA') self.stdout.write(f' - {self.mode}') - self.stdout.write(f' - {folder=}') + self.stdout.write(f' - dataset: {usda_url}') + self.stdout.write(f' - download folder: {download_folder}') self.stdout.write('') english = Language.objects.get(short_name=ENGLISH_SHORT_NAME) - zip_file = os.path.join(folder, current_file) + # Download the dataset + zip_file = os.path.join(download_folder, current_file) if os.path.exists(zip_file): self.stdout.write(f'File already downloaded {zip_file}, not downloading it again') else: @@ -70,20 +73,21 @@ def handle(self, **options): self.stdout.write('download successful') + # Extract the first file from the ZIP archive with ZipFile(zip_file, 'r') as zip_ref: file_list = zip_ref.namelist() if not file_list: raise Exception('No files found in the ZIP archive') first_file = file_list[0] - self.stdout.write(f'Extracting {first_file=}') - extracted_file_path = zip_ref.extract(first_file, path=folder) + self.stdout.write(f'Extracting {first_file}...') + extracted_file_path = zip_ref.extract(first_file, path=download_folder) # Since the file is almost JSONL, just process each line individually with open(extracted_file_path, 'r') as extracted_file: for line in extracted_file.readlines(): - # Skip the first and last lines in the file - if 'FoundationFoods' in line: + # Try to skip the first and last lines in the file + if 'FoundationFoods' in line or 'BrandedFoods' in line: continue if line.strip() == '}': @@ -98,9 +102,10 @@ def handle(self, **options): try: ingredient_data = extract_info_from_usda(json_data, english.pk) except KeyError as e: - self.stdout.write(f'--> KeyError while extracting info from USDA: {e}') + self.stdout.write(f'--> KeyError while extracting ingredient info: {e}') self.counter['skipped'] += 1 else: + # pass self.handle_data(ingredient_data) self.stdout.write(self.style.SUCCESS('Finished!')) diff --git a/wger/nutrition/tests/test_usda.py b/wger/nutrition/tests/test_usda.py index eac565e46..746228b0d 100644 --- a/wger/nutrition/tests/test_usda.py +++ b/wger/nutrition/tests/test_usda.py @@ -36,7 +36,7 @@ class ExtractInfoFromUSDATestCase(SimpleTestCase): def setUp(self): self.usda_data1 = { 'foodClass': 'FinalFood', - 'description': 'Foo with chocolate', + 'description': 'FOO WITH CHOCOLATE', 'foodNutrients': [ { 'type': 'FoodNutrient', @@ -209,27 +209,27 @@ def test_regular_response(self): result = extract_info_from_usda(self.usda_data1, 1) data = IngredientData( - name='Foo with chocolate', + name='Foo With Chocolate', remote_id='1234567', language_id=1, energy=166.0, protein=32.1, - carbohydrates=3.24, + carbohydrates=0.0, carbohydrates_sugar=None, - fat=0.0, + fat=3.24, fat_saturated=None, fibres=None, sodium=None, code=None, source_name='USDA', source_url='https://fdc.nal.usda.gov/', - common_name='Foo with chocolate', + common_name='Foo With Chocolate', brand='', status=AbstractSubmissionModel.STATUS_ACCEPTED, license_id=CC_0_LICENSE_ID, license_author='U.S. Department of Agriculture, Agricultural Research Service, ' 'Beltsville Human Nutrition Research Center. FoodData Central.', - license_title='Foo with chocolate', + license_title='Foo With Chocolate', license_object_url='https://fdc.nal.usda.gov/fdc-app.html#/food-details/1234567/nutrients', ) diff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.py index ee67832ab..a5b01d816 100644 --- a/wger/nutrition/usda.py +++ b/wger/nutrition/usda.py @@ -42,6 +42,9 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: if not product_data.get('foodNutrients'): raise KeyError('Missing key "foodNutrients"') + remote_id = str(product_data['fdcId']) + barcode = None + energy = None protein = None carbs = None @@ -52,6 +55,8 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: sodium = None fibre = None + brand = product_data.get('brandName', '') + for nutrient in product_data['foodNutrients']: if not nutrient.get('nutrient'): raise KeyError('Missing key "nutrient"') @@ -59,18 +64,18 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: nutrient_id = nutrient['nutrient']['id'] match nutrient_id: case 1008: - energy = float(nutrient.get('amount')) + energy = float(nutrient.get('amount')) # in kcal case 1003: protein = convert_to_g_per_100g(nutrient) - case 1004: + case 1005: carbs = convert_to_g_per_100g(nutrient) - case 1005: + # Total lipid (fat) | Total fat (NLEA) + case 1004 | 1085: fats = convert_to_g_per_100g(nutrient) - # These are optional case 1093: sodium = convert_to_g_per_100g(nutrient) @@ -80,16 +85,12 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: macros = [energy, protein, carbs, fats] for value in macros: if value is None: - raise KeyError(f'Could not extract all basic macros: {macros=}') + raise KeyError(f'Could not extract all basic macros: {macros=} {remote_id=}') - name = product_data['description'] + name = product_data['description'].title() if len(name) > 200: name = name[:200] - code = None - remote_id = str(product_data['fdcId']) - brand = '' - # License and author info source_name = Source.USDA.value source_url = f'https://fdc.nal.usda.gov/' @@ -100,7 +101,7 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: object_url = f'https://fdc.nal.usda.gov/fdc-app.html#/food-details/{remote_id}/nutrients' return IngredientData( - code=code, + code=barcode, remote_id=remote_id, name=name, common_name=name, From 53230ac1c6df62c644e78e4a1b958814adbfcd29 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Mon, 20 May 2024 16:18:10 +0200 Subject: [PATCH 10/34] Download the dataset to a temporary folder by default Also, add some error handling for missing folders, urls, etc. --- .../commands/import-usda-products.py | 69 ++++++++++++++++--- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py index de623380a..c33f12eb0 100644 --- a/wger/nutrition/management/commands/import-usda-products.py +++ b/wger/nutrition/management/commands/import-usda-products.py @@ -35,22 +35,54 @@ logger = logging.getLogger(__name__) -# Check https://fdc.nal.usda.gov/download-datasets.html for current file names -# current_file = 'FoodData_Central_foundation_food_json_2024-04-18.zip' -current_file = 'FoodData_Central_branded_food_json_2024-04-18.zip' -download_folder = '/Users/roland/Entwicklung/wger/server/extras/usda' - class Command(ImportProductCommand): """ - Import an Open Food facts Dump + Import an USDA dataset """ + def add_arguments(self, parser): + super().add_arguments(parser) + + parser.add_argument( + '--use-folder', + action='store', + default='', + dest='tmp_folder', + type=str, + help='Controls whether to use a temporary folder created by python (the default) or ' + 'the path provided for storing the downloaded dataset. If there are already ' + 'downloaded or extracted files here, they will be used instead of fetching them ' + 'again.' + ) + + parser.add_argument( + '--dataset-name', + action='store', + default='FoodData_Central_branded_food_json_2024-04-18.zip', + dest='dataset_name', + type=str, + help='What dataset to download, this value will be appended to ' + '"https://fdc.nal.usda.gov/fdc-datasets/". Consult ' + 'https://fdc.nal.usda.gov/download-datasets.html for current file names', + ) + def handle(self, **options): if options['mode'] == 'insert': self.mode = Mode.INSERT - usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{current_file}' + usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{options["dataset_name"]}' + + if options['tmp_folder']: + download_folder = options['tmp_folder'] + + # Check whether the folder exists + if not os.path.exists(download_folder): + self.stdout.write(self.style.ERROR(f'Folder {download_folder} does not exist!')) + return + else: + tmp_folder = tempfile.TemporaryDirectory() + download_folder = tmp_folder.name self.stdout.write('Importing entries from USDA') self.stdout.write(f' - {self.mode}') @@ -61,12 +93,17 @@ def handle(self, **options): english = Language.objects.get(short_name=ENGLISH_SHORT_NAME) # Download the dataset - zip_file = os.path.join(download_folder, current_file) + zip_file = os.path.join(download_folder, options['dataset_name']) if os.path.exists(zip_file): self.stdout.write(f'File already downloaded {zip_file}, not downloading it again') else: - self.stdout.write(f'Downloading {zip_file}... (this may take a while)') + self.stdout.write(f'Downloading {usda_url}... (this may take a while)') req = requests.get(usda_url, stream=True, headers=wger_headers()) + + if req.status_code == 404: + self.stdout.write(self.style.ERROR(f'Could not open {usda_url}!')) + return + with open(zip_file, 'wb') as fid: for chunk in req.iter_content(chunk_size=50 * 1024): fid.write(chunk) @@ -80,8 +117,14 @@ def handle(self, **options): raise Exception('No files found in the ZIP archive') first_file = file_list[0] - self.stdout.write(f'Extracting {first_file}...') - extracted_file_path = zip_ref.extract(first_file, path=download_folder) + + # If the file was already extracted to the download folder, skip + extracted_file_path = os.path.join(download_folder, first_file) + if os.path.exists(extracted_file_path): + self.stdout.write(f'File {first_file} already extracted, skipping...') + else: + self.stdout.write(f'Extracting {first_file}...') + extracted_file_path = zip_ref.extract(first_file, path=download_folder) # Since the file is almost JSONL, just process each line individually with open(extracted_file_path, 'r') as extracted_file: @@ -108,5 +151,9 @@ def handle(self, **options): # pass self.handle_data(ingredient_data) + if not options['tmp_folder']: + self.stdout.write(f'Removing temporary folder {download_folder}') + tmp_folder.cleanup() + self.stdout.write(self.style.SUCCESS('Finished!')) self.stdout.write(self.style.SUCCESS(str(self.counter))) From 5aebcd2ed887ef750d4c45a582984e38e4c72d71 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Mon, 20 May 2024 21:16:59 +0200 Subject: [PATCH 11/34] Add method to try to fetch existing USDA entries --- .../commands/import-usda-products.py | 50 ++++++++++++++++--- wger/nutrition/usda.py | 8 ++- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py index c33f12eb0..ac3ed6f2f 100644 --- a/wger/nutrition/management/commands/import-usda-products.py +++ b/wger/nutrition/management/commands/import-usda-products.py @@ -25,14 +25,17 @@ # wger from wger.core.models import Language +from wger.nutrition.dataclasses import IngredientData from wger.nutrition.management.products import ( ImportProductCommand, Mode, ) +from wger.nutrition.models import Ingredient from wger.nutrition.usda import extract_info_from_usda from wger.utils.constants import ENGLISH_SHORT_NAME from wger.utils.requests import wger_headers + logger = logging.getLogger(__name__) @@ -51,9 +54,9 @@ def add_arguments(self, parser): dest='tmp_folder', type=str, help='Controls whether to use a temporary folder created by python (the default) or ' - 'the path provided for storing the downloaded dataset. If there are already ' - 'downloaded or extracted files here, they will be used instead of fetching them ' - 'again.' + 'the path provided for storing the downloaded dataset. If there are already ' + 'downloaded or extracted files here, they will be used instead of fetching them ' + 'again.', ) parser.add_argument( @@ -63,8 +66,8 @@ def add_arguments(self, parser): dest='dataset_name', type=str, help='What dataset to download, this value will be appended to ' - '"https://fdc.nal.usda.gov/fdc-datasets/". Consult ' - 'https://fdc.nal.usda.gov/download-datasets.html for current file names', + '"https://fdc.nal.usda.gov/fdc-datasets/". Consult ' + 'https://fdc.nal.usda.gov/download-datasets.html for current file names', ) def handle(self, **options): @@ -147,9 +150,13 @@ def handle(self, **options): except KeyError as e: self.stdout.write(f'--> KeyError while extracting ingredient info: {e}') self.counter['skipped'] += 1 + except ValueError as e: + self.stdout.write(f'--> ValueError while extracting ingredient info: {e}') + self.counter['skipped'] += 1 else: # pass - self.handle_data(ingredient_data) + self.match_existing_entry(ingredient_data) + # self.handle_data(ingredient_data) if not options['tmp_folder']: self.stdout.write(f'Removing temporary folder {download_folder}') @@ -157,3 +164,34 @@ def handle(self, **options): self.stdout.write(self.style.SUCCESS('Finished!')) self.stdout.write(self.style.SUCCESS(str(self.counter))) + + def match_existing_entry(self, ingredient_data: IngredientData): + """ + One-off method. + + There are currently some entries in the database that were imported from an old USDA + import but neither the original script nor the dump are available anymore, so we can't + really know which ones they are. + + This method tries to match the ingredient_data.name with the name of an existing entry + and set the remote_id so that these can be updated regularly in the future. + """ + try: + obj, created = Ingredient.objects.update_or_create( + name__iexact=ingredient_data.name, + code=None, + defaults=ingredient_data.dict(), + ) + + if created: + self.counter['new'] += 1 + # self.stdout.write('-> added to the database') + else: + self.counter['edited'] += 1 + self.stdout.write(f'-> found and updated {obj.name}') + + except Exception as e: + # self.stdout.write('--> Error while performing update_or_create') + # self.stdout.write(repr(e)) + # self.stdout.write(repr(ingredient_data)) + self.counter['error'] += 1 diff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.py index a5b01d816..2c360fb1d 100644 --- a/wger/nutrition/usda.py +++ b/wger/nutrition/usda.py @@ -48,7 +48,7 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: energy = None protein = None carbs = None - sugars = None + carbs_sugars = None fats = None fats_saturated = None @@ -87,6 +87,10 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: if value is None: raise KeyError(f'Could not extract all basic macros: {macros=} {remote_id=}') + for value in [protein, fats, fats_saturated, carbs, carbs_sugars, sodium, fibre]: + if value and value > 100: + raise ValueError(f'Value for macronutrient is greater than 100! {macros=} {remote_id=}') + name = product_data['description'].title() if len(name) > 200: name = name[:200] @@ -109,7 +113,7 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: energy=energy, protein=protein, carbohydrates=carbs, - carbohydrates_sugar=sugars, + carbohydrates_sugar=carbs_sugars, fat=fats, fat_saturated=fats_saturated, fibres=fibre, From aa3f49b4425d7b109526bd5a9dd87671122278e8 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Mon, 20 May 2024 21:39:33 +0200 Subject: [PATCH 12/34] Rename command line options --- .../commands/import-usda-products.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py index ac3ed6f2f..831dd55ba 100644 --- a/wger/nutrition/management/commands/import-usda-products.py +++ b/wger/nutrition/management/commands/import-usda-products.py @@ -48,36 +48,36 @@ def add_arguments(self, parser): super().add_arguments(parser) parser.add_argument( - '--use-folder', + '--folder', action='store', default='', dest='tmp_folder', type=str, help='Controls whether to use a temporary folder created by python (the default) or ' - 'the path provided for storing the downloaded dataset. If there are already ' - 'downloaded or extracted files here, they will be used instead of fetching them ' - 'again.', + 'the path provided for storing the downloaded dataset. If there are already ' + 'downloaded or extracted files here, they will be used instead of fetching them ' + 'again.', ) parser.add_argument( - '--dataset-name', + '--dataset', action='store', default='FoodData_Central_branded_food_json_2024-04-18.zip', - dest='dataset_name', + dest='dataset', type=str, help='What dataset to download, this value will be appended to ' - '"https://fdc.nal.usda.gov/fdc-datasets/". Consult ' - 'https://fdc.nal.usda.gov/download-datasets.html for current file names', + '"https://fdc.nal.usda.gov/fdc-datasets/". Consult ' + 'https://fdc.nal.usda.gov/download-datasets.html for current file names', ) def handle(self, **options): if options['mode'] == 'insert': self.mode = Mode.INSERT - usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{options["dataset_name"]}' + usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{options["dataset"]}' - if options['tmp_folder']: - download_folder = options['tmp_folder'] + if options['folder']: + download_folder = options['folder'] # Check whether the folder exists if not os.path.exists(download_folder): @@ -96,7 +96,7 @@ def handle(self, **options): english = Language.objects.get(short_name=ENGLISH_SHORT_NAME) # Download the dataset - zip_file = os.path.join(download_folder, options['dataset_name']) + zip_file = os.path.join(download_folder, options['dataset']) if os.path.exists(zip_file): self.stdout.write(f'File already downloaded {zip_file}, not downloading it again') else: @@ -158,7 +158,7 @@ def handle(self, **options): self.match_existing_entry(ingredient_data) # self.handle_data(ingredient_data) - if not options['tmp_folder']: + if not options['folder']: self.stdout.write(f'Removing temporary folder {download_folder}') tmp_folder.cleanup() From 8c3ab810720c0b97dae59e7283b777b9c7e1ade5 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Mon, 20 May 2024 21:42:38 +0200 Subject: [PATCH 13/34] Commit missing migration to also increase author field length in other models --- .../0030_increase_author_field_length.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 wger/exercises/migrations/0030_increase_author_field_length.py diff --git a/wger/exercises/migrations/0030_increase_author_field_length.py b/wger/exercises/migrations/0030_increase_author_field_length.py new file mode 100644 index 000000000..45aa68c45 --- /dev/null +++ b/wger/exercises/migrations/0030_increase_author_field_length.py @@ -0,0 +1,100 @@ +# Generated by Django 4.2.6 on 2024-05-20 19:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('exercises', '0029_full_text_search'), + ] + + operations = [ + migrations.AlterField( + model_name='exercise', + name='license_author', + field=models.CharField( + blank=True, + help_text='If you are not the author, enter the name or source here.', + max_length=2500, + null=True, + verbose_name='Author(s)', + ), + ), + migrations.AlterField( + model_name='exercisebase', + name='license_author', + field=models.CharField( + blank=True, + help_text='If you are not the author, enter the name or source here.', + max_length=2500, + null=True, + verbose_name='Author(s)', + ), + ), + migrations.AlterField( + model_name='exerciseimage', + name='license_author', + field=models.CharField( + blank=True, + help_text='If you are not the author, enter the name or source here.', + max_length=2500, + null=True, + verbose_name='Author(s)', + ), + ), + migrations.AlterField( + model_name='exercisevideo', + name='license_author', + field=models.CharField( + blank=True, + help_text='If you are not the author, enter the name or source here.', + max_length=2500, + null=True, + verbose_name='Author(s)', + ), + ), + migrations.AlterField( + model_name='historicalexercise', + name='license_author', + field=models.CharField( + blank=True, + help_text='If you are not the author, enter the name or source here.', + max_length=2500, + null=True, + verbose_name='Author(s)', + ), + ), + migrations.AlterField( + model_name='historicalexercisebase', + name='license_author', + field=models.CharField( + blank=True, + help_text='If you are not the author, enter the name or source here.', + max_length=2500, + null=True, + verbose_name='Author(s)', + ), + ), + migrations.AlterField( + model_name='historicalexerciseimage', + name='license_author', + field=models.CharField( + blank=True, + help_text='If you are not the author, enter the name or source here.', + max_length=2500, + null=True, + verbose_name='Author(s)', + ), + ), + migrations.AlterField( + model_name='historicalexercisevideo', + name='license_author', + field=models.CharField( + blank=True, + help_text='If you are not the author, enter the name or source here.', + max_length=2500, + null=True, + verbose_name='Author(s)', + ), + ), + ] From e4c75c4e3113cbd6f4349beaa8ca268cced7ef90 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Tue, 21 May 2024 17:05:44 +0200 Subject: [PATCH 14/34] Fix migration order --- ...th.py => 0022_add_remote_id_increase_author_field_length.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename wger/nutrition/migrations/{0021_add_remote_id_increase_author_field_length.py => 0022_add_remote_id_increase_author_field_length.py} (97%) diff --git a/wger/nutrition/migrations/0021_add_remote_id_increase_author_field_length.py b/wger/nutrition/migrations/0022_add_remote_id_increase_author_field_length.py similarity index 97% rename from wger/nutrition/migrations/0021_add_remote_id_increase_author_field_length.py rename to wger/nutrition/migrations/0022_add_remote_id_increase_author_field_length.py index 9d11dfcce..290263d78 100644 --- a/wger/nutrition/migrations/0021_add_remote_id_increase_author_field_length.py +++ b/wger/nutrition/migrations/0022_add_remote_id_increase_author_field_length.py @@ -16,7 +16,7 @@ def set_external_id(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('nutrition', '0020_full_text_search'), + ('nutrition', '0021_add_fibers_field'), ] operations = [ From bff088e84f335b14f865102d9ba7fa9a39be2702 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Tue, 21 May 2024 19:08:09 +0200 Subject: [PATCH 15/34] Don't try to rename the index twice --- .../0022_add_remote_id_increase_author_field_length.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/wger/nutrition/migrations/0022_add_remote_id_increase_author_field_length.py b/wger/nutrition/migrations/0022_add_remote_id_increase_author_field_length.py index 290263d78..8d1992aee 100644 --- a/wger/nutrition/migrations/0022_add_remote_id_increase_author_field_length.py +++ b/wger/nutrition/migrations/0022_add_remote_id_increase_author_field_length.py @@ -20,11 +20,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RenameIndex( - model_name='ingredient', - new_name='nutrition_i_name_8f538f_gin', - old_name='nutrition_i_search__f274b7_gin', - ), migrations.AddField( model_name='ingredient', name='remote_id', From b234b2e1bc2ebd985dc4bb9fb9e2e01d27b3dcf7 Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Tue, 21 May 2024 21:25:20 +0200 Subject: [PATCH 16/34] fibre/fibres/fibers -> fiber --- wger/nutrition/api/filtersets.py | 2 +- wger/nutrition/api/serializers.py | 10 +++---- wger/nutrition/api/views.py | 2 +- wger/nutrition/fixtures/test-ingredients.json | 28 +++++++++---------- wger/nutrition/forms.py | 4 +-- wger/nutrition/helpers.py | 12 ++++---- .../migrations/0022_fiber_spelling.py | 23 +++++++++++++++ ...me_goal_fibers_nutritionplan_goal_fiber.py | 18 ++++++++++++ wger/nutrition/models/ingredient.py | 6 ++-- wger/nutrition/models/plan.py | 2 +- wger/nutrition/off.py | 6 ++-- wger/nutrition/static/js/nutrition.js | 2 +- wger/nutrition/sync.py | 2 +- wger/nutrition/templates/ingredient/view.html | 6 ++-- wger/nutrition/tests/test_ingredient.py | 12 ++++---- .../tests/test_ingredient_overview.py | 2 +- .../tests/test_nutritional_calculations.py | 2 +- .../tests/test_nutritional_values.py | 10 +++---- wger/nutrition/tests/test_off.py | 2 +- wger/nutrition/tests/test_sync.py | 4 +-- wger/nutrition/views/plan.py | 6 ++-- 21 files changed, 101 insertions(+), 60 deletions(-) create mode 100644 wger/nutrition/migrations/0022_fiber_spelling.py create mode 100644 wger/nutrition/migrations/0023_rename_goal_fibers_nutritionplan_goal_fiber.py diff --git a/wger/nutrition/api/filtersets.py b/wger/nutrition/api/filtersets.py index 3e9a0d45a..44fdfbfdf 100644 --- a/wger/nutrition/api/filtersets.py +++ b/wger/nutrition/api/filtersets.py @@ -34,7 +34,7 @@ class Meta: 'energy': ['exact'], 'fat': ['exact'], 'fat_saturated': ['exact'], - 'fibres': ['exact'], + 'fiber': ['exact'], 'name': ['exact'], 'protein': ['exact'], 'sodium': ['exact'], diff --git a/wger/nutrition/api/serializers.py b/wger/nutrition/api/serializers.py index d3e31ae78..301e3ad93 100644 --- a/wger/nutrition/api/serializers.py +++ b/wger/nutrition/api/serializers.py @@ -124,7 +124,7 @@ class Meta: 'carbohydrates_sugar', 'fat', 'fat_saturated', - 'fibres', + 'fiber', 'sodium', 'license', 'license_title', @@ -160,7 +160,7 @@ class Meta: 'carbohydrates_sugar', 'fat', 'fat_saturated', - 'fibres', + 'fiber', 'sodium', 'weight_units', 'language', @@ -265,7 +265,7 @@ class NutritionalValuesSerializer(serializers.Serializer): carbohydrates_sugar = serializers.FloatField() fat = serializers.FloatField() fat_saturated = serializers.FloatField() - fibres = serializers.FloatField() + fiber = serializers.FloatField() sodium = serializers.FloatField() @@ -312,7 +312,7 @@ class Meta: 'goal_protein', 'goal_carbohydrates', 'goal_fat', - 'goal_fibers', + 'goal_fiber', # 'nutritional_values', ] @@ -336,6 +336,6 @@ class Meta: 'goal_protein', 'goal_carbohydrates', 'goal_fat', - 'goal_fibers', + 'goal_fiber', 'meals', ] diff --git a/wger/nutrition/api/views.py b/wger/nutrition/api/views.py index 7b0ac078d..c7a7d6ff9 100644 --- a/wger/nutrition/api/views.py +++ b/wger/nutrition/api/views.py @@ -129,7 +129,7 @@ def get_values(self, request, pk): 'carbohydrates_sugar': 0, 'fat': 0, 'fat_saturated': 0, - 'fibres': 0, + 'fiber': 0, 'sodium': 0, 'errors': [], } diff --git a/wger/nutrition/fixtures/test-ingredients.json b/wger/nutrition/fixtures/test-ingredients.json index d4152ebef..3e019d812 100644 --- a/wger/nutrition/fixtures/test-ingredients.json +++ b/wger/nutrition/fixtures/test-ingredients.json @@ -15,7 +15,7 @@ "fat": "8.19", "carbohydrates_sugar": "0.0", "fat_saturated": "3.244", - "fibres": "0.0", + "fiber": "0.0", "protein": "25.63", "carbohydrates": "0.1251", "code": "1234567890987654321", @@ -42,7 +42,7 @@ "fat": "9", "carbohydrates_sugar": "1", "fat_saturated": "3", - "fibres": "5.0", + "fiber": "5.0", "protein": "25.63", "carbohydrates": "0.1251", "code": "1234567890", @@ -69,7 +69,7 @@ "fat": "9", "carbohydrates_sugar": "1", "fat_saturated": "3", - "fibres": "5.0", + "fiber": "5.0", "protein": "25.63", "carbohydrates": "0.1551", "code": "1223334444", @@ -96,7 +96,7 @@ "fat": "1", "carbohydrates_sugar": "99", "fat_saturated": "0", - "fibres": "0", + "fiber": "0", "protein": "0", "carbohydrates": "99.0", "code": "0987654321", @@ -123,7 +123,7 @@ "fat": "1", "carbohydrates_sugar": "4", "fat_saturated": "0", - "fibres": "0", + "fiber": "0", "protein": "0", "carbohydrates": "5.0", "code": "555555555", @@ -150,7 +150,7 @@ "fat": "2", "carbohydrates_sugar": "4", "fat_saturated": "0", - "fibres": "20", + "fiber": "20", "protein": "10", "carbohydrates": "5.0", "code": "1112222333333334444444", @@ -177,7 +177,7 @@ "fat": "2", "carbohydrates_sugar": "4", "fat_saturated": "0", - "fibres": "20", + "fiber": "20", "protein": "10", "carbohydrates": "5.0", "code": "11222222333", @@ -204,7 +204,7 @@ "fat": "2", "carbohydrates_sugar": "4", "fat_saturated": "0", - "fibres": "20", + "fiber": "20", "protein": "10", "carbohydrates": "5.0", "code": "9988877766655", @@ -231,7 +231,7 @@ "fat": "2", "carbohydrates_sugar": "4", "fat_saturated": "0", - "fibres": "20", + "fiber": "20", "protein": "10", "carbohydrates": "5.0", "code": "123454321", @@ -258,7 +258,7 @@ "fat": "2", "carbohydrates_sugar": "4", "fat_saturated": "0", - "fibres": "20", + "fiber": "20", "protein": "10", "carbohydrates": "5.0", "code": "0009999888777", @@ -285,7 +285,7 @@ "fat": "2", "carbohydrates_sugar": "4", "fat_saturated": "0", - "fibres": "20", + "fiber": "20", "protein": "10", "carbohydrates": "5.0", "code": "7787878787878787", @@ -312,7 +312,7 @@ "fat": "2", "carbohydrates_sugar": "4", "fat_saturated": "0", - "fibres": "20", + "fiber": "20", "protein": "10", "carbohydrates": "5.0", "code": "29292929292929", @@ -339,7 +339,7 @@ "fat": "2", "carbohydrates_sugar": "4", "fat_saturated": "0", - "fibres": "20", + "fiber": "20", "protein": "10", "carbohydrates": "5.0", "code": "19191919191", @@ -366,7 +366,7 @@ "fat": "2", "carbohydrates_sugar": "4", "fat_saturated": "0", - "fibres": "20", + "fiber": "20", "protein": "10", "carbohydrates": "5.0", "code": "4444444444444", diff --git a/wger/nutrition/forms.py b/wger/nutrition/forms.py index 85e4df1c7..976e5bac5 100644 --- a/wger/nutrition/forms.py +++ b/wger/nutrition/forms.py @@ -343,7 +343,7 @@ class Meta: 'carbohydrates_sugar', 'fat', 'fat_saturated', - 'fibres', + 'fiber', 'sodium', 'license', 'license_author', @@ -371,7 +371,7 @@ def __init__(self, *args, **kwargs): Column('fat_saturated', css_class='col-6'), css_class='form-row', ), - 'fibres', + 'fiber', 'sodium', Row( Column('license', css_class='col-6'), diff --git a/wger/nutrition/helpers.py b/wger/nutrition/helpers.py index 39d3879d4..634b6121b 100644 --- a/wger/nutrition/helpers.py +++ b/wger/nutrition/helpers.py @@ -74,8 +74,8 @@ def get_nutritional_values(self, use_metric=True): if self.ingredient.fat_saturated: values.fat_saturated = self.ingredient.fat_saturated * item_weight / 100 - if self.ingredient.fibres: - values.fibres = self.ingredient.fibres * item_weight / 100 + if self.ingredient.fiber: + values.fiber = self.ingredient.fiber * item_weight / 100 if self.ingredient.sodium: values.sodium = self.ingredient.sodium * item_weight / 100 @@ -110,7 +110,7 @@ class NutritionalValues: carbohydrates_sugar: Union[Decimal, int, float, None] = None fat: Union[Decimal, int, float] = 0 fat_saturated: Union[Decimal, int, float, None] = None - fibres: Union[Decimal, int, float, None] = None + fiber: Union[Decimal, int, float, None] = None sodium: Union[Decimal, int, float, None] = None @property @@ -132,9 +132,9 @@ def __add__(self, other: 'NutritionalValues'): fat_saturated=self.fat_saturated + other.fat_saturated if self.fat_saturated and other.fat_saturated else self.fat_saturated or other.fat_saturated, - fibres=self.fibres + other.fibres - if self.fibres and other.fibres - else self.fibres or other.fibres, + fiber=self.fiber + other.fiber + if self.fiber and other.fiber + else self.fiber or other.fiber, sodium=self.sodium + other.sodium if self.sodium and other.sodium else self.sodium or other.sodium, diff --git a/wger/nutrition/migrations/0022_fiber_spelling.py b/wger/nutrition/migrations/0022_fiber_spelling.py new file mode 100644 index 000000000..ad28bd083 --- /dev/null +++ b/wger/nutrition/migrations/0022_fiber_spelling.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.6 on 2024-05-21 19:32 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nutrition', '0021_add_fibers_field'), + ] + + operations = [ + migrations.RemoveField( + model_name='ingredient', + name='fibres', + ), + migrations.AddField( + model_name='ingredient', + name='fiber', + field=models.DecimalField(blank=True, decimal_places=3, help_text='In g per 100g of product', max_digits=6, null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(100)], verbose_name='Fiber'), + ), + ] diff --git a/wger/nutrition/migrations/0023_rename_goal_fibers_nutritionplan_goal_fiber.py b/wger/nutrition/migrations/0023_rename_goal_fibers_nutritionplan_goal_fiber.py new file mode 100644 index 000000000..8a26f5140 --- /dev/null +++ b/wger/nutrition/migrations/0023_rename_goal_fibers_nutritionplan_goal_fiber.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.6 on 2024-05-21 19:40 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('nutrition', '0022_fiber_spelling'), + ] + + operations = [ + migrations.RenameField( + model_name='nutritionplan', + old_name='goal_fibers', + new_name='goal_fiber', + ), + ] diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.py index 252bbf085..b2c9fe2be 100644 --- a/wger/nutrition/models/ingredient.py +++ b/wger/nutrition/models/ingredient.py @@ -171,12 +171,12 @@ class Ingredient(AbstractSubmissionModel, AbstractLicenseModel, models.Model): validators=[MinValueValidator(0), MaxValueValidator(100)], ) - fibres = models.DecimalField( + fiber = models.DecimalField( decimal_places=3, max_digits=6, blank=True, null=True, - verbose_name=_('Fibres'), + verbose_name=_('Fiber'), help_text=_('In g per 100g of product'), validators=[MinValueValidator(0), MaxValueValidator(100)], ) @@ -339,7 +339,7 @@ def __eq__(self, other): 'energy', 'fat', 'fat_saturated', - 'fibres', + 'fiber', 'name', 'protein', 'sodium', diff --git a/wger/nutrition/models/plan.py b/wger/nutrition/models/plan.py index 257c56f18..8cea374fb 100644 --- a/wger/nutrition/models/plan.py +++ b/wger/nutrition/models/plan.py @@ -82,7 +82,7 @@ class Meta: goal_carbohydrates = models.IntegerField(null=True, default=None) - goal_fibers = models.IntegerField(null=True, default=None) + goal_fiber = models.IntegerField(null=True, default=None) goal_fat = models.IntegerField(null=True, default=None) diff --git a/wger/nutrition/off.py b/wger/nutrition/off.py index 9e99f141c..aac8307ad 100644 --- a/wger/nutrition/off.py +++ b/wger/nutrition/off.py @@ -49,7 +49,7 @@ class IngredientData: carbohydrates_sugar: float fat: float fat_saturated: float - fibres: Optional[float] + fiber: Optional[float] sodium: Optional[float] code: str source_name: str @@ -101,7 +101,7 @@ def extract_info_from_off(product_data, language: int): # these are optional sodium = product_data['nutriments'].get('sodium_100g', None) - fibre = product_data['nutriments'].get('fiber_100g', None) + fiber = product_data['nutriments'].get('fiber_100g', None) brand = product_data.get('brands', None) # License and author info @@ -119,7 +119,7 @@ def extract_info_from_off(product_data, language: int): carbohydrates_sugar=sugars, fat=fat, fat_saturated=saturated, - fibres=fibre, + fiber=fiber, sodium=sodium, code=code, source_name=source_name, diff --git a/wger/nutrition/static/js/nutrition.js b/wger/nutrition/static/js/nutrition.js index 888417729..4efe2f2d9 100644 --- a/wger/nutrition/static/js/nutrition.js +++ b/wger/nutrition/static/js/nutrition.js @@ -39,7 +39,7 @@ function updateIngredientValue(url) { $('#value-carbohydrates-sugar').html(parseFloat(data.carbohydrates_sugar).toFixed(2)); $('#value-fat').html(parseFloat(data.fat).toFixed(2)); $('#value-fat-saturated').html(parseFloat(data.fat_saturated).toFixed(2)); - $('#value-fibres').html(parseFloat(data.fibres).toFixed(2)); + $('#value-fiber').html(parseFloat(data.fiber).toFixed(2)); $('#value-sodium').html(parseFloat(data.sodium).toFixed(2)); }); } diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.py index 3f3f491df..3a0631afc 100644 --- a/wger/nutrition/sync.py +++ b/wger/nutrition/sync.py @@ -226,7 +226,7 @@ def sync_ingredients( 'fat': data['fat'], 'fat_saturated': data['fat_saturated'], 'protein': data['protein'], - 'fibres': data['fibres'], + 'fiber': data['fiber'], 'sodium': data['sodium'], }, ) diff --git a/wger/nutrition/templates/ingredient/view.html b/wger/nutrition/templates/ingredient/view.html index 27a9d03f5..3adc780f9 100644 --- a/wger/nutrition/templates/ingredient/view.html +++ b/wger/nutrition/templates/ingredient/view.html @@ -131,10 +131,10 @@

{% translate "Ingredient is pending review" %}

- {% translate "Fibres" %} + {% translate "Fiber" %} - {% if ingredient.fibres %} - {{ingredient.fibres|floatformat:1}} {% translate "g" context "weight unit, i.e. grams" %} + {% if ingredient.fiber %} + {{ingredient.fiber|floatformat:1}} {% translate "g" context "weight unit, i.e. grams" %} {% else %} {% translate "n.A." %} {% endif %} diff --git a/wger/nutrition/tests/test_ingredient.py b/wger/nutrition/tests/test_ingredient.py index 51afabd4d..b9f3362a1 100644 --- a/wger/nutrition/tests/test_ingredient.py +++ b/wger/nutrition/tests/test_ingredient.py @@ -85,7 +85,7 @@ class EditIngredientTestCase(WgerEditTestCase): 'fat': 10, 'carbohydrates_sugar': 5, 'fat_saturated': 3.14, - 'fibres': 2.1, + 'fiber': 2.1, 'protein': 20, 'carbohydrates': 10, 'license': 2, @@ -119,7 +119,7 @@ class AddIngredientTestCase(WgerAddTestCase): 'fat': 10, 'carbohydrates_sugar': 5, 'fat_saturated': 3.14, - 'fibres': 2.1, + 'fiber': 2.1, 'protein': 20, 'carbohydrates': 10, 'license': 2, @@ -154,7 +154,7 @@ class IngredientNameShortTestCase(WgerTestCase): 'fat': 10, 'carbohydrates_sugar': 5, 'fat_saturated': 3.14, - 'fibres': 2.1, + 'fiber': 2.1, 'protein': 20, 'carbohydrates': 10, 'license': 2, @@ -324,7 +324,7 @@ def calculate_value(self): 'fat': 0.0819, 'carbohydrates_sugar': None, 'fat_saturated': 0.03244, - 'fibres': None, + 'fiber': None, 'protein': 0.2563, 'carbohydrates': 0.00125, }, @@ -348,7 +348,7 @@ def calculate_value(self): 'fat': 9.13185, 'carbohydrates_sugar': None, 'fat_saturated': 3.61706, - 'fibres': None, + 'fiber': None, 'protein': 28.57745, 'carbohydrates': 0.139375, }, @@ -506,7 +506,7 @@ def test_fetch_from_off_success(self, mock_api): self.assertEqual(ingredient.fat, 40) self.assertEqual(ingredient.fat_saturated, 11) self.assertEqual(ingredient.sodium, 5) - self.assertEqual(ingredient.fibres, None) + self.assertEqual(ingredient.fiber, None) self.assertEqual(ingredient.brand, 'The bar company') self.assertEqual(ingredient.license_author, 'open food facts, MrX') diff --git a/wger/nutrition/tests/test_ingredient_overview.py b/wger/nutrition/tests/test_ingredient_overview.py index 449144936..59f152814 100644 --- a/wger/nutrition/tests/test_ingredient_overview.py +++ b/wger/nutrition/tests/test_ingredient_overview.py @@ -37,7 +37,7 @@ def test_overview(self): 'fat': 8.19, 'carbohydrates_sugar': 0.0, 'fat_saturated': 3.24, - 'fibres': 0.0, + 'fiber': 0.0, 'protein': 25.63, 'carbohydrates': 0.0, 'license': 1, diff --git a/wger/nutrition/tests/test_nutritional_calculations.py b/wger/nutrition/tests/test_nutritional_calculations.py index c2244daf6..0bcc745fa 100644 --- a/wger/nutrition/tests/test_nutritional_calculations.py +++ b/wger/nutrition/tests/test_nutritional_calculations.py @@ -60,7 +60,7 @@ def test_calculations(self): self.assertAlmostEqual(item_values.fat, ingredient.fat, 2) self.assertAlmostEqual(item_values.fat_saturated, ingredient.fat_saturated, 2) self.assertAlmostEqual(item_values.sodium, ingredient.sodium, 2) - self.assertAlmostEqual(item_values.fibres, None, 2) + self.assertAlmostEqual(item_values.fiber, None, 2) meal_nutritional_values = meal.get_nutritional_values() self.assertEqual(item_values, meal_nutritional_values) diff --git a/wger/nutrition/tests/test_nutritional_values.py b/wger/nutrition/tests/test_nutritional_values.py index 903774ede..c92ba6866 100644 --- a/wger/nutrition/tests/test_nutritional_values.py +++ b/wger/nutrition/tests/test_nutritional_values.py @@ -40,7 +40,7 @@ def test_addition(self): carbohydrates_sugar=70, fat=60, fat_saturated=50, - fibres=40, + fiber=40, sodium=30, ) values2 = NutritionalValues( @@ -50,7 +50,7 @@ def test_addition(self): carbohydrates_sugar=7, fat=6, fat_saturated=5, - fibres=4, + fiber=4, sodium=3, ) values3 = values1 + values2 @@ -64,7 +64,7 @@ def test_addition(self): carbohydrates_sugar=77, fat=66, fat_saturated=55, - fibres=44, + fiber=44, sodium=33, ), ) @@ -73,7 +73,7 @@ def test_addition_nullable_values(self): """Test that the addition works correctly for the nullable values""" values1 = NutritionalValues() - values2 = NutritionalValues(carbohydrates_sugar=10, fat_saturated=20, fibres=30, sodium=40) + values2 = NutritionalValues(carbohydrates_sugar=10, fat_saturated=20, fiber=30, sodium=40) values3 = values1 + values2 self.assertEqual( @@ -85,7 +85,7 @@ def test_addition_nullable_values(self): carbohydrates_sugar=10, fat=0, fat_saturated=20, - fibres=30, + fiber=30, sodium=40, ), ) diff --git a/wger/nutrition/tests/test_off.py b/wger/nutrition/tests/test_off.py index de652c26e..d2d7e8171 100644 --- a/wger/nutrition/tests/test_off.py +++ b/wger/nutrition/tests/test_off.py @@ -67,7 +67,7 @@ def test_regular_response(self): carbohydrates_sugar=30, fat=40, fat_saturated=11, - fibres=None, + fiber=None, sodium=5, code='1234', source_name='Open Food Facts', diff --git a/wger/nutrition/tests/test_sync.py b/wger/nutrition/tests/test_sync.py index 0609ec802..a53f9ac30 100644 --- a/wger/nutrition/tests/test_sync.py +++ b/wger/nutrition/tests/test_sync.py @@ -50,7 +50,7 @@ def json(): "carbohydrates_sugar": "27.000", "fat": "18.000", "fat_saturated": "4.500", - "fibres": "2.000", + "fiber": "2.000", "sodium": "0.356", "license": 5, "license_title": " Gâteau double chocolat ", @@ -73,7 +73,7 @@ def json(): "carbohydrates_sugar": "5.600", "fat": "11.000", "fat_saturated": "4.600", - "fibres": None, + "fiber": None, "sodium": "0.820", "license": 5, "license_title": " Maxi Hot Dog New York Style", diff --git a/wger/nutrition/views/plan.py b/wger/nutrition/views/plan.py index 86af162e7..cbdb7d30a 100644 --- a/wger/nutrition/views/plan.py +++ b/wger/nutrition/views/plan.py @@ -276,8 +276,8 @@ def export_pdf(request, id: int): ) data.append( [ - Paragraph(_('Fibres'), styleSheet['Normal']), - Paragraph(str(plan_data['total'].fibres), styleSheet['Normal']), + Paragraph(_('Fiber'), styleSheet['Normal']), + Paragraph(str(plan_data['total'].fiber), styleSheet['Normal']), ] ) data.append( @@ -299,7 +299,7 @@ def export_pdf(request, id: int): table_style.append(('BACKGROUND', (0, 6), (-1, 6), row_color)) # Fats table_style.append(('SPAN', (1, 7), (-1, 7))) # Saturated fats table_style.append(('LEFTPADDING', (0, 7), (0, 7), 15)) - table_style.append(('SPAN', (1, 8), (-1, 8))) # Fibres + table_style.append(('SPAN', (1, 8), (-1, 8))) # Fiber table_style.append(('SPAN', (1, 9), (-1, 9))) # Sodium t = Table(data, style=table_style) t._argW[0] = 6 * cm From dc1a4d1d83c92295cb85d09f5c4cf7b3ec46afec Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Tue, 21 May 2024 22:23:31 +0200 Subject: [PATCH 17/34] Some products have empty names, skip them --- wger/nutrition/off.py | 7 +++---- wger/nutrition/usda.py | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/wger/nutrition/off.py b/wger/nutrition/off.py index 9423fa216..57ac8be2e 100644 --- a/wger/nutrition/off.py +++ b/wger/nutrition/off.py @@ -22,7 +22,6 @@ from wger.utils.constants import ODBL_LICENSE_ID from wger.utils.models import AbstractSubmissionModel - OFF_REQUIRED_TOP_LEVEL = [ 'product_name', 'code', @@ -43,9 +42,9 @@ def extract_info_from_off(product_data: dict, language: int) -> IngredientData: raise KeyError('Missing required nutrition key') # Basics - name = product_data['product_name'] - if name is None: - raise KeyError('Product name is None') + name = product_data.get('product_name') + if not name: + raise KeyError('Product name is empty') if len(name) > 200: name = name[:200] diff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.py index 2c360fb1d..1683bdacb 100644 --- a/wger/nutrition/usda.py +++ b/wger/nutrition/usda.py @@ -92,6 +92,8 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: raise ValueError(f'Value for macronutrient is greater than 100! {macros=} {remote_id=}') name = product_data['description'].title() + if not name: + raise ValueError(f'Name is empty! {remote_id=}') if len(name) > 200: name = name[:200] From 9aa837ffa03b5f4f7ea40521b493544917333a6b Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Tue, 21 May 2024 22:57:24 +0200 Subject: [PATCH 18/34] Further increase the author field length --- .../0030_increase_author_field_length.py | 16 ++++++++-------- ...add_remote_id_increase_author_field_length.py | 4 ++-- wger/utils/models.py | 3 +-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/wger/exercises/migrations/0030_increase_author_field_length.py b/wger/exercises/migrations/0030_increase_author_field_length.py index 45aa68c45..6b84e2183 100644 --- a/wger/exercises/migrations/0030_increase_author_field_length.py +++ b/wger/exercises/migrations/0030_increase_author_field_length.py @@ -15,7 +15,7 @@ class Migration(migrations.Migration): field=models.CharField( blank=True, help_text='If you are not the author, enter the name or source here.', - max_length=2500, + max_length=3500, null=True, verbose_name='Author(s)', ), @@ -26,7 +26,7 @@ class Migration(migrations.Migration): field=models.CharField( blank=True, help_text='If you are not the author, enter the name or source here.', - max_length=2500, + max_length=3500, null=True, verbose_name='Author(s)', ), @@ -37,7 +37,7 @@ class Migration(migrations.Migration): field=models.CharField( blank=True, help_text='If you are not the author, enter the name or source here.', - max_length=2500, + max_length=3500, null=True, verbose_name='Author(s)', ), @@ -48,7 +48,7 @@ class Migration(migrations.Migration): field=models.CharField( blank=True, help_text='If you are not the author, enter the name or source here.', - max_length=2500, + max_length=3500, null=True, verbose_name='Author(s)', ), @@ -59,7 +59,7 @@ class Migration(migrations.Migration): field=models.CharField( blank=True, help_text='If you are not the author, enter the name or source here.', - max_length=2500, + max_length=3500, null=True, verbose_name='Author(s)', ), @@ -70,7 +70,7 @@ class Migration(migrations.Migration): field=models.CharField( blank=True, help_text='If you are not the author, enter the name or source here.', - max_length=2500, + max_length=3500, null=True, verbose_name='Author(s)', ), @@ -81,7 +81,7 @@ class Migration(migrations.Migration): field=models.CharField( blank=True, help_text='If you are not the author, enter the name or source here.', - max_length=2500, + max_length=3500, null=True, verbose_name='Author(s)', ), @@ -92,7 +92,7 @@ class Migration(migrations.Migration): field=models.CharField( blank=True, help_text='If you are not the author, enter the name or source here.', - max_length=2500, + max_length=3500, null=True, verbose_name='Author(s)', ), diff --git a/wger/nutrition/migrations/0022_add_remote_id_increase_author_field_length.py b/wger/nutrition/migrations/0022_add_remote_id_increase_author_field_length.py index 8d1992aee..85b98cf5d 100644 --- a/wger/nutrition/migrations/0022_add_remote_id_increase_author_field_length.py +++ b/wger/nutrition/migrations/0022_add_remote_id_increase_author_field_length.py @@ -35,7 +35,7 @@ class Migration(migrations.Migration): field=models.CharField( blank=True, help_text='If you are not the author, enter the name or source here.', - max_length=2500, + max_length=3500, null=True, verbose_name='Author(s)', ), @@ -46,7 +46,7 @@ class Migration(migrations.Migration): field=models.CharField( blank=True, help_text='If you are not the author, enter the name or source here.', - max_length=2500, + max_length=3500, null=True, verbose_name='Author(s)', ), diff --git a/wger/utils/models.py b/wger/utils/models.py index 5e7fe89bc..699560924 100644 --- a/wger/utils/models.py +++ b/wger/utils/models.py @@ -20,7 +20,6 @@ from wger.core.models import License from wger.utils.constants import CC_BY_SA_4_ID - """ Abstract model classes """ @@ -61,7 +60,7 @@ class Meta: license_author = models.CharField( verbose_name=_('Author(s)'), - max_length=2500, + max_length=3500, blank=True, null=True, help_text=_('If you are not the author, enter the name or source here.'), From e4a6f7ed169a4860dfc862f997f13f9d0e13e860 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Tue, 21 May 2024 23:11:21 +0200 Subject: [PATCH 19/34] Increase number of returned results in the API ingredient search --- wger/nutrition/api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wger/nutrition/api/views.py b/wger/nutrition/api/views.py index 7b0ac078d..ee5e639f8 100644 --- a/wger/nutrition/api/views.py +++ b/wger/nutrition/api/views.py @@ -232,7 +232,7 @@ def search(request): else: query = query.filter(name__icontains=term) - for ingredient in query[:100]: + for ingredient in query[:150]: if hasattr(ingredient, 'image'): image_obj = ingredient.image image = image_obj.image.url From 3dbeb62777bdd4ed91ef222a7e75c68fd9831bca Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Wed, 22 May 2024 07:43:29 +0200 Subject: [PATCH 20/34] Refactor import scripts Now it is possible to import the OFF dumps or delta files directly from the JSONL files, without needing to import them to mongoDB --- .../commands/import-off-products.py | 132 ++++++++++++------ .../commands/import-usda-products.py | 60 ++------ wger/nutrition/management/products.py | 81 ++++++++++- wger/nutrition/off.py | 1 + wger/utils/models.py | 1 + 5 files changed, 179 insertions(+), 96 deletions(-) diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.py index e76d4af05..1e8eed13b 100644 --- a/wger/nutrition/management/commands/import-off-products.py +++ b/wger/nutrition/management/commands/import-off-products.py @@ -16,6 +16,10 @@ import logging import os +# Third Party +import requests +from pymongo import MongoClient + # wger from wger.core.models import Language from wger.nutrition.management.products import ( @@ -42,39 +46,87 @@ def add_arguments(self, parser): '--jsonl', action='store_true', default=False, - dest='usejsonl', + dest='use_jsonl', help='Use the JSONL dump of the Open Food Facts database.' - '(this option does not require mongo)' + '(this option does not require mongo)', ) - def products_jsonl(self,languages,completeness): - import json - import requests - from gzip import GzipFile - off_url='https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz' - download_dir=os.path.expanduser('~/.cache/off_cache') - os.makedirs(download_dir,exist_ok=True) - gzipdb=os.path.join(download_dir,os.path.basename(off_url)) - if os.path.exists(gzipdb): - self.stdout.write(f'Already downloaded {gzipdb}, skipping download') - else: - self.stdout.write(f'downloading {gzipdb}... (this may take a while)') - req=requests.get(off_url,stream=True) - with open(gzipdb,'wb') as fid: - for chunk in req.iter_content(chunk_size=50*1024): - fid.write(chunk) - with GzipFile(gzipdb,'rb') as gzid: - for line in gzid: - try: - product=json.loads(line) - if product['completeness'] < completeness: - continue - if not product['lang'] in languages: - continue - yield product - except: - self.stdout.write(f' Error parsing and/or filtering json record, skipping') - continue + parser.add_argument( + '--delta-updates', + action='store_true', + default=False, + dest='delta_updates', + help='Downloads and imports the most recent delta file', + ) + + def import_mongo(self, languages: dict[str:int]): + client = MongoClient('mongodb://off:off-wger@127.0.0.1', port=27017) + db = client.admin + for product in db.products.find({'lang': {'$in': list(languages.keys())}}): + try: + ingredient_data = extract_info_from_off(product, languages[product['lang']]) + except KeyError as e: + # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}') + # self.stdout.write(repr(e)) + # pprint(product) + self.counter['skipped'] += 1 + else: + self.process_ingredient(ingredient_data) + + def import_daily_delta(self, languages: dict[str:int]): + download_folder, tmp_folder = self.get_download_folder( + '/Users/roland/Entwicklung/wger/server/extras/usda' + ) + + base_url = 'https://static.openfoodfacts.org/data/delta/' + + # Fetch the index page with requests and read the result + index_url = base_url + 'index.txt' + req = requests.get(index_url) + index_content = req.text + newest_entry = index_content.split('\n')[0] + + file_path = os.path.join(download_folder, newest_entry) + + # Fetch the newest entry and extract the contents + delta_url = base_url + newest_entry + self.download_file(delta_url, file_path) + + for entry in self.iterate_gz_file_contents(file_path, list(languages.keys())): + try: + ingredient_data = extract_info_from_off(entry, languages[entry['lang']]) + except KeyError as e: + # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}') + self.counter['skipped'] += 1 + else: + self.process_ingredient(ingredient_data) + + if tmp_folder: + self.stdout.write(f'Removing temporary folder {download_folder}') + tmp_folder.cleanup() + + def import_full_dump(self, languages: dict[str:int]): + off_url = 'https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz' + + download_folder, tmp_folder = self.get_download_folder( + '/Users/roland/Entwicklung/wger/server/extras/open-food-facts/dump' + ) + + file_path = os.path.join(download_folder, os.path.basename(off_url)) + self.download_file(off_url, file_path) + + for entry in self.iterate_gz_file_contents(file_path, list(languages.keys())): + try: + ingredient_data = extract_info_from_off(entry, languages[entry['lang']]) + except KeyError as e: + # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}') + self.counter['skipped'] += 1 + else: + self.process_ingredient(ingredient_data) + + if tmp_folder: + self.stdout.write(f'Removing temporary folder {download_folder}') + tmp_folder.cleanup() def handle(self, **options): try: @@ -91,21 +143,13 @@ def handle(self, **options): self.stdout.write(f' - {self.mode}') self.stdout.write('') - client = MongoClient('mongodb://off:off-wger@127.0.0.1', port=27017) - db = client.admin - languages = {lang.short_name: lang.pk for lang in Language.objects.all()} - - for product in db.products.find({'lang': {'$in': list(languages.keys())}}): - try: - ingredient_data = extract_info_from_off(product, languages[product['lang']]) - except KeyError as e: - # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}') - # self.stdout.write(repr(e)) - # pprint(product) - self.counter['skipped'] += 1 - else: - self.handle_data(ingredient_data) + if options['delta_updates']: + self.import_daily_delta(languages) + elif options['use_jsonl']: + self.import_full_dump(languages) + else: + self.import_mongo(languages) self.stdout.write(self.style.SUCCESS('Finished!')) self.stdout.write(self.style.SUCCESS(str(self.counter))) diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py index 831dd55ba..160c9b96b 100644 --- a/wger/nutrition/management/commands/import-usda-products.py +++ b/wger/nutrition/management/commands/import-usda-products.py @@ -16,13 +16,9 @@ import json import logging import os -import tempfile from json import JSONDecodeError from zipfile import ZipFile -# Third Party -import requests - # wger from wger.core.models import Language from wger.nutrition.dataclasses import IngredientData @@ -33,7 +29,6 @@ from wger.nutrition.models import Ingredient from wger.nutrition.usda import extract_info_from_usda from wger.utils.constants import ENGLISH_SHORT_NAME -from wger.utils.requests import wger_headers logger = logging.getLogger(__name__) @@ -47,18 +42,6 @@ class Command(ImportProductCommand): def add_arguments(self, parser): super().add_arguments(parser) - parser.add_argument( - '--folder', - action='store', - default='', - dest='tmp_folder', - type=str, - help='Controls whether to use a temporary folder created by python (the default) or ' - 'the path provided for storing the downloaded dataset. If there are already ' - 'downloaded or extracted files here, they will be used instead of fetching them ' - 'again.', - ) - parser.add_argument( '--dataset', action='store', @@ -66,55 +49,31 @@ def add_arguments(self, parser): dest='dataset', type=str, help='What dataset to download, this value will be appended to ' - '"https://fdc.nal.usda.gov/fdc-datasets/". Consult ' - 'https://fdc.nal.usda.gov/download-datasets.html for current file names', + '"https://fdc.nal.usda.gov/fdc-datasets/". Consult ' + 'https://fdc.nal.usda.gov/download-datasets.html for current file names', ) def handle(self, **options): if options['mode'] == 'insert': self.mode = Mode.INSERT - usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{options["dataset"]}' - - if options['folder']: - download_folder = options['folder'] - - # Check whether the folder exists - if not os.path.exists(download_folder): - self.stdout.write(self.style.ERROR(f'Folder {download_folder} does not exist!')) - return - else: - tmp_folder = tempfile.TemporaryDirectory() - download_folder = tmp_folder.name + dataset_url = f'https://fdc.nal.usda.gov/fdc-datasets/{options["dataset"]}' + download_folder, tmp_folder = self.get_download_folder(options['folder']) self.stdout.write('Importing entries from USDA') self.stdout.write(f' - {self.mode}') - self.stdout.write(f' - dataset: {usda_url}') + self.stdout.write(f' - dataset: {dataset_url}') self.stdout.write(f' - download folder: {download_folder}') self.stdout.write('') english = Language.objects.get(short_name=ENGLISH_SHORT_NAME) # Download the dataset - zip_file = os.path.join(download_folder, options['dataset']) - if os.path.exists(zip_file): - self.stdout.write(f'File already downloaded {zip_file}, not downloading it again') - else: - self.stdout.write(f'Downloading {usda_url}... (this may take a while)') - req = requests.get(usda_url, stream=True, headers=wger_headers()) - - if req.status_code == 404: - self.stdout.write(self.style.ERROR(f'Could not open {usda_url}!')) - return - - with open(zip_file, 'wb') as fid: - for chunk in req.iter_content(chunk_size=50 * 1024): - fid.write(chunk) - - self.stdout.write('download successful') + zip_file_path = os.path.join(download_folder, options['dataset']) + self.download_file(dataset_url, zip_file_path) # Extract the first file from the ZIP archive - with ZipFile(zip_file, 'r') as zip_ref: + with ZipFile(zip_file_path, 'r') as zip_ref: file_list = zip_ref.namelist() if not file_list: raise Exception('No files found in the ZIP archive') @@ -154,11 +113,10 @@ def handle(self, **options): self.stdout.write(f'--> ValueError while extracting ingredient info: {e}') self.counter['skipped'] += 1 else: - # pass self.match_existing_entry(ingredient_data) # self.handle_data(ingredient_data) - if not options['folder']: + if tmp_folder: self.stdout.write(f'Removing temporary folder {download_folder}') tmp_folder.cleanup() diff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.py index 50965e549..fa8205f13 100644 --- a/wger/nutrition/management/products.py +++ b/wger/nutrition/management/products.py @@ -14,15 +14,26 @@ # Standard Library import enum +import json import logging +import os +import tempfile from collections import Counter +from gzip import GzipFile +from json import JSONDecodeError +from typing import Optional # Django from django.core.management.base import BaseCommand +# Third Party +import requests +from tqdm import tqdm + # wger from wger.nutrition.dataclasses import IngredientData from wger.nutrition.models import Ingredient +from wger.utils.requests import wger_headers logger = logging.getLogger(__name__) @@ -66,10 +77,22 @@ def add_arguments(self, parser): 'already present. Default: update', ) + parser.add_argument( + '--folder', + action='store', + default='', + dest='folder', + type=str, + help='Controls whether to use a temporary folder created by python (the default) or ' + 'the path provided for storing the downloaded dataset. If there are already ' + 'downloaded or extracted files here, they will be used instead of fetching them ' + 'again.', + ) + def handle(self, **options): raise NotImplementedError('Do not run this command on its own!') - def handle_data(self, ingredient_data: IngredientData): + def process_ingredient(self, ingredient_data: IngredientData): # # Add entries as new products if self.mode == Mode.INSERT: @@ -120,3 +143,59 @@ def handle_data(self, ingredient_data: IngredientData): self.stdout.write(repr(e)) # self.stdout.write(repr(ingredient_data)) self.counter['error'] += 1 + + def get_download_folder(self, folder: str) -> tuple[str, Optional[tempfile.TemporaryDirectory]]: + if folder: + tmp_folder = None + download_folder = folder + + # Check whether the folder exists + if not os.path.exists(download_folder): + self.stdout.write(self.style.ERROR(f'Folder {download_folder} does not exist!')) + raise Exception('Folder does not exist') + else: + tmp_folder = tempfile.TemporaryDirectory() + download_folder = tmp_folder.name + + return download_folder, tmp_folder + + def parse_file_content(self, path: str): + with GzipFile(path, 'rb') as gzid: + for line in gzid: + try: + product = json.loads(line) + yield product + except JSONDecodeError as e: + self.stdout.write(f' Error parsing and/or filtering json record, skipping') + continue + + def iterate_gz_file_contents(self, path: str, languages: list[str]): + with GzipFile(path, 'rb') as gzid: + for line in gzid: + try: + product = json.loads(line) + if not product['lang'] in languages: + continue + yield product + except JSONDecodeError as e: + self.stdout.write(f' Error parsing and/or filtering json record, skipping') + continue + + def download_file(self, url: str, destination: str) -> None: + if os.path.exists(destination): + self.stdout.write(f'File already downloaded {destination}, not downloading it again') + return + + self.stdout.write(f'Downloading {url}... (this may take a while)') + response = requests.get(url, stream=True, headers=wger_headers()) + total_size = int(response.headers.get('content-length', 0)) + size = int(response.headers['content-length']) / (1024 * 1024) + + if response.status_code == 404: + raise Exception(f'Could not open {url}!') + + with open(destination, 'wb') as fid: + with tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading') as pbar: + for chunk in response.iter_content(chunk_size=50 * 1024): + fid.write(chunk) + pbar.update(len(chunk)) diff --git a/wger/nutrition/off.py b/wger/nutrition/off.py index 57ac8be2e..2b367ee0d 100644 --- a/wger/nutrition/off.py +++ b/wger/nutrition/off.py @@ -22,6 +22,7 @@ from wger.utils.constants import ODBL_LICENSE_ID from wger.utils.models import AbstractSubmissionModel + OFF_REQUIRED_TOP_LEVEL = [ 'product_name', 'code', diff --git a/wger/utils/models.py b/wger/utils/models.py index 21ba70eab..aa6660b2d 100644 --- a/wger/utils/models.py +++ b/wger/utils/models.py @@ -20,6 +20,7 @@ from wger.core.models import License from wger.utils.constants import CC_BY_SA_4_ID + """ Abstract model classes """ From fdca08d4092c2485c5cbe8bf5a81352781e3fa0b Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Wed, 22 May 2024 16:31:22 +0200 Subject: [PATCH 21/34] Make sync of OFF daily delta files a celery task This can run automatically once per day if configured --- extras/docker/development/settings.py | 1 + wger/nutrition/tasks.py | 33 +++++++++++++++++++++++---- wger/settings_global.py | 8 +++---- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/extras/docker/development/settings.py b/extras/docker/development/settings.py index 59f37f0e5..a0be29daa 100644 --- a/extras/docker/development/settings.py +++ b/extras/docker/development/settings.py @@ -99,6 +99,7 @@ WGER_SETTINGS["SYNC_EXERCISE_IMAGES_CELERY"] = env.bool("SYNC_EXERCISE_IMAGES_CELERY", False) WGER_SETTINGS["SYNC_EXERCISE_VIDEOS_CELERY"] = env.bool("SYNC_EXERCISE_VIDEOS_CELERY", False) WGER_SETTINGS["SYNC_INGREDIENTS_CELERY"] = env.bool("SYNC_INGREDIENTS_CELERY", False) +WGER_SETTINGS["SYNC_OFF_DAILY_DELTA_CELERY"] = env.bool("SYNC_OFF_DAILY_DELTA_CELERY", False) WGER_SETTINGS["USE_RECAPTCHA"] = env.bool("USE_RECAPTCHA", False) WGER_SETTINGS["USE_CELERY"] = env.bool("USE_CELERY", False) diff --git a/wger/nutrition/tasks.py b/wger/nutrition/tasks.py index b3182d3b4..f476042d6 100644 --- a/wger/nutrition/tasks.py +++ b/wger/nutrition/tasks.py @@ -14,10 +14,14 @@ # Standard Library import logging -import random +from random import ( + choice, + randint, +) # Django from django.conf import settings +from django.core.management import call_command # Third Party from celery.schedules import crontab @@ -62,15 +66,34 @@ def sync_all_ingredients_task(): sync_ingredients(logger.info) +@app.task +def sync_off_daily_delta(): + """ + Fetches OFF's daily delta product updates + """ + call_command('import-off-products', '--delta-updates') + + @app.on_after_finalize.connect def setup_periodic_tasks(sender, **kwargs): if settings.WGER_SETTINGS['SYNC_INGREDIENTS_CELERY']: sender.add_periodic_task( crontab( - hour=str(random.randint(0, 23)), - minute=str(random.randint(0, 59)), - day_of_month=f'{random.randint(1, 12)},{random.randint(18, 28)}', + hour=str(randint(0, 23)), + minute=str(randint(0, 59)), + day_of_month=str(randint(1, 28)), + month_of_year=choice(['1, 4, 7, 10', '2, 5, 8, 11', '3, 6, 9, 12']), ), sync_all_ingredients_task.s(), - name='Sync exercises', + name='Sync ingredients', + ) + + if settings.WGER_SETTINGS['SYNC_OFF_DAILY_DELTA_CELERY']: + sender.add_periodic_task( + crontab( + hour=str(randint(0, 23)), + minute=str(randint(0, 59)), + ), + sync_off_daily_delta.s(), + name='Sync OFF daily delta updates', ) diff --git a/wger/settings_global.py b/wger/settings_global.py index 2cf80e560..6221c05bb 100644 --- a/wger/settings_global.py +++ b/wger/settings_global.py @@ -171,8 +171,7 @@ 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ], - 'debug': - False + 'debug': False }, }, ] @@ -257,7 +256,7 @@ LANGUAGE_CODE = 'en' # All translation files are in one place -LOCALE_PATHS = (os.path.join(SITE_ROOT, 'locale'), ) +LOCALE_PATHS = (os.path.join(SITE_ROOT, 'locale'),) # Primary keys are AutoFields DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' @@ -498,7 +497,7 @@ # # Ignore these URLs if they cause 404 # -IGNORABLE_404_URLS = (re.compile(r'^/favicon\.ico$'), ) +IGNORABLE_404_URLS = (re.compile(r'^/favicon\.ico$'),) # # Password rules @@ -537,6 +536,7 @@ 'SYNC_EXERCISE_IMAGES_CELERY': False, 'SYNC_EXERCISE_VIDEOS_CELERY': False, 'SYNC_INGREDIENTS_CELERY': False, + 'SYNC_OFF_DAILY_DELTA_CELERY': False, 'TWITTER': False, 'MASTODON': 'https://fosstodon.org/@wger', 'USE_CELERY': False, From 69b53e543c53f44f7033e94eaa6fbe33cb49390e Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Wed, 22 May 2024 16:32:16 +0200 Subject: [PATCH 22/34] Refactor and remove hard coded paths --- .../commands/import-off-products.py | 52 +++++++++---------- wger/nutrition/management/products.py | 37 ++++++------- 2 files changed, 41 insertions(+), 48 deletions(-) diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.py index 1e8eed13b..a841a838c 100644 --- a/wger/nutrition/management/commands/import-off-products.py +++ b/wger/nutrition/management/commands/import-off-products.py @@ -28,7 +28,6 @@ ) from wger.nutrition.off import extract_info_from_off - logger = logging.getLogger(__name__) @@ -39,6 +38,9 @@ class Command(ImportProductCommand): help = 'Import an Open Food Facts dump. Please consult extras/docker/open-food-facts' + deltas_base_url = 'https://static.openfoodfacts.org/data/delta/' + full_off_dump_url = 'https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz' + def add_arguments(self, parser): super().add_arguments(parser) @@ -48,7 +50,7 @@ def add_arguments(self, parser): default=False, dest='use_jsonl', help='Use the JSONL dump of the Open Food Facts database.' - '(this option does not require mongo)', + '(this option does not require mongo)', ) parser.add_argument( @@ -65,23 +67,17 @@ def import_mongo(self, languages: dict[str:int]): for product in db.products.find({'lang': {'$in': list(languages.keys())}}): try: ingredient_data = extract_info_from_off(product, languages[product['lang']]) - except KeyError as e: + except (KeyError, ValueError) as e: # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}') - # self.stdout.write(repr(e)) - # pprint(product) self.counter['skipped'] += 1 else: self.process_ingredient(ingredient_data) - def import_daily_delta(self, languages: dict[str:int]): - download_folder, tmp_folder = self.get_download_folder( - '/Users/roland/Entwicklung/wger/server/extras/usda' - ) - - base_url = 'https://static.openfoodfacts.org/data/delta/' + def import_daily_delta(self, languages: dict[str:int], destination: str): + download_folder, tmp_folder = self.get_download_folder(destination) # Fetch the index page with requests and read the result - index_url = base_url + 'index.txt' + index_url = self.deltas_base_url + 'index.txt' req = requests.get(index_url) index_content = req.text newest_entry = index_content.split('\n')[0] @@ -89,14 +85,16 @@ def import_daily_delta(self, languages: dict[str:int]): file_path = os.path.join(download_folder, newest_entry) # Fetch the newest entry and extract the contents - delta_url = base_url + newest_entry + delta_url = self.deltas_base_url + newest_entry self.download_file(delta_url, file_path) for entry in self.iterate_gz_file_contents(file_path, list(languages.keys())): try: ingredient_data = extract_info_from_off(entry, languages[entry['lang']]) - except KeyError as e: - # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}') + except (KeyError, ValueError) as e: + self.stdout.write( + f'--> {ingredient_data.remote_id=} KeyError while extracting info from OFF: {e}' + ) self.counter['skipped'] += 1 else: self.process_ingredient(ingredient_data) @@ -105,20 +103,16 @@ def import_daily_delta(self, languages: dict[str:int]): self.stdout.write(f'Removing temporary folder {download_folder}') tmp_folder.cleanup() - def import_full_dump(self, languages: dict[str:int]): - off_url = 'https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz' + def import_full_dump(self, languages: dict[str:int], destination: str): + download_folder, tmp_folder = self.get_download_folder(destination) - download_folder, tmp_folder = self.get_download_folder( - '/Users/roland/Entwicklung/wger/server/extras/open-food-facts/dump' - ) - - file_path = os.path.join(download_folder, os.path.basename(off_url)) - self.download_file(off_url, file_path) + file_path = os.path.join(download_folder, os.path.basename(self.full_off_dump_url)) + self.download_file(self.full_off_dump_url, file_path) for entry in self.iterate_gz_file_contents(file_path, list(languages.keys())): try: ingredient_data = extract_info_from_off(entry, languages[entry['lang']]) - except KeyError as e: + except (KeyError, ValueError) as e: # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}') self.counter['skipped'] += 1 else: @@ -141,13 +135,19 @@ def handle(self, **options): self.stdout.write('Importing entries from Open Food Facts') self.stdout.write(f' - {self.mode}') + if options['delta_updates']: + self.stdout.write(f' - importing only delta updates') + elif options['use_jsonl']: + self.stdout.write(f' - importing the full dump') + else: + self.stdout.write(f' - importing from mongo') self.stdout.write('') languages = {lang.short_name: lang.pk for lang in Language.objects.all()} if options['delta_updates']: - self.import_daily_delta(languages) + self.import_daily_delta(languages, options['folder']) elif options['use_jsonl']: - self.import_full_dump(languages) + self.import_full_dump(languages, options['folder']) else: self.import_mongo(languages) diff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.py index fa8205f13..86cc0b611 100644 --- a/wger/nutrition/management/products.py +++ b/wger/nutrition/management/products.py @@ -72,9 +72,11 @@ def add_arguments(self, parser): default='update', dest='mode', type=str, - help='Script mode, "insert" or "update". Insert will insert the ingredients as new ' - 'entries in the database, while update will try to update them if they are ' - 'already present. Default: update', + help=( + 'Script mode, "insert" or "update". Insert will insert the ingredients as new ' + 'entries in the database, while update will try to update them if they are ' + 'already present. Default: update' + ), ) parser.add_argument( @@ -83,10 +85,12 @@ def add_arguments(self, parser): default='', dest='folder', type=str, - help='Controls whether to use a temporary folder created by python (the default) or ' - 'the path provided for storing the downloaded dataset. If there are already ' - 'downloaded or extracted files here, they will be used instead of fetching them ' - 'again.', + help=( + 'Controls whether to use a temporary folder created by python (the default) or ' + 'the path provided for storing the downloaded dataset. If there are already ' + 'downloaded or extracted files here, they will be used instead of fetching them ' + 'again.' + ), ) def handle(self, **options): @@ -139,9 +143,8 @@ def process_ingredient(self, ingredient_data: IngredientData): # self.stdout.write('-> updated') except Exception as e: - self.stdout.write('--> Error while performing update_or_create') - self.stdout.write(repr(e)) - # self.stdout.write(repr(ingredient_data)) + self.stdout.write(f'--> Error while performing update_or_create: {e}') + self.stdout.write(repr(ingredient_data)) self.counter['error'] += 1 def get_download_folder(self, folder: str) -> tuple[str, Optional[tempfile.TemporaryDirectory]]: @@ -157,18 +160,9 @@ def get_download_folder(self, folder: str) -> tuple[str, Optional[tempfile.Tempo tmp_folder = tempfile.TemporaryDirectory() download_folder = tmp_folder.name + self.stdout.write(f'Using folder {download_folder} for storing downloaded files') return download_folder, tmp_folder - def parse_file_content(self, path: str): - with GzipFile(path, 'rb') as gzid: - for line in gzid: - try: - product = json.loads(line) - yield product - except JSONDecodeError as e: - self.stdout.write(f' Error parsing and/or filtering json record, skipping') - continue - def iterate_gz_file_contents(self, path: str, languages: list[str]): with GzipFile(path, 'rb') as gzid: for line in gzid: @@ -183,13 +177,12 @@ def iterate_gz_file_contents(self, path: str, languages: list[str]): def download_file(self, url: str, destination: str) -> None: if os.path.exists(destination): - self.stdout.write(f'File already downloaded {destination}, not downloading it again') + self.stdout.write(f'File already downloaded at {destination}') return self.stdout.write(f'Downloading {url}... (this may take a while)') response = requests.get(url, stream=True, headers=wger_headers()) total_size = int(response.headers.get('content-length', 0)) - size = int(response.headers['content-length']) / (1024 * 1024) if response.status_code == 404: raise Exception(f'Could not open {url}!') From e3aa521f4c33640dc35f6a8770fe1d63a364cb8a Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Wed, 22 May 2024 16:34:01 +0200 Subject: [PATCH 23/34] Move some common data checks to the IngredientData class This is specially important if we regularly import the delta files, since these can have bad quality. We will probably need to add a bit more before putting this live --- wger/__init__.py | 2 +- wger/nutrition/dataclasses.py | 24 ++++++++++++++ .../commands/import-off-products.py | 4 +-- wger/nutrition/off.py | 11 ++----- wger/nutrition/tests/test_usda.py | 10 +++--- wger/nutrition/usda.py | 33 ++++++++----------- 6 files changed, 49 insertions(+), 35 deletions(-) diff --git a/wger/__init__.py b/wger/__init__.py index 9fb6d18ba..e8fc70dc5 100644 --- a/wger/__init__.py +++ b/wger/__init__.py @@ -11,7 +11,7 @@ MIN_APP_VERSION = (1, 6, 0, 'final', 1) -VERSION = (2, 3, 0, 'alpha', 1) +VERSION = (2, 3, 0, 'alpha', 2) RELEASE = True diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.py index b190d5c6f..b1c2b0555 100644 --- a/wger/nutrition/dataclasses.py +++ b/wger/nutrition/dataclasses.py @@ -44,5 +44,29 @@ class IngredientData: license_title: str license_object_url: str + def sanity_checks(self): + if not self.name: + raise ValueError(f'Name is empty!') + self.name = self.name[:200] + self.brand = self.brand[:200] + self.common_name = self.common_name[:200] + + macros = [ + 'protein', + 'fat', + 'fat_saturated', + 'carbohydrates', + 'carbohydrates_sugar', + 'sodium', + 'fibres', + ] + for macro in macros: + value = getattr(self, macro) + if value and value > 100: + raise ValueError(f'Value for {macro} is greater than 100: {value}') + + if self.carbohydrates + self.protein + self.fat > 100: + raise ValueError(f'Total of carbohydrates, protein and fat is greater than 100!') + def dict(self): return asdict(self) diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.py index a841a838c..3ffb6de67 100644 --- a/wger/nutrition/management/commands/import-off-products.py +++ b/wger/nutrition/management/commands/import-off-products.py @@ -49,8 +49,8 @@ def add_arguments(self, parser): action='store_true', default=False, dest='use_jsonl', - help='Use the JSONL dump of the Open Food Facts database.' - '(this option does not require mongo)', + help=('Use the JSONL dump of the Open Food Facts database.' + '(this option does not require mongo)') ) parser.add_argument( diff --git a/wger/nutrition/off.py b/wger/nutrition/off.py index 2b367ee0d..7a52507b0 100644 --- a/wger/nutrition/off.py +++ b/wger/nutrition/off.py @@ -44,14 +44,7 @@ def extract_info_from_off(product_data: dict, language: int) -> IngredientData: # Basics name = product_data.get('product_name') - if not name: - raise KeyError('Product name is empty') - if len(name) > 200: - name = name[:200] - common_name = product_data.get('generic_name', '') - if len(common_name) > 200: - common_name = common_name[:200] # If the energy is not available in kcal, convert from kJ if 'energy-kcal_100g' in product_data['nutriments']: @@ -79,7 +72,7 @@ def extract_info_from_off(product_data: dict, language: int) -> IngredientData: authors = ', '.join(product_data.get('editors_tags', ['open food facts'])) object_url = f'https://world.openfoodfacts.org/product/{code}/' - return IngredientData( + ingredient_data = IngredientData( remote_id=code, name=name, language_id=language, @@ -102,3 +95,5 @@ def extract_info_from_off(product_data: dict, language: int) -> IngredientData: license_title=name, license_object_url=object_url, ) + ingredient_data.sanity_checks() + return ingredient_data diff --git a/wger/nutrition/tests/test_usda.py b/wger/nutrition/tests/test_usda.py index 746228b0d..4fa0fc643 100644 --- a/wger/nutrition/tests/test_usda.py +++ b/wger/nutrition/tests/test_usda.py @@ -19,7 +19,7 @@ # wger from wger.nutrition.dataclasses import IngredientData from wger.nutrition.usda import ( - convert_to_g_per_100g, + convert_to_grams, extract_info_from_usda, ) from wger.utils.constants import CC_0_LICENSE_ID @@ -255,7 +255,7 @@ def test_converting_grams(self): """ entry = {'nutrient': {'unitName': 'g'}, 'amount': '5.0'} - self.assertEqual(convert_to_g_per_100g(entry), 5.0) + self.assertEqual(convert_to_grams(entry), 5.0) def test_converting_milligrams(self): """ @@ -263,7 +263,7 @@ def test_converting_milligrams(self): """ entry = {'nutrient': {'unitName': 'mg'}, 'amount': '5000'} - self.assertEqual(convert_to_g_per_100g(entry), 5.0) + self.assertEqual(convert_to_grams(entry), 5.0) def test_converting_unknown_unit(self): """ @@ -271,7 +271,7 @@ def test_converting_unknown_unit(self): """ entry = {'nutrient': {'unitName': 'kg'}, 'amount': '5.0'} - self.assertRaises(ValueError, convert_to_g_per_100g, entry) + self.assertRaises(ValueError, convert_to_grams, entry) def test_converting_invalid_amount(self): """ @@ -279,4 +279,4 @@ def test_converting_invalid_amount(self): """ entry = {'nutrient': {'unitName': 'g'}, 'amount': 'invalid'} - self.assertRaises(ValueError, convert_to_g_per_100g, entry) + self.assertRaises(ValueError, convert_to_grams, entry) diff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.py index 1683bdacb..ac0f0cb36 100644 --- a/wger/nutrition/usda.py +++ b/wger/nutrition/usda.py @@ -20,9 +20,9 @@ from wger.utils.models import AbstractSubmissionModel -def convert_to_g_per_100g(entry: dict) -> float: +def convert_to_grams(entry: dict) -> float: """ - Convert a nutrient entry to grams per 100g + Convert a nutrient entry to grams """ nutrient = entry['nutrient'] @@ -53,7 +53,7 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: fats_saturated = None sodium = None - fibre = None + fiber = None brand = product_data.get('brandName', '') @@ -63,39 +63,32 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: nutrient_id = nutrient['nutrient']['id'] match nutrient_id: + # in kcal case 1008: - energy = float(nutrient.get('amount')) # in kcal + energy = float(nutrient.get('amount')) case 1003: - protein = convert_to_g_per_100g(nutrient) + protein = convert_to_grams(nutrient) case 1005: - carbs = convert_to_g_per_100g(nutrient) + carbs = convert_to_grams(nutrient) # Total lipid (fat) | Total fat (NLEA) case 1004 | 1085: - fats = convert_to_g_per_100g(nutrient) + fats = convert_to_grams(nutrient) case 1093: - sodium = convert_to_g_per_100g(nutrient) + sodium = convert_to_grams(nutrient) case 1079: - fibre = convert_to_g_per_100g(nutrient) + fiber = convert_to_grams(nutrient) macros = [energy, protein, carbs, fats] for value in macros: if value is None: raise KeyError(f'Could not extract all basic macros: {macros=} {remote_id=}') - for value in [protein, fats, fats_saturated, carbs, carbs_sugars, sodium, fibre]: - if value and value > 100: - raise ValueError(f'Value for macronutrient is greater than 100! {macros=} {remote_id=}') - name = product_data['description'].title() - if not name: - raise ValueError(f'Name is empty! {remote_id=}') - if len(name) > 200: - name = name[:200] # License and author info source_name = Source.USDA.value @@ -106,7 +99,7 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: ) object_url = f'https://fdc.nal.usda.gov/fdc-app.html#/food-details/{remote_id}/nutrients' - return IngredientData( + ingredient_data = IngredientData( code=barcode, remote_id=remote_id, name=name, @@ -118,7 +111,7 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: carbohydrates_sugar=carbs_sugars, fat=fats, fat_saturated=fats_saturated, - fibres=fibre, + fibres=fiber, sodium=sodium, source_name=source_name, source_url=source_url, @@ -129,3 +122,5 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: license_title=name, license_object_url=object_url, ) + ingredient_data.sanity_checks() + return ingredient_data From 234cef49fb59b20dcc123aa378215db570ead110 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Wed, 22 May 2024 16:44:46 +0200 Subject: [PATCH 24/34] Merge the branch term-consistency Also, make sure just rename the field and don't accidentally nuke the fibre field ;) --- wger/nutrition/dataclasses.py | 4 ++-- .../migrations/0022_fiber_spelling.py | 23 ------------------- ...n_goal_fiber.py => 0023_fiber_spelling.py} | 10 +++++--- wger/nutrition/tests/test_usda.py | 4 ++-- wger/nutrition/usda.py | 2 +- 5 files changed, 12 insertions(+), 31 deletions(-) delete mode 100644 wger/nutrition/migrations/0022_fiber_spelling.py rename wger/nutrition/migrations/{0023_rename_goal_fibers_nutritionplan_goal_fiber.py => 0023_fiber_spelling.py} (51%) diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.py index b1c2b0555..49b9dfb49 100644 --- a/wger/nutrition/dataclasses.py +++ b/wger/nutrition/dataclasses.py @@ -31,7 +31,7 @@ class IngredientData: carbohydrates_sugar: Optional[float] fat: float fat_saturated: Optional[float] - fibres: Optional[float] + fiber: Optional[float] sodium: Optional[float] code: Optional[str] source_name: str @@ -58,7 +58,7 @@ def sanity_checks(self): 'carbohydrates', 'carbohydrates_sugar', 'sodium', - 'fibres', + 'fiber', ] for macro in macros: value = getattr(self, macro) diff --git a/wger/nutrition/migrations/0022_fiber_spelling.py b/wger/nutrition/migrations/0022_fiber_spelling.py deleted file mode 100644 index ad28bd083..000000000 --- a/wger/nutrition/migrations/0022_fiber_spelling.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 4.2.6 on 2024-05-21 19:32 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('nutrition', '0021_add_fibers_field'), - ] - - operations = [ - migrations.RemoveField( - model_name='ingredient', - name='fibres', - ), - migrations.AddField( - model_name='ingredient', - name='fiber', - field=models.DecimalField(blank=True, decimal_places=3, help_text='In g per 100g of product', max_digits=6, null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(100)], verbose_name='Fiber'), - ), - ] diff --git a/wger/nutrition/migrations/0023_rename_goal_fibers_nutritionplan_goal_fiber.py b/wger/nutrition/migrations/0023_fiber_spelling.py similarity index 51% rename from wger/nutrition/migrations/0023_rename_goal_fibers_nutritionplan_goal_fiber.py rename to wger/nutrition/migrations/0023_fiber_spelling.py index 8a26f5140..a2b0dd123 100644 --- a/wger/nutrition/migrations/0023_rename_goal_fibers_nutritionplan_goal_fiber.py +++ b/wger/nutrition/migrations/0023_fiber_spelling.py @@ -1,15 +1,19 @@ -# Generated by Django 4.2.6 on 2024-05-21 19:40 +# Generated by Django 4.2.6 on 2024-05-21 19:32 from django.db import migrations class Migration(migrations.Migration): - dependencies = [ - ('nutrition', '0022_fiber_spelling'), + ('nutrition', '0022_add_remote_id_increase_author_field_length'), ] operations = [ + migrations.RenameField( + model_name='ingredient', + old_name='fibres', + new_name='fiber', + ), migrations.RenameField( model_name='nutritionplan', old_name='goal_fibers', diff --git a/wger/nutrition/tests/test_usda.py b/wger/nutrition/tests/test_usda.py index 4fa0fc643..8dc7895d1 100644 --- a/wger/nutrition/tests/test_usda.py +++ b/wger/nutrition/tests/test_usda.py @@ -218,7 +218,7 @@ def test_regular_response(self): carbohydrates_sugar=None, fat=3.24, fat_saturated=None, - fibres=None, + fiber=None, sodium=None, code=None, source_name='USDA', @@ -228,7 +228,7 @@ def test_regular_response(self): status=AbstractSubmissionModel.STATUS_ACCEPTED, license_id=CC_0_LICENSE_ID, license_author='U.S. Department of Agriculture, Agricultural Research Service, ' - 'Beltsville Human Nutrition Research Center. FoodData Central.', + 'Beltsville Human Nutrition Research Center. FoodData Central.', license_title='Foo With Chocolate', license_object_url='https://fdc.nal.usda.gov/fdc-app.html#/food-details/1234567/nutrients', ) diff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.py index ac0f0cb36..2c432b6a0 100644 --- a/wger/nutrition/usda.py +++ b/wger/nutrition/usda.py @@ -111,7 +111,7 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData: carbohydrates_sugar=carbs_sugars, fat=fats, fat_saturated=fats_saturated, - fibres=fiber, + fiber=fiber, sodium=sodium, source_name=source_name, source_url=source_url, From e3214eb7456e98a79c22825af53cfed7c0ee3288 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Thu, 23 May 2024 17:40:00 +0200 Subject: [PATCH 25/34] Consolidate fiber related migrations into one --- .../migrations/0021_add_fibers_field.py | 7 +++++- .../migrations/0023_fiber_spelling.py | 22 ------------------- 2 files changed, 6 insertions(+), 23 deletions(-) delete mode 100644 wger/nutrition/migrations/0023_fiber_spelling.py diff --git a/wger/nutrition/migrations/0021_add_fibers_field.py b/wger/nutrition/migrations/0021_add_fibers_field.py index 4d5c63d42..9ed0b9496 100644 --- a/wger/nutrition/migrations/0021_add_fibers_field.py +++ b/wger/nutrition/migrations/0021_add_fibers_field.py @@ -16,7 +16,12 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='nutritionplan', - name='goal_fibers', + name='goal_fiber', field=models.IntegerField(default=None, null=True), ), + migrations.RenameField( + model_name='ingredient', + old_name='fibres', + new_name='fiber', + ), ] diff --git a/wger/nutrition/migrations/0023_fiber_spelling.py b/wger/nutrition/migrations/0023_fiber_spelling.py deleted file mode 100644 index a2b0dd123..000000000 --- a/wger/nutrition/migrations/0023_fiber_spelling.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Django 4.2.6 on 2024-05-21 19:32 - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ('nutrition', '0022_add_remote_id_increase_author_field_length'), - ] - - operations = [ - migrations.RenameField( - model_name='ingredient', - old_name='fibres', - new_name='fiber', - ), - migrations.RenameField( - model_name='nutritionplan', - old_name='goal_fibers', - new_name='goal_fiber', - ), - ] From 15a5bc0a5733721c9a13ae46a71a587596c6316a Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Thu, 23 May 2024 17:43:16 +0200 Subject: [PATCH 26/34] Bump minimum app version Needed because the change of "fibres" to "fiber" --- wger/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wger/__init__.py b/wger/__init__.py index e8fc70dc5..0c35a8b6d 100644 --- a/wger/__init__.py +++ b/wger/__init__.py @@ -8,8 +8,7 @@ # Local from .celery_configuration import app - -MIN_APP_VERSION = (1, 6, 0, 'final', 1) +MIN_APP_VERSION = (1, 7, 3, 'final', 1) VERSION = (2, 3, 0, 'alpha', 2) RELEASE = True From 88af124197d96b1aa1f63b4ff81ae26aa8f63994 Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Thu, 23 May 2024 19:02:45 +0200 Subject: [PATCH 27/34] Revert "Consolidate fiber related migrations into one" This reverts commit e3214eb7456e98a79c22825af53cfed7c0ee3288. --- .../migrations/0021_add_fibers_field.py | 7 +----- .../migrations/0023_fiber_spelling.py | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 wger/nutrition/migrations/0023_fiber_spelling.py diff --git a/wger/nutrition/migrations/0021_add_fibers_field.py b/wger/nutrition/migrations/0021_add_fibers_field.py index 9ed0b9496..4d5c63d42 100644 --- a/wger/nutrition/migrations/0021_add_fibers_field.py +++ b/wger/nutrition/migrations/0021_add_fibers_field.py @@ -16,12 +16,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='nutritionplan', - name='goal_fiber', + name='goal_fibers', field=models.IntegerField(default=None, null=True), ), - migrations.RenameField( - model_name='ingredient', - old_name='fibres', - new_name='fiber', - ), ] diff --git a/wger/nutrition/migrations/0023_fiber_spelling.py b/wger/nutrition/migrations/0023_fiber_spelling.py new file mode 100644 index 000000000..a2b0dd123 --- /dev/null +++ b/wger/nutrition/migrations/0023_fiber_spelling.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.6 on 2024-05-21 19:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ('nutrition', '0022_add_remote_id_increase_author_field_length'), + ] + + operations = [ + migrations.RenameField( + model_name='ingredient', + old_name='fibres', + new_name='fiber', + ), + migrations.RenameField( + model_name='nutritionplan', + old_name='goal_fibers', + new_name='goal_fiber', + ), + ] From 53494468812a00d52b3ff7c84eb634bc988af1af Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Fri, 24 May 2024 19:52:14 +0200 Subject: [PATCH 28/34] Remove the status for ingredients Ingredients / products will only be imported through automatic processes. We should just point users in case of missing products to Open Food Facts, and not handle this on our side. Admins can still add entries if they want. --- wger/__init__.py | 1 + wger/core/templates/navigation.html | 51 +++-- wger/nutrition/api/filtersets.py | 1 - wger/nutrition/api/views.py | 3 +- wger/nutrition/dataclasses.py | 1 - wger/nutrition/fixtures/test-ingredients.json | 14 -- .../commands/import-off-products.py | 7 +- .../0024_remove_ingredient_status.py | 55 +++++ wger/nutrition/models/ingredient.py | 57 +---- wger/nutrition/off.py | 2 - wger/nutrition/sitemap.py | 2 +- .../templates/ingredient/pending.html | 37 ---- wger/nutrition/templates/ingredient/view.html | 26 --- wger/nutrition/tests/test_ingredient.py | 10 - .../tests/test_ingredient_overview.py | 2 +- wger/nutrition/tests/test_off.py | 2 - .../tests/test_submitted_ingredient.py | 202 ------------------ wger/nutrition/tests/test_usda.py | 4 +- wger/nutrition/urls.py | 15 -- wger/nutrition/usda.py | 2 - wger/nutrition/views/ingredient.py | 52 +---- wger/utils/managers.py | 48 ----- wger/utils/models.py | 31 --- 23 files changed, 101 insertions(+), 524 deletions(-) create mode 100644 wger/nutrition/migrations/0024_remove_ingredient_status.py delete mode 100644 wger/nutrition/templates/ingredient/pending.html delete mode 100644 wger/nutrition/tests/test_submitted_ingredient.py delete mode 100644 wger/utils/managers.py diff --git a/wger/__init__.py b/wger/__init__.py index 0c35a8b6d..a00e199c0 100644 --- a/wger/__init__.py +++ b/wger/__init__.py @@ -8,6 +8,7 @@ # Local from .celery_configuration import app + MIN_APP_VERSION = (1, 7, 3, 'final', 1) VERSION = (2, 3, 0, 'alpha', 2) diff --git a/wger/core/templates/navigation.html b/wger/core/templates/navigation.html index 830dbf675..4bba8122f 100644 --- a/wger/core/templates/navigation.html +++ b/wger/core/templates/navigation.html @@ -3,7 +3,8 @@
- wger logo + wger logo