diff --git a/.github/imgs/demo.gif b/.github/imgs/demo.gif index 1139d46..b41b2da 100644 Binary files a/.github/imgs/demo.gif and b/.github/imgs/demo.gif differ diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index ff36fcd..9f75b62 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -37,7 +37,7 @@ jobs: MAILERSEND_API_KEY: ${{ secrets.MAILERSEND_API_KEY }} FROM_ADDRESS: no-reply@${{ secrets.DOMAIN }} run: | - python coles_vs_woolies send shopping-list.json \ + python coles_vs_woolies shopping-list.json \ ${{ (github.event_name == 'workflow_dispatch' && inputs.dry_run) && '--dry_run' || '' }} local: needs: github @@ -62,5 +62,5 @@ jobs: MAILERSEND_API_KEY: ${{ secrets.MAILERSEND_API_KEY }} FROM_ADDRESS: no-reply@${{ secrets.DOMAIN }} run: | - python coles_vs_woolies send shopping-list.json \ + python coles_vs_woolies shopping-list.json \ ${{ (github.event_name == 'workflow_dispatch' && inputs.dry_run) && '--dry_run' || '' }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 185d3ce..0b40b0f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .idea -.github/actions-runner shopping-list.json shopping-list.bkup.json diff --git a/CHANGELOG.md b/CHANGELOG.md index cddebda..3b437e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,18 @@ # CHANGELOG.md +## v1.3.0 + +* Added support for merchant-exclusive products +* Added Jaccard similarity sorting for better accuracy results +* Dropped Python3.9 support +* Refactored CLI entry +* Fixed missing merchant in email +* Fixed IGA no-results if query >50 chars +* Fixed email formatting for no-offer-merchants + ## v1.2.0 -* Added 'iga' merchant +* Added 'iga' merchant * Added CLI import shopping list from txt file * Added CLI import of multiple shopping lists from json file diff --git a/README.md b/README.md index dd7b6db..9c6525b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # 🍎 coles_vs_woolies 🍏 -![Python](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11-blue) +![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11-blue) [![pass](https://github.com/MattTimms/coles_vs_woolies/actions/workflows/test.yml/badge.svg)](https://github.com/MattTimms/coles_vs_woolies/actions/workflows/test.yml) [![working just fine for me](https://github.com/MattTimms/coles_vs_woolies/actions/workflows/run.yml/badge.svg)](https://github.com/MattTimms/coles_vs_woolies/actions/workflows/run.yml) @@ -54,32 +54,25 @@ pip install -r requirements.txt ```shell $ python coles_vs_woolies --help -# usage: coles_vs_woolies [-h] {display,send} ... +# usage: coles_vs_woolies [-h] [-o OUT_DIR] [-d] file_path # # Compare prices between Aussie grocers # # positional arguments: -# {display,send} -# display Display product price comparisons -# send Email product price comparisons +# file_path File path to a JSON config shopping list; see `shopping-list.example.json` # # options: -# -h, --help show this help message and exit -# -# example: -# python coles_vs_woolies display -# "Cadbury Dairy Milk Chocolate Block 180g" -# "Connoisseur Ice Cream Vanilla Caramel Brownie 1L" -# -# python coles_vs_woolies send -# "Cadbury Dairy Milk Chocolate Block 180g" -# "Connoisseur Ice Cream Vanilla Caramel Brownie 4 Pack" -# --to_addrs +# -h, --help show this help message and exit +# -o OUT_DIR, --out_dir OUT_DIR +# Directory for saving copy of the email HTML template. +# -d, --dry_run Disable email delivery ``` ```shell cp .env.example .env # populate .env to simplify calls +cp shopping-list.example.json shopping-list.json +# populate the shopping list with your email & desired items ``` ## Install w/ GitHub Actions diff --git a/coles_vs_woolies/__main__.py b/coles_vs_woolies/__main__.py index ebad6e1..e729581 100644 --- a/coles_vs_woolies/__main__.py +++ b/coles_vs_woolies/__main__.py @@ -1,93 +1,41 @@ import argparse import json -import os -import textwrap from argparse import ArgumentParser -from typing import List -from pydantic import BaseModel +from pydantic import BaseModel, Extra -from coles_vs_woolies.main import display, send +from coles_vs_woolies.main import send -def cli(): - example_usage = '''example: - python coles_vs_woolies display - "Cadbury Dairy Milk Chocolate Block 180g" - "Connoisseur Ice Cream Vanilla Caramel Brownie 4 Pack" - - python coles_vs_woolies send - "Cadbury Dairy Milk Chocolate Block 180g" - "Connoisseur Ice Cream Vanilla Caramel Brownie 4 Pack" - --to_addrs - ''' +class ShoppingList(BaseModel, extra=Extra.allow): + """ Model for the `shopping-list` json config file. """ + to_addrs: list[str] + products: list[str] + +def cli(): parser = ArgumentParser( prog='coles_vs_woolies', description='Compare prices between Aussie grocers', formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=textwrap.dedent(example_usage) ) - - subparsers = parser.add_subparsers(dest='action') - - help_product = 'List of descriptive product search terms. Brand, package weight or size should be included. ' \ - 'Can be file path. E.g. "Cadbury Dairy Milk Chocolate Block 180g"' \ - '"Connoisseur Ice Cream Vanilla Caramel Brownie 4 Pack"' - - # Display parser - display_parser = subparsers.add_parser('display', help='Display product price comparisons') - display_parser.add_argument('products', nargs='+', help=help_product) - - # Send parser - send_parser = subparsers.add_parser('send', help='Email product price comparisons') - send_parser.add_argument('products', nargs='+', help=help_product) - send_parser.add_argument('-t', '--to_addrs', nargs='+', help="Recipients' email address.", required=False) - send_parser.add_argument('-o', '--out_dir', type=str, help='Directory for saving copy of the email HTML template.', - required=False) - send_parser.add_argument('-d', '--dry_run', action='store_true', help='Disable email delivery', - default=False, required=False) + parser.add_argument('file_path', type=str, + help='File path to a JSON config shopping list; see `shopping-list.example.json`') + parser.add_argument('-o', '--out_dir', type=str, required=False, + help='Directory for saving copy of the email HTML template.') + parser.add_argument('-d', '--dry_run', action='store_true', default=False, required=False, + help='Disable email delivery') # Parse inputs kwargs = vars(parser.parse_args()) - action = kwargs.pop('action') - - _product_inputs = kwargs.pop('products') - if os.path.isfile(fp := _product_inputs[0]) and fp.endswith('.json'): - with open(fp, 'r') as f: - jobs = [_JsonInput.parse_obj(x) for x in json.load(f)] - _ = kwargs.pop('to_addrs', None) - for job in jobs: - _run(action, job.products, to_addrs=job.to_addrs, **kwargs) - else: - if action == 'send' and kwargs.get('to_addrs', None) is None: - parser.error('the following arguments are required: -t/--to_addrs') - products = _parse_product_inputs(_product_inputs) - _run(action, products, **kwargs) - - -class _JsonInput(BaseModel): - to_addrs: List[str] - products: List[str] - - -def _run(action: str, products: List[str], **kwargs): - if action == 'send': - send(products=products, **kwargs) - else: - display(products=products) - - -def _parse_product_inputs(args: List[str]) -> List[str]: - """ Return product list from input list of products/file-paths """ - products = [] - for input_ in args: - if os.path.isfile(input_): - with open(input_, 'r') as f: - products.extend(f.read().splitlines()) - else: - products.append(input_) - return sorted(list(set(products))) + with open(kwargs.pop('file_path'), 'r') as fp: + shopping_lists: list[ShoppingList] = [ShoppingList.parse_obj(list_) for list_ in json.load(fp)] + + # Run for each shopping list + for shopping_list in shopping_lists: + send(products=shopping_list.products, + to_addrs=shopping_list.to_addrs, + **kwargs) if __name__ == '__main__': diff --git a/coles_vs_woolies/emailing/generate.py b/coles_vs_woolies/emailing/generate.py index a1a2502..f62aa8e 100644 --- a/coles_vs_woolies/emailing/generate.py +++ b/coles_vs_woolies/emailing/generate.py @@ -1,10 +1,11 @@ -import pathlib import datetime +import pathlib from rich.console import Console from rich.table import Table -from coles_vs_woolies.search.types import ProductOffers +from coles_vs_woolies.search import available_merchants_names +from coles_vs_woolies.search.types import Merchant, ProductOffers _SCRIPT_DIR = pathlib.Path(__file__).parent.absolute() _TEMPLATE_DIR = _SCRIPT_DIR / 'templates' @@ -25,10 +26,10 @@ def generate_weekly_email(product_offers: ProductOffers, out_path: str = None) - with open(_TEMPLATE_DIR / 'snippets/table_row.html', 'r', encoding="utf-8") as f: html_template_table_row: str = f.read() - # Replace template variables + # Build merchant offer HTML rows from template rows = [] - green = '#008000' - light_grey = '#afafaf' + green, light_grey = '#008000', '#afafaf' + html_padding = '0' for product_name, offers in product_offers.items(): row_template = html_template_table_row row_template = row_template.replace('{{ product }}', product_name) @@ -36,26 +37,38 @@ def generate_weekly_email(product_offers: ProductOffers, out_path: str = None) - # Replace merchant offers lowest_price = min(offers).price is_sales = any((offer.is_on_special for offer in offers)) + merchants_with_offers: set[Merchant] = set() for offer in offers: merchant = offer.merchant - price = offer.price if offer.price is not None else 'n/a' - colour = green if is_sales and price == lowest_price else light_grey - zero_padding = '0' if len(str(price).split('.')[-1]) == 1 else '' + merchants_with_offers.add(merchant) - html_replacement = f'${price}{zero_padding}' + # Determine text replacement details + price = f'${offer.price}' if offer.price is not None else '-' + colour = green if is_sales and offer.price == lowest_price else light_grey + zero_padding = html_padding if len(price.split('.')[-1]) == 1 else '' + + # Insert merchant offer into HTML template + html_replacement = f'{price}{zero_padding}' row_template = row_template.replace('{{ %(merchant)s_price }}' % {"merchant": merchant}, html_replacement) + # Format email for merchants without offers + for missing_merchant in available_merchants_names.difference(merchants_with_offers): + html_replacement = f'-00' + row_template = row_template.replace('{{ %(merchant)s_price }}' % {"merchant": missing_merchant}, + html_replacement) + rows.append(row_template) + # Build HTML table of merchant offers html_template_table = html_template_table.replace('{{ rows }}', ''.join(rows)) html_template = html_template.replace('{{ table }}', html_template_table) - # Add time + # Add timespan to template year, week, weekday = datetime.datetime.now().isocalendar() week_start, week_fin = (week - 1, week) if weekday < 3 else (week, week + 1) start = datetime.datetime.fromisocalendar(year, week_start, 3) fin = datetime.datetime.fromisocalendar(year, week_fin, 2) - html_template = html_template.replace('{{ intro }}', + html_template = html_template.replace('{{ timespan }}', f"Deals from {start.strftime('%a %d/%m')} till {fin.strftime('%a %d/%m')}") # Output formatted template diff --git a/coles_vs_woolies/emailing/mailer_send.py b/coles_vs_woolies/emailing/mailer_send.py index 5b5c54e..4af7768 100644 --- a/coles_vs_woolies/emailing/mailer_send.py +++ b/coles_vs_woolies/emailing/mailer_send.py @@ -1,8 +1,7 @@ import datetime import os -from typing import List -from dotenv import load_dotenv, find_dotenv +from dotenv import find_dotenv, load_dotenv from mailersend import emails load_dotenv(dotenv_path=find_dotenv()) @@ -11,7 +10,7 @@ def send(email_html: str, - to_addrs: List[str], + to_addrs: list[str], from_addr: str = None, mailersend_api_key: str = None): """ diff --git a/coles_vs_woolies/emailing/templates/weekly.html b/coles_vs_woolies/emailing/templates/weekly.html index e2aaf7d..7060601 100644 --- a/coles_vs_woolies/emailing/templates/weekly.html +++ b/coles_vs_woolies/emailing/templates/weekly.html @@ -161,8 +161,8 @@

- {{ intro }} + style="color:#4a5566;margin-top:20px;margin-bottom:20px;margin-right:0;margin-left:0;font-size:13px;line-height:28px;"> + {{ timespan }}

diff --git a/coles_vs_woolies/examples.py b/coles_vs_woolies/examples.py index d0e5301..8dec15c 100644 --- a/coles_vs_woolies/examples.py +++ b/coles_vs_woolies/examples.py @@ -1,13 +1,14 @@ """ A collection of display examples for product comparisons """ from collections import defaultdict -from typing import Dict, List, Literal +from typing import Literal from rich import box from rich.console import Console from rich.table import Table -from coles_vs_woolies.search.types import ProductOffers, Merchant, Product +from coles_vs_woolies.search.similarity import jaccard_similarity +from coles_vs_woolies.search.types import Merchant, Product, ProductOffers _console = Console() @@ -23,13 +24,14 @@ def compare_offers(product_offers: ProductOffers): for i, _product in enumerate(products): txt_colour = 'green' if is_sales and i in cheapest_product_idx else 'grey50' # txt_colour = None if not i else 'grey50' - _console.print(f' {_product.merchant.upper()}: {_product}', style=txt_colour) + similarity = jaccard_similarity(name, _product.display_name) + _console.print(f' {_product.merchant.upper()}: {_product} | {similarity=:.2f}', style=txt_colour) _console.print('\n') def best_offers_by_merchant(product_offers: ProductOffers): # Collect the cheapest offer - cheapest_products_by_merchant: Dict[Merchant | Literal['either'], List[Product]] = defaultdict(list) + cheapest_products_by_merchant: dict[Merchant | Literal['either'], list[Product]] = defaultdict(list) for products in product_offers.values(): is_all_same_price = len(set(p.price for p in products)) == 1 if is_all_same_price: diff --git a/coles_vs_woolies/main.py b/coles_vs_woolies/main.py index 9d43064..cc1ab82 100644 --- a/coles_vs_woolies/main.py +++ b/coles_vs_woolies/main.py @@ -1,23 +1,25 @@ -from typing import List, Any, Dict - import arrow from rich import print -from coles_vs_woolies.examples import compare_offers, best_offers_by_merchant, generate_offer_table -from coles_vs_woolies.emailing.generate import generate_weekly_email from coles_vs_woolies.emailing import mailer_send +from coles_vs_woolies.emailing.generate import generate_weekly_email +from coles_vs_woolies.examples import best_offers_by_merchant, compare_offers, generate_offer_table from coles_vs_woolies.search import available_merchants +from coles_vs_woolies.search.similarity import jaccard_similarity from coles_vs_woolies.search.types import ProductOffers +MAGIC_SIMILARITY_MIN = 0.3 -def get_product_offers(product_names: List[str]) -> ProductOffers: + +def get_product_offers(product_names: list[str]) -> ProductOffers: """ Returns ProductOffers object with optimistic search results for product_names. """ - product_offers: Dict[str, List[Any]] = {} + product_offers: ProductOffers = {} for name in product_names: product_offers[name] = [] for merchant in available_merchants: merchant_product_search = merchant.im_feeling_lucky(name) - if (product := next(merchant_product_search, None)) is not None: + if (product := next(merchant_product_search, None)) is not None and \ + jaccard_similarity(name, product.display_name) > MAGIC_SIMILARITY_MIN: product_offers[name].append(product) if not product_offers[name]: @@ -29,7 +31,7 @@ def get_product_offers(product_names: List[str]) -> ProductOffers: return product_offers -def display(products: List[str]): +def display(products: list[str]): """ Displays various product comparisons. """ product_offers = get_product_offers(products) @@ -39,8 +41,8 @@ def display(products: List[str]): generate_offer_table(product_offers) -def send(*, products: List[str], - to_addrs: List[str], +def send(*, products: list[str], + to_addrs: list[str], from_addr: str = None, mailersend_api_key: str = None, out_dir: str = None, @@ -57,6 +59,13 @@ def send(*, products: List[str], :param dry_run: set to run without sending emails out. :return: """ + # Censor email addresses for logging + censored_emails = [] + for addr in to_addrs: + prefix, domain = addr.split('@') + censored_emails.append(f'{prefix[:5].ljust(10, "*")}@{domain}') + print(f'Haggling for: {censored_emails}') + product_offers = get_product_offers(products) # Prepare file path for saving generated email template @@ -72,6 +81,7 @@ def send(*, products: List[str], # Display emailed product comparison generate_offer_table(product_offers) + compare_offers(product_offers) if __name__ == '__main__': diff --git a/coles_vs_woolies/search/__init__.py b/coles_vs_woolies/search/__init__.py index 6d180e2..c99cb97 100644 --- a/coles_vs_woolies/search/__init__.py +++ b/coles_vs_woolies/search/__init__.py @@ -1,3 +1,4 @@ from .merchant import coles, iga, woolies -available_merchants = [coles, woolies, iga] \ No newline at end of file +available_merchants = [coles, woolies, iga] +available_merchants_names = {'coles', 'woolies', 'iga'} diff --git a/coles_vs_woolies/search/merchant/coles.py b/coles_vs_woolies/search/merchant/coles.py index b82a72e..227e1e1 100644 --- a/coles_vs_woolies/search/merchant/coles.py +++ b/coles_vs_woolies/search/merchant/coles.py @@ -1,11 +1,12 @@ import json -from typing import List, Literal, Optional, Generator +from typing import Generator, Literal, Optional from bs4 import BeautifulSoup from pydantic import BaseModel, Extra from coles_vs_woolies.search import types from coles_vs_woolies.search.session import new_session +from coles_vs_woolies.search.similarity import jaccard_similarity _session = new_session() @@ -90,7 +91,7 @@ class ProductPageSearchResult(BaseModel, extra=Extra.allow): pageSize: int # 48 keyword: str # "cadbury chocolate" resultType: int # 1 - results: List[Product] + results: list[Product] def search_exact(self, product_name: str) -> Optional[Product]: """ Return product that matches product_name within list of paged results """ @@ -104,6 +105,7 @@ def search_exact(self, product_name: str) -> Optional[Product]: def im_feeling_lucky(search_term: str) -> Generator[Product, None, None]: paginated_search = search(search_term) for page in paginated_search: + page.results.sort(key=lambda x: jaccard_similarity(search_term, x.display_name), reverse=True) for product in page.results: yield product diff --git a/coles_vs_woolies/search/merchant/iga.py b/coles_vs_woolies/search/merchant/iga.py index 6c91178..3c655b2 100644 --- a/coles_vs_woolies/search/merchant/iga.py +++ b/coles_vs_woolies/search/merchant/iga.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, List, Generator, Dict +from typing import Any, Generator, Optional from pydantic import BaseModel, Extra @@ -29,14 +29,14 @@ def __str__(self): # categories: ... # defaultCategory: ... description: str - image: Dict[str, str] + image: dict[str, Optional[str]] isFavorite: bool isPastPurchased: bool name: str # price: str # "$3.20" n.b. omitted due to clash with class-property priceLabel: str priceNumeric: float # 3.2 - pricePerUnit: str # "$0.64/100ml" + pricePerUnit: Optional[str] # "$0.64/100ml" priceSource: str productId: str sellBy: str @@ -83,7 +83,7 @@ def fetch_product(cls, product_id: str, store_id: int = _DEFAULT_STORE): class ProductPageSearchResult(BaseModel, extra=Extra.allow): count: int # page count - items: List[Product] + items: list[Product] total: int # total results @@ -98,7 +98,7 @@ def im_feeling_lucky(search_term: str) -> Generator[Product, None, None]: def search(search_term: str) -> Generator[ProductPageSearchResult, None, None]: url = f'https://www.igashop.com.au/api/storefront/stores/{_DEFAULT_STORE}/search' params = { - 'q': search_term, + 'q': search_term[:50], # n.b. no results if query > 50 'skip': 0, 'take': 40 } @@ -114,4 +114,3 @@ def search(search_term: str) -> Generator[ProductPageSearchResult, None, None]: if __name__ == '__main__': gen = search('Cadbury Dairy Milk Chocolate Block 180g') print(next(gen)) - print(1) diff --git a/coles_vs_woolies/search/merchant/woolies.py b/coles_vs_woolies/search/merchant/woolies.py index c9e4b47..ab6c5cc 100644 --- a/coles_vs_woolies/search/merchant/woolies.py +++ b/coles_vs_woolies/search/merchant/woolies.py @@ -1,10 +1,11 @@ import urllib.parse -from typing import Any, Optional, List, Generator +from typing import Any, Generator, Optional from pydantic import BaseModel, Extra from coles_vs_woolies.search import types from coles_vs_woolies.search.session import new_session +from coles_vs_woolies.search.similarity import jaccard_similarity def _woolies_session(): @@ -120,13 +121,13 @@ def fetch_product(cls, product_id: str): class ProductSearchResult(BaseModel, extra=Extra.allow): - Products: Optional[List[Product]] + Products: Optional[list[Product]] Name: str DisplayName: str class ProductPageSearchResult(BaseModel, extra=Extra.allow): - Products: Optional[List[ProductSearchResult]] + Products: Optional[list[ProductSearchResult]] SearchResultsCount: int Corrections: Optional[Any] SuggestedTerm: Optional[Any] @@ -135,6 +136,7 @@ class ProductPageSearchResult(BaseModel, extra=Extra.allow): def im_feeling_lucky(search_term: str) -> Generator[Product, None, None]: paginated_search = search(search_term) for page in paginated_search: + page.Products.sort(key=lambda x: jaccard_similarity(search_term, x.Products[0].display_name), reverse=True) for product in page.Products: for _product in product.Products: yield _product diff --git a/coles_vs_woolies/search/types.py b/coles_vs_woolies/search/types.py index 4b11165..ae0e1ac 100644 --- a/coles_vs_woolies/search/types.py +++ b/coles_vs_woolies/search/types.py @@ -1,5 +1,5 @@ import abc -from typing import Optional, Literal, Dict, List +from typing import Literal, Optional Merchant = Literal['coles', 'woolies', 'iga'] @@ -32,7 +32,7 @@ def __lt__(self, other: 'Product'): return (self.price or 1e6) < (other.price or 1e6) # default to big number when no price available -ProductOffers = Dict[str, List[Product]] # {'product_name': [Product, ...]} +ProductOffers = dict[str, list[Product]] # {'product_name': [Product, ...]} # class OfferDict(dict): # @classmethod @@ -41,7 +41,7 @@ def __lt__(self, other: 'Product'): # def price_by_merchant(self, merchant: Merchant) -> Optional[float]: # return self[merchant].price # -# def to_list(self, sort: bool = False) -> List[_Product]: +# def to_list(self, sort: bool = False) -> list[_Product]: # if sort: # return sorted(self.items(), key=_sort_by_price_key) # else: diff --git a/pyproject.toml b/pyproject.toml index 160fa66..c787143 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,11 @@ [project] name = "coles_vs_woolies" -version = "1.2.0" +version = "1.3.0" description = "Compare & be notified of the best offers of your favourite items from Aussie grocers 🦘🍌" readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" license = { text = "GPL-3.0-or-later" } -keywords = ["coles", "woolies", "woolworths"] +keywords = ["coles", "woolies", "woolworths", "iga"] authors = [{ name = "Matthew Timms", email = "matthewtimms@live.com.au" }] classifiers = [ "Development Status :: 4 - Beta", diff --git a/shopping-list.example.txt b/shopping-list.example.txt deleted file mode 100644 index a04e156..0000000 --- a/shopping-list.example.txt +++ /dev/null @@ -1,5 +0,0 @@ -Cadbury Dairy Milk Chocolate Block 180g -Cadbury Dairy Milk Vanilla Sticks 4 Pack -Connoisseur Ice Cream Vanilla Caramel Brownie 4 Pack -Connoisseur Ice Cream Vanilla Caramel Brownie 1L -The Juice Brothers 1.5L \ No newline at end of file