From 2086975e8d34eb48ec4f06413053a2b04531a677 Mon Sep 17 00:00:00 2001 From: Drew Winstel Date: Wed, 14 Oct 2020 09:25:27 -0500 Subject: [PATCH 01/12] Add nullable fields for last venue update/check times --- venues/migrations/0028_auto_20201014_1424.py | 31 ++++++++++++++++++++ venues/models.py | 6 ++++ 2 files changed, 37 insertions(+) create mode 100644 venues/migrations/0028_auto_20201014_1424.py diff --git a/venues/migrations/0028_auto_20201014_1424.py b/venues/migrations/0028_auto_20201014_1424.py new file mode 100644 index 00000000..4a117f67 --- /dev/null +++ b/venues/migrations/0028_auto_20201014_1424.py @@ -0,0 +1,31 @@ +# Generated by Django 3.0.8 on 2020-10-14 14:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("venues", "0027_auto_20200124_2059"), + ] + + operations = [ + migrations.AddField( + model_name="venue", + name="tap_list_last_check_time", + field=models.DateTimeField( + blank=True, + null=True, + verbose_name="The last time the venue's tap list was refreshed", + ), + ), + migrations.AddField( + model_name="venue", + name="tap_list_last_update_time", + field=models.DateTimeField( + blank=True, + null=True, + verbose_name="The last time the venue's tap list was updated", + ), + ), + ] diff --git a/venues/models.py b/venues/models.py index b92a8020..947ac8ae 100644 --- a/venues/models.py +++ b/venues/models.py @@ -65,6 +65,12 @@ class Venue(models.Model): max_length=25, blank=True, ) + tap_list_last_check_time = models.DateTimeField( + "The last time the venue's tap list was refreshed", blank=True, null=True + ) + tap_list_last_update_time = models.DateTimeField( + "The last time the venue's tap list was updated", blank=True, null=True + ) def __str__(self): return self.name From cdf1727ecec7b171d8fb77b390f27e285947eb74 Mon Sep 17 00:00:00 2001 From: Drew Winstel Date: Thu, 15 Oct 2020 08:23:07 -0500 Subject: [PATCH 02/12] Add method to base parser to set timestamp based on returned value --- tap_list_providers/base.py | 15 +++++++++++++-- tap_list_providers/test/test_base.py | 27 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/tap_list_providers/base.py b/tap_list_providers/base.py index 084ccf74..c6a97e05 100644 --- a/tap_list_providers/base.py +++ b/tap_list_providers/base.py @@ -9,10 +9,12 @@ from decimal import Decimal, InvalidOperation from urllib.parse import urlparse, unquote import logging +import datetime from django.db.models import Prefetch, Q from django.db.models.functions import Length from django.db import transaction +from django.utils.timezone import now from kombu.exceptions import OperationalError from venues.models import Venue @@ -51,12 +53,13 @@ class BaseTapListProvider: def __init__(self): + self.check_timestamp = now() self.styles = {} if not hasattr(self, "provider_name"): # Don't define this attribute if the child does for us self.provider_name = None - def handle_venue(self, venue): + def handle_venue(self, venue: Venue) -> datetime.datetime: raise NotImplementedError("You need to implement this yourself") @classmethod @@ -92,7 +95,15 @@ def get_venues(self): def handle_venues(self, venues): for venue in venues: LOG.debug("Fetching beers at %s", venue) - self.handle_venue(venue) + update_time = self.handle_venue(venue) + self.update_venue_timestamps(venue, update_time) + + def update_venue_timestamps(self, venue: Venue, update_time: datetime.datetime = None) -> None: + """Update the venue last checked and last updated times""" + venue.tap_list_last_check_time = self.check_timestamp + if update_time: + venue.tap_list_last_update_time = update_time + venue.save() def get_style(self, name): name = name.strip() diff --git a/tap_list_providers/test/test_base.py b/tap_list_providers/test/test_base.py index 692732b8..03f6272a 100644 --- a/tap_list_providers/test/test_base.py +++ b/tap_list_providers/test/test_base.py @@ -1,7 +1,11 @@ +import datetime + from django.test import TestCase +from django.utils.timezone import now from unittest import TestCase as UnittestTestCase from tap_list_providers.base import fix_urls, BaseTapListProvider +from venues.test.factories import VenueFactory from beers.test.factories import ManufacturerFactory, StyleFactory from beers.models import Manufacturer, ManufacturerAlternateName @@ -91,3 +95,26 @@ def test_yazoo(self): provider = BaseTapListProvider() name = provider.reformat_beer_name("Yazoo Brewing Company Hefeweizen", "Yazoo") self.assertEqual(name, "Hefeweizen") + + +class TimestampTestCase(TestCase): + + def setUp(self): + self.venue = VenueFactory() + self.provider = BaseTapListProvider() + + def test_initial_conditions(self): + self.assertIsNone(self.venue.tap_list_last_check_time) + self.assertIsNone(self.venue.tap_list_last_update_time) + self.assertIsNotNone(self.provider.check_timestamp) + + def test_timestamp_no_time(self): + self.provider.update_venue_timestamps(self.venue, None) + self.assertIsNone(self.venue.tap_list_last_update_time) + self.assertEqual(self.venue.tap_list_last_check_time, self.provider.check_timestamp) + + def test_with_time(self): + timestamp = now() - datetime.timedelta(days=1) + self.provider.update_venue_timestamps(self.venue, timestamp) + self.assertEqual(self.venue.tap_list_last_update_time, timestamp) + self.assertEqual(self.venue.tap_list_last_check_time, self.provider.check_timestamp) From 35cf785b9e663c38f59e1e3675d4ea7da4783916 Mon Sep 17 00:00:00 2001 From: Drew Winstel Date: Thu, 15 Oct 2020 09:26:19 -0500 Subject: [PATCH 03/12] Wire in timestamp parsing for beermenus --- .../management/commands/parsebeermenus.py | 6 +++++- tap_list_providers/parsers/beermenus.py | 17 +++++++++++++---- tap_list_providers/test/test_beermenus.py | 8 ++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/tap_list_providers/management/commands/parsebeermenus.py b/tap_list_providers/management/commands/parsebeermenus.py index 4b3c9938..2a61ef1c 100644 --- a/tap_list_providers/management/commands/parsebeermenus.py +++ b/tap_list_providers/management/commands/parsebeermenus.py @@ -18,5 +18,9 @@ def handle(self, *args, **options): with transaction.atomic(): for venue in tap_list_provider.get_venues(): self.stdout.write("Processing %s" % venue.name) - tap_list_provider.handle_venue(venue) + timestamp = tap_list_provider.handle_venue(venue) + + print('timestamp', timestamp) + tap_list_provider.update_venue_timestamps(venue, timestamp) + self.stdout.write(self.style.SUCCESS("Done!")) diff --git a/tap_list_providers/parsers/beermenus.py b/tap_list_providers/parsers/beermenus.py index 855d7abe..320b8db6 100644 --- a/tap_list_providers/parsers/beermenus.py +++ b/tap_list_providers/parsers/beermenus.py @@ -1,4 +1,7 @@ +"""Parser for beermenus dot com""" + from decimal import Decimal +import datetime from dataclasses import dataclass import logging import os @@ -71,6 +74,7 @@ def __init__( self.categories = categories self.soup = None self.save_fetched_data = save_fetched_data + self.updated_date = None super().__init__() def fetch_data(self) -> str: @@ -100,14 +104,14 @@ def parse_html(self, data: str) -> List[BeerData]: )[0] # they just give us a date. I'm going to arbitrarily declare that to be # midnight UTC because who cares if we're off by a day - updated_date = UTC.localize( + self.updated_date = UTC.localize( parse( updated_span.text.split()[1], dayfirst=False, ) ) # TODO save this to the venue - LOG.debug("Last updated: %s", updated_date) + LOG.debug("Last updated: %s", self.updated_date) # the beer lists are in
    s beers = [] @@ -218,7 +222,7 @@ def parse_beers(self, beers: List[BeerData]) -> None: beer.brewery_name = brewery_a.text beer.brewery_location = brewery_p.text.split(MIDDOT)[1].strip() - def handle_venue(self, venue: Venue) -> None: + def handle_venue(self, venue: Venue) -> datetime.datetime: self.categories = venue.api_configuration.beermenus_categories self.location_url = self.URL.format(venue.api_configuration.beermenus_slug) data = self.fetch_data() @@ -261,6 +265,7 @@ def handle_venue(self, venue: Venue) -> None: tap_number, beer, ) + return self.updated_date def parse_beer_tag(tag: Tag) -> BeerData: @@ -289,7 +294,7 @@ def parse_beer_tag(tag: Tag) -> BeerData: ) -if __name__ == "__main__": +def main(): import argparse LOCATIONS = { @@ -319,3 +324,7 @@ def parse_beer_tag(tag: Tag) -> BeerData: for beer in beers: print(f"{beer.name} by {beer.brewery_name} ({beer.abv}%, {beer.style})") + + +if __name__ == "__main__": + main() diff --git a/tap_list_providers/test/test_beermenus.py b/tap_list_providers/test/test_beermenus.py index 3c7dd97c..2fabe763 100644 --- a/tap_list_providers/test/test_beermenus.py +++ b/tap_list_providers/test/test_beermenus.py @@ -4,6 +4,7 @@ from django.core.management import call_command from django.test import TestCase +from django.utils.timezone import now import responses from beers.models import Beer, Manufacturer, ManufacturerAlternateName @@ -71,6 +72,9 @@ def beer_menu_callback(self, request): @responses.activate def test_import_beermenus_data(self): """Test parsing the HTML and JS data""" + timestamp = now() + self.assertIsNone(self.venue.tap_list_last_check_time) + self.assertIsNone(self.venue.tap_list_last_update_time) for url, name, html in self.locations: responses.add( responses.GET, @@ -152,3 +156,7 @@ def test_import_beermenus_data(self): tap.beer.manufacturer.name, "Rocket Republic Brewing Company", ) + self.venue.refresh_from_db() + self.assertIsNotNone(self.venue.tap_list_last_check_time) + self.assertGreater(self.venue.tap_list_last_check_time, timestamp) + self.assertIsNotNone(self.venue.tap_list_last_update_time) From 7d058de7e3349feba59cf3c5fda723b778359eb4 Mon Sep 17 00:00:00 2001 From: Drew Winstel Date: Thu, 15 Oct 2020 09:26:47 -0500 Subject: [PATCH 04/12] silence pylint complaining about celery --- tap_list_providers/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tap_list_providers/base.py b/tap_list_providers/base.py index c6a97e05..bdc0be4e 100644 --- a/tap_list_providers/base.py +++ b/tap_list_providers/base.py @@ -100,6 +100,7 @@ def handle_venues(self, venues): def update_venue_timestamps(self, venue: Venue, update_time: datetime.datetime = None) -> None: """Update the venue last checked and last updated times""" + LOG.debug('Setting check time for %s to %s and update time for to %s', venue, self.check_timestamp, update_time) venue.tap_list_last_check_time = self.check_timestamp if update_time: venue.tap_list_last_update_time = update_time @@ -421,7 +422,7 @@ def get_beer(self, name, manufacturer, pricing=None, venue=None, **defaults): if str(exc).casefold() == "max number of clients reached".casefold(): LOG.error("Reached redis limit!") # fall back to doing it synchronously - look_up_beer(beer.id) + look_up_beer(beer.id) # pylint: disable=no-value-for-parameter else: raise return beer From 734f2134690152c406b96b8b039ee3cf31a5db30 Mon Sep 17 00:00:00 2001 From: Drew Winstel Date: Thu, 15 Oct 2020 09:27:33 -0500 Subject: [PATCH 05/12] Wire in timestamp parsing for digitalpour --- .../management/commands/parsedigitalpour.py | 5 +++-- tap_list_providers/parsers/digitalpour.py | 12 +++++++++++- tap_list_providers/test/test_digitalpour.py | 6 ++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/tap_list_providers/management/commands/parsedigitalpour.py b/tap_list_providers/management/commands/parsedigitalpour.py index e90b1a81..b1fec682 100644 --- a/tap_list_providers/management/commands/parsedigitalpour.py +++ b/tap_list_providers/management/commands/parsedigitalpour.py @@ -7,7 +7,7 @@ class Command(BaseCommand): - help = "Populates any venues using the DigitalPour tap list provider with" " beers" + help = "Populates any venues using the DigitalPour tap list provider with beers" def add_arguments(self, parser): # does not take any arguments @@ -18,5 +18,6 @@ def handle(self, *args, **options): with transaction.atomic(): for venue in tap_list_provider.get_venues(): self.stdout.write("Processing %s" % venue.name) - tap_list_provider.handle_venue(venue) + timestamp = tap_list_provider.handle_venue(venue) + tap_list_provider.update_venue_timestamps(venue, timestamp) self.stdout.write(self.style.SUCCESS("Done!")) diff --git a/tap_list_providers/parsers/digitalpour.py b/tap_list_providers/parsers/digitalpour.py index 93c3d3f6..88ff95d0 100644 --- a/tap_list_providers/parsers/digitalpour.py +++ b/tap_list_providers/parsers/digitalpour.py @@ -1,3 +1,6 @@ +"""Parser for DigitalPour""" + +import datetime from decimal import Decimal import logging import os @@ -8,6 +11,7 @@ import configurations from django.utils.timezone import now from django.core.exceptions import ImproperlyConfigured, AppRegistryNotReady +from pytz import UTC # boilerplate code necessary for launching outside manage.py try: @@ -19,6 +23,7 @@ from ..base import BaseTapListProvider from taps.models import Tap +from venues.models import Venue LOG = logging.getLogger(__name__) @@ -34,16 +39,18 @@ class DigitalPourParser(BaseTapListProvider): def __init__(self, location=None): """Constructor.""" self.url = None + self.update_date = None if location: self.url = self.URL.format(location[0], location[1], self.APIKEY) super().__init__() - def handle_venue(self, venue): + def handle_venue(self, venue: Venue) -> datetime.datetime: venue_id = venue.api_configuration.digital_pour_venue_id location_number = venue.api_configuration.digital_pour_location_number self.url = self.URL.format(venue_id, location_number, self.APIKEY) data = self.fetch() taps = {tap.tap_number: tap for tap in venue.taps.all()} + self.update_date = UTC.localize(datetime.datetime(1970, 1, 1, 0, 0, 0)) manufacturers = {} for entry in data: if not entry["Active"]: @@ -57,6 +64,8 @@ def handle_venue(self, venue): tap = Tap(venue=venue, tap_number=tap_info["tap_number"]) tap.time_added = tap_info["added"] tap.time_updated = tap_info["updated"] + if tap.time_updated and tap.time_updated > self.update_date: + self.update_date = tap.time_updated tap.estimated_percent_remaining = tap_info["percent_full"] if tap_info["gas_type"] in [i[0] for i in Tap.GAS_CHOICES]: tap.gas_type = tap_info["gas_type"] @@ -116,6 +125,7 @@ def handle_venue(self, venue): # 4. assign the beer to the tap tap.beer = beer tap.save() + return self.update_date def parse_beer(self, entry): """Parse beer info from JSON entry.""" diff --git a/tap_list_providers/test/test_digitalpour.py b/tap_list_providers/test/test_digitalpour.py index b9124a67..fd11c1cc 100644 --- a/tap_list_providers/test/test_digitalpour.py +++ b/tap_list_providers/test/test_digitalpour.py @@ -5,6 +5,7 @@ from django.core.management import call_command from django.test import TestCase +from django.utils.timezone import now import responses from beers.models import Beer, Manufacturer @@ -40,6 +41,7 @@ def setUpTestData(cls): @responses.activate def test_import_digitalpour_data(self): """Test parsing the JSON data""" + timestamp = now() responses.add( responses.GET, DigitalPourParser.URL.format( @@ -146,6 +148,10 @@ def test_import_digitalpour_data(self): "https://s3.amazonaws.com/digitalpourproducerlogos/" "57ac9c3c5e002c172c8a6ede.jpg", ) + self.venue.refresh_from_db() + self.assertIsNotNone(self.venue.tap_list_last_check_time) + self.assertGreater(self.venue.tap_list_last_check_time, timestamp) + self.assertIsNotNone(self.venue.tap_list_last_update_time) def test_digital_pour_mead(self): tap = { From 6d57cf8b91dae5364dc7ec78970d99b92aa767ec Mon Sep 17 00:00:00 2001 From: Drew Winstel Date: Thu, 15 Oct 2020 09:28:04 -0500 Subject: [PATCH 06/12] Wire in timestamp parsing for stem and stein --- .../management/commands/parsestemandstein.py | 5 +++-- tap_list_providers/parsers/stemandstein.py | 19 ++++++++++++++++++- tap_list_providers/test/test_stemandstein.py | 6 ++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tap_list_providers/management/commands/parsestemandstein.py b/tap_list_providers/management/commands/parsestemandstein.py index 5305043f..1c772968 100644 --- a/tap_list_providers/management/commands/parsestemandstein.py +++ b/tap_list_providers/management/commands/parsestemandstein.py @@ -8,7 +8,7 @@ class Command(BaseCommand): help = ( - "Populates any venues using the Stem and Stein tap list provider with" " beers" + "Populates any venues using the Stem and Stein tap list provider with beers" ) def add_arguments(self, parser): @@ -20,5 +20,6 @@ def handle(self, *args, **options): with transaction.atomic(): for venue in tap_list_provider.get_venues(): self.stdout.write("Processing %s" % venue.name) - tap_list_provider.handle_venue(venue) + timestamp = tap_list_provider.handle_venue(venue) + tap_list_provider.update_venue_timestamps(venue, timestamp) self.stdout.write(self.style.SUCCESS("Done!")) diff --git a/tap_list_providers/parsers/stemandstein.py b/tap_list_providers/parsers/stemandstein.py index 22ed848e..c2f32ea9 100644 --- a/tap_list_providers/parsers/stemandstein.py +++ b/tap_list_providers/parsers/stemandstein.py @@ -1,4 +1,5 @@ """HTML scraper for The Stem & Stein""" +import datetime from urllib.parse import parse_qsl from html.parser import HTMLParser from decimal import Decimal @@ -8,6 +9,8 @@ from bs4 import BeautifulSoup import requests import configurations +import pytz +from dateutil.parser import parse from django.db.models import Q from django.core.exceptions import ImproperlyConfigured, AppRegistryNotReady @@ -24,6 +27,7 @@ from taps.models import Tap +CENTRAL_TIME = pytz.timezone('America/Chicago') LOG = logging.getLogger(__name__) @@ -133,6 +137,8 @@ def fill_in_beer_details(self, beer): ).text beer_parser = BeautifulSoup(beer_html, "html.parser") jumbotron = beer_parser.find("div", {"class": "jumbotron"}) + tap_table = beer_parser.find('table', {'id': 'tapList'}) + tap_body = tap_table.find('tbody') image_div = jumbotron.find( "div", {"style": "display:table-cell;vertical-align:top;width:17px;"}, @@ -190,6 +196,12 @@ def fill_in_beer_details(self, beer): beer=beer, defaults={"price": price}, ) + time_tapped = None + for row in tap_body.find_all('tr'): + cells = list(row.find_all('td')) + if cells[-1].text.endswith('(so far)'): + time_tapped = CENTRAL_TIME.localize(parse(cells[0].text)) + return time_tapped def handle_venue(self, venue): self.venue = venue @@ -199,8 +211,11 @@ def handle_venue(self, venue): existing_taps = {i.tap_number: i for i in venue.taps.all()} LOG.debug("existing taps %s", existing_taps) taps_hit = [] + latest_time = CENTRAL_TIME.localize(datetime.datetime(1970, 1, 1, 0)) for tap_number, beer in taps.items(): - self.fill_in_beer_details(beer) + time_tapped = self.fill_in_beer_details(beer) + if time_tapped > latest_time: + latest_time = time_tapped try: tap = existing_taps[tap_number] except KeyError: @@ -209,9 +224,11 @@ def handle_venue(self, venue): tap_number=tap_number, ) tap.beer = beer + tap.time_added = time_tapped tap.save() taps_hit.append(tap.tap_number) LOG.debug("Deleting all taps except %s", taps_hit) Tap.objects.filter(venue=venue,).exclude( tap_number__in=taps_hit, ).delete() + return latest_time diff --git a/tap_list_providers/test/test_stemandstein.py b/tap_list_providers/test/test_stemandstein.py index 6f861434..dbe858c0 100644 --- a/tap_list_providers/test/test_stemandstein.py +++ b/tap_list_providers/test/test_stemandstein.py @@ -4,6 +4,7 @@ from django.core.management import call_command from django.test import TestCase +from django.utils.timezone import now import responses from beers.models import ( @@ -80,6 +81,7 @@ def setUpTestData(cls): @responses.activate def test_import_stemandstein_data(self): """Test parsing the JSON data""" + timestamp = now() for pk, html_data in self.html_data.items(): if pk == "root": url = "https://thestemandstein.com/" @@ -159,6 +161,10 @@ def test_import_stemandstein_data(self): # style is set to Fruit Ale but the beer name is preserved self.assertEqual(tap.beer.style_id, style.id) self.assertTrue(tap.beer.name.endswith("Fruit Ale")) + self.venue.refresh_from_db() + self.assertIsNotNone(self.venue.tap_list_last_check_time) + self.assertGreater(self.venue.tap_list_last_check_time, timestamp) + self.assertIsNotNone(self.venue.tap_list_last_update_time) def test_guess_manufacturer_good_people(self): mfg_names = [ From 34113a0473ae9ee4ddc690cd40bf90d97a9aef4c Mon Sep 17 00:00:00 2001 From: Drew Winstel Date: Thu, 15 Oct 2020 09:37:21 -0500 Subject: [PATCH 07/12] Wire in timestamp parsing for taphunter --- tap_list_providers/management/commands/parsetaphunter.py | 5 +++-- tap_list_providers/parsers/taphunter.py | 8 ++++++++ tap_list_providers/test/test_taphunter.py | 6 ++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tap_list_providers/management/commands/parsetaphunter.py b/tap_list_providers/management/commands/parsetaphunter.py index bdcc56d6..b0ec2060 100644 --- a/tap_list_providers/management/commands/parsetaphunter.py +++ b/tap_list_providers/management/commands/parsetaphunter.py @@ -7,7 +7,7 @@ class Command(BaseCommand): - help = "Populates any venues using the TapHunter tap list provider with" " beers" + help = "Populates any venues using the TapHunter tap list provider with beers" def add_arguments(self, parser): # does not take any arguments @@ -18,5 +18,6 @@ def handle(self, *args, **options): with transaction.atomic(): for venue in tap_list_provider.get_venues(): self.stdout.write("Processing %s" % venue.name) - tap_list_provider.handle_venue(venue) + timestamp = tap_list_provider.handle_venue(venue) + tap_list_provider.update_venue_timestamps(venue, timestamp) self.stdout.write(self.style.SUCCESS("Done!")) diff --git a/tap_list_providers/parsers/taphunter.py b/tap_list_providers/parsers/taphunter.py index 23107ee6..eacc0607 100644 --- a/tap_list_providers/parsers/taphunter.py +++ b/tap_list_providers/parsers/taphunter.py @@ -1,5 +1,6 @@ """Parse beers from TapHunter""" import argparse +import datetime import decimal import logging import os @@ -7,7 +8,9 @@ import configurations import requests +from dateutil.parser import parse from django.core.exceptions import ImproperlyConfigured, AppRegistryNotReady +from pytz import UTC # boilerplate code necessary for launching outside manage.py try: @@ -49,6 +52,7 @@ def handle_venue(self, venue): use_sequential_taps = any( (tap_info["serving_info"]["tap_number"] == "" for tap_info in data["taps"]) ) + latest_timestamp = UTC.localize(datetime.datetime(1970, 1, 1, 12)) for index, entry in enumerate(data["taps"]): # 1. parse the tap tap_info = self.parse_tap(entry) @@ -73,6 +77,9 @@ def handle_venue(self, venue): tap = Tap(venue=venue, tap_number=tap_number) tap.time_added = tap_info["added"] tap.time_updated = tap_info["updated"] + parsed_time = parse(tap_info['updated']) + if parsed_time > latest_timestamp: + latest_timestamp = parsed_time if "percent_full" in tap_info: tap.estimated_percent_remaining = tap_info["percent_full"] else: @@ -123,6 +130,7 @@ def handle_venue(self, venue): # 4. assign the beer to the tap tap.beer = beer tap.save() + return latest_timestamp def parse_beer(self, tap): beer = { diff --git a/tap_list_providers/test/test_taphunter.py b/tap_list_providers/test/test_taphunter.py index fc720309..18d7d69f 100644 --- a/tap_list_providers/test/test_taphunter.py +++ b/tap_list_providers/test/test_taphunter.py @@ -4,6 +4,7 @@ from django.core.management import call_command from django.test import TestCase +from django.utils.timezone import now import responses from beers.models import Beer, Manufacturer @@ -38,6 +39,7 @@ def setUpTestData(cls): @responses.activate def test_import_taphunter_data(self): """Test parsing the JSON data""" + timestamp = now() responses.add( responses.GET, TaphunterParser.URL.format(self.venue_cfg.taphunter_location), @@ -132,3 +134,7 @@ def test_import_taphunter_data(self): 3, price_instance, ) + self.venue.refresh_from_db() + self.assertIsNotNone(self.venue.tap_list_last_check_time) + self.assertGreater(self.venue.tap_list_last_check_time, timestamp) + self.assertIsNotNone(self.venue.tap_list_last_update_time) From 88553065e31d6826adb40c9b67ec5a00d7f6c5b9 Mon Sep 17 00:00:00 2001 From: Drew Winstel Date: Thu, 15 Oct 2020 09:41:25 -0500 Subject: [PATCH 08/12] wire in timestamp parsing for taplist.io --- tap_list_providers/management/commands/parsetaplistio.py | 3 ++- tap_list_providers/parsers/taplist_io.py | 1 + tap_list_providers/test/test_taplist_io.py | 6 ++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tap_list_providers/management/commands/parsetaplistio.py b/tap_list_providers/management/commands/parsetaplistio.py index 63749a0b..1bd15b9c 100644 --- a/tap_list_providers/management/commands/parsetaplistio.py +++ b/tap_list_providers/management/commands/parsetaplistio.py @@ -18,5 +18,6 @@ def handle(self, *args, **options): with transaction.atomic(): for venue in tap_list_provider.get_venues(): self.stdout.write("Processing %s" % venue.name) - tap_list_provider.handle_venue(venue) + timestamp = tap_list_provider.handle_venue(venue) + tap_list_provider.update_venue_timestamps(venue, timestamp) self.stdout.write(self.style.SUCCESS("Done!")) diff --git a/tap_list_providers/parsers/taplist_io.py b/tap_list_providers/parsers/taplist_io.py index 7ad0a88f..3d826f55 100644 --- a/tap_list_providers/parsers/taplist_io.py +++ b/tap_list_providers/parsers/taplist_io.py @@ -72,6 +72,7 @@ def handle_venue(self, venue): if current_tap.time_updated != timestamp: current_tap.time_updated = timestamp current_tap.save() + return timestamp def parse_tap(self, tap_dict): if tap_dict["current_keg"] is None: diff --git a/tap_list_providers/test/test_taplist_io.py b/tap_list_providers/test/test_taplist_io.py index 554f2467..20b759ce 100644 --- a/tap_list_providers/test/test_taplist_io.py +++ b/tap_list_providers/test/test_taplist_io.py @@ -5,6 +5,7 @@ from dateutil.parser import parse from django.core.management import call_command from django.test import TestCase +from django.utils.timezone import now import responses from beers.models import Beer, Manufacturer, ManufacturerAlternateName @@ -42,6 +43,7 @@ def setUpTestData(cls): @responses.activate def test_import_taplist_io_data(self): """Test parsing the JSON data""" + timestamp = now() responses.add( responses.GET, TaplistDotIOParser.URL.format( @@ -103,3 +105,7 @@ def test_import_taplist_io_data(self): # NOTE: Yes, really. self.assertEqual(tap.beer.style.name, "An elusive IPA") self.assertEqual(tap.time_updated, self.timestamp) + self.venue.refresh_from_db() + self.assertIsNotNone(self.venue.tap_list_last_check_time) + self.assertGreater(self.venue.tap_list_last_check_time, timestamp) + self.assertIsNotNone(self.venue.tap_list_last_update_time) From f8550bc9e1f365964335f5c2a39a648ea12fccb2 Mon Sep 17 00:00:00 2001 From: Drew Winstel Date: Thu, 15 Oct 2020 09:50:29 -0500 Subject: [PATCH 09/12] Wire in untappd timestamp parsing --- .../management/commands/parseuntappd.py | 3 ++- tap_list_providers/parsers/untappd.py | 12 ++++++++++-- tap_list_providers/test/test_untappd.py | 8 +++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/tap_list_providers/management/commands/parseuntappd.py b/tap_list_providers/management/commands/parseuntappd.py index 51d969af..7c47cb8c 100644 --- a/tap_list_providers/management/commands/parseuntappd.py +++ b/tap_list_providers/management/commands/parseuntappd.py @@ -18,5 +18,6 @@ def handle(self, *args, **options): with transaction.atomic(): for venue in tap_list_provider.get_venues(): self.stdout.write("Processing %s" % venue.name) - tap_list_provider.handle_venue(venue) + timestamp = tap_list_provider.handle_venue(venue) + tap_list_provider.update_venue_timestamps(venue, timestamp) self.stdout.write(self.style.SUCCESS("Done!")) diff --git a/tap_list_providers/parsers/untappd.py b/tap_list_providers/parsers/untappd.py index 6931f592..552f22b1 100644 --- a/tap_list_providers/parsers/untappd.py +++ b/tap_list_providers/parsers/untappd.py @@ -1,5 +1,6 @@ """Parse data from untappd""" import argparse +import datetime from decimal import Decimal from pprint import PrettyPrinter import logging @@ -12,6 +13,7 @@ import configurations from django.core.exceptions import ImproperlyConfigured, AppRegistryNotReady from django.db.models import Q +from pytz import UTC # boilerplate code necessary for launching outside manage.py try: @@ -69,6 +71,7 @@ def handle_venue(self, venue): use_sequential_taps = any( tap_info["tap_number"] is None for tap_info in tap_list ) + latest_timestamp = UTC.localize(datetime.datetime(1970, 1, 1, 12)) for index, tap_info in enumerate(tap_list): # 1. get the tap # if the venue doesn't give tap numbers, just use a 1-based @@ -82,9 +85,13 @@ def handle_venue(self, venue): except KeyError: tap = Tap(venue=venue, tap_number=tap_number) if tap_info["added"]: - tap.time_added = tap_info["added"] + tap.time_added = dateutil.parser.parse(tap_info["added"]) + if tap.time_added > latest_timestamp: + latest_timestamp = tap.time_added if tap_info["updated"]: - tap.time_updated = tap_info["updated"] + tap.time_updated = dateutil.parser.parse(tap_info["updated"]) + if tap.time_updated > latest_timestamp: + latest_timestamp = tap.time_updated # 2. parse the manufacturer try: manufacturer = manufacturers[tap_info["manufacturer"]["name"]] @@ -112,6 +119,7 @@ def handle_venue(self, venue): # 4. assign the beer to the tap tap.beer = beer tap.save() + return latest_timestamp def parse_html_and_js(self, data): # Pull the relevant HTML from the JS. diff --git a/tap_list_providers/test/test_untappd.py b/tap_list_providers/test/test_untappd.py index 40d567bc..2a3c191d 100644 --- a/tap_list_providers/test/test_untappd.py +++ b/tap_list_providers/test/test_untappd.py @@ -5,6 +5,7 @@ from django.core.management import call_command from django.test import TestCase +from django.utils.timezone import now import responses from beers.models import Beer, Manufacturer @@ -42,7 +43,8 @@ def setUpTestData(cls): @responses.activate @mock.patch("tap_list_providers.base.look_up_beer") def test_import_untappd_data(self, mock_beer_lookup): - """Test parsing the JSON data""" + """Test parsing the HTML data""" + timestamp = now() responses.add( responses.GET, UntappdParser.URL.format( @@ -116,6 +118,10 @@ def test_import_untappd_data(self, mock_beer_lookup): price_instance, ) mock_beer_lookup.delay.assert_called_with(tap.beer.id) + self.venue.refresh_from_db() + self.assertIsNotNone(self.venue.tap_list_last_check_time) + self.assertGreater(self.venue.tap_list_last_check_time, timestamp) + self.assertIsNotNone(self.venue.tap_list_last_update_time) class StyleParsingTestCase(TestCase): From c6e438bd445ebf39f60812d7d5efd2280d66200d Mon Sep 17 00:00:00 2001 From: Drew Winstel Date: Thu, 15 Oct 2020 09:54:12 -0500 Subject: [PATCH 10/12] run black --- tap_list_providers/base.py | 11 +++++++++-- .../management/commands/parsebeermenus.py | 2 -- .../management/commands/parsestemandstein.py | 4 +--- tap_list_providers/parsers/stemandstein.py | 12 ++++++------ tap_list_providers/parsers/taphunter.py | 2 +- tap_list_providers/test/test_base.py | 13 ++++++++----- 6 files changed, 25 insertions(+), 19 deletions(-) diff --git a/tap_list_providers/base.py b/tap_list_providers/base.py index bdc0be4e..c2bb644c 100644 --- a/tap_list_providers/base.py +++ b/tap_list_providers/base.py @@ -98,9 +98,16 @@ def handle_venues(self, venues): update_time = self.handle_venue(venue) self.update_venue_timestamps(venue, update_time) - def update_venue_timestamps(self, venue: Venue, update_time: datetime.datetime = None) -> None: + def update_venue_timestamps( + self, venue: Venue, update_time: datetime.datetime = None + ) -> None: """Update the venue last checked and last updated times""" - LOG.debug('Setting check time for %s to %s and update time for to %s', venue, self.check_timestamp, update_time) + LOG.debug( + "Setting check time for %s to %s and update time for to %s", + venue, + self.check_timestamp, + update_time, + ) venue.tap_list_last_check_time = self.check_timestamp if update_time: venue.tap_list_last_update_time = update_time diff --git a/tap_list_providers/management/commands/parsebeermenus.py b/tap_list_providers/management/commands/parsebeermenus.py index 2a61ef1c..bfb6aa09 100644 --- a/tap_list_providers/management/commands/parsebeermenus.py +++ b/tap_list_providers/management/commands/parsebeermenus.py @@ -19,8 +19,6 @@ def handle(self, *args, **options): for venue in tap_list_provider.get_venues(): self.stdout.write("Processing %s" % venue.name) timestamp = tap_list_provider.handle_venue(venue) - - print('timestamp', timestamp) tap_list_provider.update_venue_timestamps(venue, timestamp) self.stdout.write(self.style.SUCCESS("Done!")) diff --git a/tap_list_providers/management/commands/parsestemandstein.py b/tap_list_providers/management/commands/parsestemandstein.py index 1c772968..5b7a1f49 100644 --- a/tap_list_providers/management/commands/parsestemandstein.py +++ b/tap_list_providers/management/commands/parsestemandstein.py @@ -7,9 +7,7 @@ class Command(BaseCommand): - help = ( - "Populates any venues using the Stem and Stein tap list provider with beers" - ) + help = "Populates any venues using the Stem and Stein tap list provider with beers" def add_arguments(self, parser): # does not take any arguments diff --git a/tap_list_providers/parsers/stemandstein.py b/tap_list_providers/parsers/stemandstein.py index c2f32ea9..dc63b786 100644 --- a/tap_list_providers/parsers/stemandstein.py +++ b/tap_list_providers/parsers/stemandstein.py @@ -27,7 +27,7 @@ from taps.models import Tap -CENTRAL_TIME = pytz.timezone('America/Chicago') +CENTRAL_TIME = pytz.timezone("America/Chicago") LOG = logging.getLogger(__name__) @@ -137,8 +137,8 @@ def fill_in_beer_details(self, beer): ).text beer_parser = BeautifulSoup(beer_html, "html.parser") jumbotron = beer_parser.find("div", {"class": "jumbotron"}) - tap_table = beer_parser.find('table', {'id': 'tapList'}) - tap_body = tap_table.find('tbody') + tap_table = beer_parser.find("table", {"id": "tapList"}) + tap_body = tap_table.find("tbody") image_div = jumbotron.find( "div", {"style": "display:table-cell;vertical-align:top;width:17px;"}, @@ -197,9 +197,9 @@ def fill_in_beer_details(self, beer): defaults={"price": price}, ) time_tapped = None - for row in tap_body.find_all('tr'): - cells = list(row.find_all('td')) - if cells[-1].text.endswith('(so far)'): + for row in tap_body.find_all("tr"): + cells = list(row.find_all("td")) + if cells[-1].text.endswith("(so far)"): time_tapped = CENTRAL_TIME.localize(parse(cells[0].text)) return time_tapped diff --git a/tap_list_providers/parsers/taphunter.py b/tap_list_providers/parsers/taphunter.py index eacc0607..42020b3b 100644 --- a/tap_list_providers/parsers/taphunter.py +++ b/tap_list_providers/parsers/taphunter.py @@ -77,7 +77,7 @@ def handle_venue(self, venue): tap = Tap(venue=venue, tap_number=tap_number) tap.time_added = tap_info["added"] tap.time_updated = tap_info["updated"] - parsed_time = parse(tap_info['updated']) + parsed_time = parse(tap_info["updated"]) if parsed_time > latest_timestamp: latest_timestamp = parsed_time if "percent_full" in tap_info: diff --git a/tap_list_providers/test/test_base.py b/tap_list_providers/test/test_base.py index 03f6272a..96a18cb5 100644 --- a/tap_list_providers/test/test_base.py +++ b/tap_list_providers/test/test_base.py @@ -98,11 +98,10 @@ def test_yazoo(self): class TimestampTestCase(TestCase): - def setUp(self): self.venue = VenueFactory() self.provider = BaseTapListProvider() - + def test_initial_conditions(self): self.assertIsNone(self.venue.tap_list_last_check_time) self.assertIsNone(self.venue.tap_list_last_update_time) @@ -111,10 +110,14 @@ def test_initial_conditions(self): def test_timestamp_no_time(self): self.provider.update_venue_timestamps(self.venue, None) self.assertIsNone(self.venue.tap_list_last_update_time) - self.assertEqual(self.venue.tap_list_last_check_time, self.provider.check_timestamp) - + self.assertEqual( + self.venue.tap_list_last_check_time, self.provider.check_timestamp + ) + def test_with_time(self): timestamp = now() - datetime.timedelta(days=1) self.provider.update_venue_timestamps(self.venue, timestamp) self.assertEqual(self.venue.tap_list_last_update_time, timestamp) - self.assertEqual(self.venue.tap_list_last_check_time, self.provider.check_timestamp) + self.assertEqual( + self.venue.tap_list_last_check_time, self.provider.check_timestamp + ) From c5cdf9e3be1b9071fd8ea1045bc7550a97315791 Mon Sep 17 00:00:00 2001 From: Drew Winstel Date: Thu, 15 Oct 2020 09:54:24 -0500 Subject: [PATCH 11/12] drop stray print --- beers/admin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beers/admin.py b/beers/admin.py index ae9996f4..ad136c73 100644 --- a/beers/admin.py +++ b/beers/admin.py @@ -78,7 +78,6 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs): "manufacturer": models.Manufacturer.objects.order_by("name"), } order_qs = fields.get(db_field.name) - print(db_field.name, order_qs is not None) if order_qs: kwargs["queryset"] = order_qs From 04b7c15109d1d113f4d723a67e3a03bd825b9671 Mon Sep 17 00:00:00 2001 From: Drew Winstel Date: Sun, 25 Oct 2020 10:09:00 -0500 Subject: [PATCH 12/12] mark timestamps as read only in serializer --- venues/serializers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/venues/serializers.py b/venues/serializers.py index e4907e62..21697e1b 100644 --- a/venues/serializers.py +++ b/venues/serializers.py @@ -45,7 +45,12 @@ class Meta: # also marking the on_downtown_craft_beer_trail field as read-only # to protect us from ourselves - read_only_fields = ["untappd_url", "on_downtown_craft_beer_trail"] + read_only_fields = [ + "untappd_url", + "on_downtown_craft_beer_trail", + "tap_list_last_check_time", + "tap_list_last_update_time", + ] class VenueBySlugSerializer(VenueSerializer):