Store updated and last fetched timestamps for venues (#347)

* Add nullable fields for last venue update/check times

* Add method to base parser to set timestamp based on returned value

* Wire in timestamp parsing for beermenus

* silence pylint complaining about celery

* Wire in timestamp parsing for digitalpour

* Wire in timestamp parsing for stem and stein

* Wire in timestamp parsing for taphunter

* wire in timestamp parsing for taplist.io

* Wire in untappd timestamp parsing

* run black

* drop stray print

* mark timestamps as read only in serializer

beers/admin.py

@@ -78,7 +78,6 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs):
             "manufacturer": models.Manufacturer.objects.order_by("name"),
         }
         order_qs = fields.get(db_field.name)
-        print(db_field.name, order_qs is not None)
         if order_qs:
             kwargs["queryset"] = order_qs
 

tap_list_providers/base.py

@@ -9,10 +9,12 @@
 from decimal import Decimal, InvalidOperation
 from urllib.parse import urlparse, unquote
 import logging
+import datetime
 
 from django.db.models import Prefetch, Q
 from django.db.models.functions import Length
 from django.db import transaction
+from django.utils.timezone import now
 from kombu.exceptions import OperationalError
 
 from venues.models import Venue
@@ -51,12 +53,13 @@
 
 class BaseTapListProvider:
     def __init__(self):
+        self.check_timestamp = now()
         self.styles = {}
         if not hasattr(self, "provider_name"):
             # Don't define this attribute if the child does for us
             self.provider_name = None
 
-    def handle_venue(self, venue):
+    def handle_venue(self, venue: Venue) -> datetime.datetime:
         raise NotImplementedError("You need to implement this yourself")
 
     @classmethod
@@ -92,7 +95,23 @@ def get_venues(self):
     def handle_venues(self, venues):
         for venue in venues:
             LOG.debug("Fetching beers at %s", venue)
-            self.handle_venue(venue)
+            update_time = self.handle_venue(venue)
+            self.update_venue_timestamps(venue, update_time)
+
+    def update_venue_timestamps(
+        self, venue: Venue, update_time: datetime.datetime = None
+    ) -> None:
+        """Update the venue last checked and last updated times"""
+        LOG.debug(
+            "Setting check time for %s to %s and update time for to %s",
+            venue,
+            self.check_timestamp,
+            update_time,
+        )
+        venue.tap_list_last_check_time = self.check_timestamp
+        if update_time:
+            venue.tap_list_last_update_time = update_time
+        venue.save()
 
     def get_style(self, name):
         name = name.strip()
@@ -410,7 +429,7 @@ def get_beer(self, name, manufacturer, pricing=None, venue=None, **defaults):
                 if str(exc).casefold() == "max number of clients reached".casefold():
                     LOG.error("Reached redis limit!")
                     # fall back to doing it synchronously
-                    look_up_beer(beer.id)
+                    look_up_beer(beer.id)  # pylint: disable=no-value-for-parameter
                 else:
                     raise
         return beer

tap_list_providers/management/commands/parsebeermenus.py

@@ -18,5 +18,7 @@ def handle(self, *args, **options):
         with transaction.atomic():
             for venue in tap_list_provider.get_venues():
                 self.stdout.write("Processing %s" % venue.name)
-                tap_list_provider.handle_venue(venue)
+                timestamp = tap_list_provider.handle_venue(venue)
+                tap_list_provider.update_venue_timestamps(venue, timestamp)
+
         self.stdout.write(self.style.SUCCESS("Done!"))

tap_list_providers/management/commands/parsedigitalpour.py

@@ -7,7 +7,7 @@
 
 
 class Command(BaseCommand):
-    help = "Populates any venues using the DigitalPour tap list provider with" " beers"
+    help = "Populates any venues using the DigitalPour tap list provider with beers"
 
     def add_arguments(self, parser):
         # does not take any arguments
@@ -18,5 +18,6 @@ def handle(self, *args, **options):
         with transaction.atomic():
             for venue in tap_list_provider.get_venues():
                 self.stdout.write("Processing %s" % venue.name)
-                tap_list_provider.handle_venue(venue)
+                timestamp = tap_list_provider.handle_venue(venue)
+                tap_list_provider.update_venue_timestamps(venue, timestamp)
         self.stdout.write(self.style.SUCCESS("Done!"))

tap_list_providers/management/commands/parsestemandstein.py

@@ -7,9 +7,7 @@
 
 
 class Command(BaseCommand):
-    help = (
-        "Populates any venues using the Stem and Stein tap list provider with" " beers"
-    )
+    help = "Populates any venues using the Stem and Stein tap list provider with beers"
 
     def add_arguments(self, parser):
         # does not take any arguments
@@ -20,5 +18,6 @@ def handle(self, *args, **options):
         with transaction.atomic():
             for venue in tap_list_provider.get_venues():
                 self.stdout.write("Processing %s" % venue.name)
-                tap_list_provider.handle_venue(venue)
+                timestamp = tap_list_provider.handle_venue(venue)
+                tap_list_provider.update_venue_timestamps(venue, timestamp)
         self.stdout.write(self.style.SUCCESS("Done!"))

tap_list_providers/management/commands/parsetaphunter.py

@@ -7,7 +7,7 @@
 
 
 class Command(BaseCommand):
-    help = "Populates any venues using the TapHunter tap list provider with" " beers"
+    help = "Populates any venues using the TapHunter tap list provider with beers"
 
     def add_arguments(self, parser):
         # does not take any arguments
@@ -18,5 +18,6 @@ def handle(self, *args, **options):
         with transaction.atomic():
             for venue in tap_list_provider.get_venues():
                 self.stdout.write("Processing %s" % venue.name)
-                tap_list_provider.handle_venue(venue)
+                timestamp = tap_list_provider.handle_venue(venue)
+                tap_list_provider.update_venue_timestamps(venue, timestamp)
         self.stdout.write(self.style.SUCCESS("Done!"))

tap_list_providers/management/commands/parsetaplistio.py

@@ -18,5 +18,6 @@ def handle(self, *args, **options):
         with transaction.atomic():
             for venue in tap_list_provider.get_venues():
                 self.stdout.write("Processing %s" % venue.name)
-                tap_list_provider.handle_venue(venue)
+                timestamp = tap_list_provider.handle_venue(venue)
+                tap_list_provider.update_venue_timestamps(venue, timestamp)
         self.stdout.write(self.style.SUCCESS("Done!"))

tap_list_providers/management/commands/parseuntappd.py

@@ -18,5 +18,6 @@ def handle(self, *args, **options):
         with transaction.atomic():
             for venue in tap_list_provider.get_venues():
                 self.stdout.write("Processing %s" % venue.name)
-                tap_list_provider.handle_venue(venue)
+                timestamp = tap_list_provider.handle_venue(venue)
+                tap_list_provider.update_venue_timestamps(venue, timestamp)
         self.stdout.write(self.style.SUCCESS("Done!"))

tap_list_providers/parsers/beermenus.py

@@ -1,4 +1,7 @@
+"""Parser for beermenus dot com"""
+
 from decimal import Decimal
+import datetime
 from dataclasses import dataclass
 import logging
 import os
@@ -71,6 +74,7 @@ def __init__(
         self.categories = categories
         self.soup = None
         self.save_fetched_data = save_fetched_data
+        self.updated_date = None
         super().__init__()
 
     def fetch_data(self) -> str:
@@ -100,14 +104,14 @@ def parse_html(self, data: str) -> List[BeerData]:
         )[0]
         # they just give us a date. I'm going to arbitrarily declare that to be
         # midnight UTC because who cares if we're off by a day
-        updated_date = UTC.localize(
+        self.updated_date = UTC.localize(
             parse(
                 updated_span.text.split()[1],
                 dayfirst=False,
             )
         )
         # TODO save this to the venue
-        LOG.debug("Last updated: %s", updated_date)
+        LOG.debug("Last updated: %s", self.updated_date)
 
         # the beer lists are in <ul>s
         beers = []
@@ -218,7 +222,7 @@ def parse_beers(self, beers: List[BeerData]) -> None:
             beer.brewery_name = brewery_a.text
             beer.brewery_location = brewery_p.text.split(MIDDOT)[1].strip()
 
-    def handle_venue(self, venue: Venue) -> None:
+    def handle_venue(self, venue: Venue) -> datetime.datetime:
         self.categories = venue.api_configuration.beermenus_categories
         self.location_url = self.URL.format(venue.api_configuration.beermenus_slug)
         data = self.fetch_data()
@@ -261,6 +265,7 @@ def handle_venue(self, venue: Venue) -> None:
                     tap_number,
                     beer,
                 )
+        return self.updated_date
 
 
 def parse_beer_tag(tag: Tag) -> BeerData:
@@ -289,7 +294,7 @@ def parse_beer_tag(tag: Tag) -> BeerData:
     )
 
 
-if __name__ == "__main__":
+def main():
     import argparse
 
     LOCATIONS = {
@@ -319,3 +324,7 @@ def parse_beer_tag(tag: Tag) -> BeerData:
 
     for beer in beers:
         print(f"{beer.name} by {beer.brewery_name} ({beer.abv}%, {beer.style})")
+
+
+if __name__ == "__main__":
+    main()

tap_list_providers/parsers/digitalpour.py

@@ -1,3 +1,6 @@
+"""Parser for DigitalPour"""
+
+import datetime
 from decimal import Decimal
 import logging
 import os
@@ -8,6 +11,7 @@
 import configurations
 from django.utils.timezone import now
 from django.core.exceptions import ImproperlyConfigured, AppRegistryNotReady
+from pytz import UTC
 
 # boilerplate code necessary for launching outside manage.py
 try:
@@ -19,6 +23,7 @@
     from ..base import BaseTapListProvider
 
 from taps.models import Tap
+from venues.models import Venue
 
 
 LOG = logging.getLogger(__name__)
@@ -34,16 +39,18 @@ class DigitalPourParser(BaseTapListProvider):
     def __init__(self, location=None):
         """Constructor."""
         self.url = None
+        self.update_date = None
         if location:
             self.url = self.URL.format(location[0], location[1], self.APIKEY)
         super().__init__()
 
-    def handle_venue(self, venue):
+    def handle_venue(self, venue: Venue) -> datetime.datetime:
         venue_id = venue.api_configuration.digital_pour_venue_id
         location_number = venue.api_configuration.digital_pour_location_number
         self.url = self.URL.format(venue_id, location_number, self.APIKEY)
         data = self.fetch()
         taps = {tap.tap_number: tap for tap in venue.taps.all()}
+        self.update_date = UTC.localize(datetime.datetime(1970, 1, 1, 0, 0, 0))
         manufacturers = {}
         for entry in data:
             if not entry["Active"]:
@@ -57,6 +64,8 @@ def handle_venue(self, venue):
                 tap = Tap(venue=venue, tap_number=tap_info["tap_number"])
             tap.time_added = tap_info["added"]
             tap.time_updated = tap_info["updated"]
+            if tap.time_updated and tap.time_updated > self.update_date:
+                self.update_date = tap.time_updated
             tap.estimated_percent_remaining = tap_info["percent_full"]
             if tap_info["gas_type"] in [i[0] for i in Tap.GAS_CHOICES]:
                 tap.gas_type = tap_info["gas_type"]
@@ -116,6 +125,7 @@ def handle_venue(self, venue):
             # 4. assign the beer to the tap
             tap.beer = beer
             tap.save()
+        return self.update_date
 
     def parse_beer(self, entry):
         """Parse beer info from JSON entry."""

tap_list_providers/parsers/stemandstein.py

@@ -1,4 +1,5 @@
 """HTML scraper for The Stem & Stein"""
+import datetime
 from urllib.parse import parse_qsl
 from html.parser import HTMLParser
 from decimal import Decimal
@@ -8,6 +9,8 @@
 from bs4 import BeautifulSoup
 import requests
 import configurations
+import pytz
+from dateutil.parser import parse
 from django.db.models import Q
 from django.core.exceptions import ImproperlyConfigured, AppRegistryNotReady
 
@@ -24,6 +27,7 @@
 from taps.models import Tap
 
 
+CENTRAL_TIME = pytz.timezone("America/Chicago")
 LOG = logging.getLogger(__name__)
 
 
@@ -133,6 +137,8 @@ def fill_in_beer_details(self, beer):
         ).text
         beer_parser = BeautifulSoup(beer_html, "html.parser")
         jumbotron = beer_parser.find("div", {"class": "jumbotron"})
+        tap_table = beer_parser.find("table", {"id": "tapList"})
+        tap_body = tap_table.find("tbody")
         image_div = jumbotron.find(
             "div",
             {"style": "display:table-cell;vertical-align:top;width:17px;"},
@@ -190,6 +196,12 @@ def fill_in_beer_details(self, beer):
             beer=beer,
             defaults={"price": price},
         )
+        time_tapped = None
+        for row in tap_body.find_all("tr"):
+            cells = list(row.find_all("td"))
+            if cells[-1].text.endswith("(so far)"):
+                time_tapped = CENTRAL_TIME.localize(parse(cells[0].text))
+        return time_tapped
 
     def handle_venue(self, venue):
         self.venue = venue
@@ -199,8 +211,11 @@ def handle_venue(self, venue):
         existing_taps = {i.tap_number: i for i in venue.taps.all()}
         LOG.debug("existing taps %s", existing_taps)
         taps_hit = []
+        latest_time = CENTRAL_TIME.localize(datetime.datetime(1970, 1, 1, 0))
         for tap_number, beer in taps.items():
-            self.fill_in_beer_details(beer)
+            time_tapped = self.fill_in_beer_details(beer)
+            if time_tapped > latest_time:
+                latest_time = time_tapped
             try:
                 tap = existing_taps[tap_number]
             except KeyError:
@@ -209,9 +224,11 @@ def handle_venue(self, venue):
                     tap_number=tap_number,
                 )
             tap.beer = beer
+            tap.time_added = time_tapped
             tap.save()
             taps_hit.append(tap.tap_number)
         LOG.debug("Deleting all taps except %s", taps_hit)
         Tap.objects.filter(venue=venue,).exclude(
             tap_number__in=taps_hit,
         ).delete()
+        return latest_time

tap_list_providers/parsers/taphunter.py

@@ -1,13 +1,16 @@
 """Parse beers from TapHunter"""
 import argparse
+import datetime
 import decimal
 import logging
 import os
 import json
 
 import configurations
 import requests
+from dateutil.parser import parse
 from django.core.exceptions import ImproperlyConfigured, AppRegistryNotReady
+from pytz import UTC
 
 # boilerplate code necessary for launching outside manage.py
 try:
@@ -49,6 +52,7 @@ def handle_venue(self, venue):
         use_sequential_taps = any(
             (tap_info["serving_info"]["tap_number"] == "" for tap_info in data["taps"])
         )
+        latest_timestamp = UTC.localize(datetime.datetime(1970, 1, 1, 12))
         for index, entry in enumerate(data["taps"]):
             # 1. parse the tap
             tap_info = self.parse_tap(entry)
@@ -73,6 +77,9 @@ def handle_venue(self, venue):
                 tap = Tap(venue=venue, tap_number=tap_number)
             tap.time_added = tap_info["added"]
             tap.time_updated = tap_info["updated"]
+            parsed_time = parse(tap_info["updated"])
+            if parsed_time > latest_timestamp:
+                latest_timestamp = parsed_time
             if "percent_full" in tap_info:
                 tap.estimated_percent_remaining = tap_info["percent_full"]
             else:
@@ -123,6 +130,7 @@ def handle_venue(self, venue):
             # 4. assign the beer to the tap
             tap.beer = beer
             tap.save()
+        return latest_timestamp
 
     def parse_beer(self, tap):
         beer = {

tap_list_providers/parsers/taplist_io.py

@@ -72,6 +72,7 @@ def handle_venue(self, venue):
             if current_tap.time_updated != timestamp:
                 current_tap.time_updated = timestamp
                 current_tap.save()
+        return timestamp
 
     def parse_tap(self, tap_dict):
         if tap_dict["current_keg"] is None: