DBC22-1857 Augment task to copy webcam images locally

.env.example

@@ -69,3 +69,8 @@ DRIVEBC_WEATHER_API_TOKEN_URL=<weather api token url>
 DRIVEBC_WEATHER_AREAS_API_BASE_URL=<weather areas api base url>
 WEATHER_CLIENT_ID=<weather client id>
 WEATHER_CLIENT_SECRET=<weather client secret>
+
+# TESTS
+# optional: set to config.test.DbcRunner to user test runner allowing for
+# skipping db creation entirely
+TEST_RUNNER=config.test.DbcRunner

docker-compose.yml

@@ -9,6 +9,7 @@ x-function: &backend
 
   volumes:
     - ./src/backend:/app/backend
+    - ./src/images/webcams:/app/images/webcams
 
   stdin_open: true
   tty: true

src/backend/apps/cms/models.py

@@ -58,7 +58,7 @@ class Bulletin(Page, BaseModel):
     page_body = "Use this page for creating bulletins."
     teaser = models.CharField(max_length=250, blank=True)
     body = RichTextField()
-    image = models.ForeignKey(Image, on_delete=models.CASCADE, null=True, blank=False)
+    image = models.ForeignKey(Image, on_delete=models.SET_NULL, null=True, blank=False)
     image_alt_text = models.CharField(max_length=125, default='', blank=False)
 
     def rendered_body(self):
@@ -93,7 +93,7 @@ class Ferry(Page, BaseModel):
     location = models.GeometryField(blank=True, null=True)
 
     url = models.URLField(blank=True)
-    image = models.ForeignKey(Image, on_delete=models.CASCADE, blank=True, null=True)
+    image = models.ForeignKey(Image, on_delete=models.SET_NULL, blank=True, null=True)
 
     description = RichTextField(max_length=750, blank=True)
     seasonal_description = RichTextField(max_length=100, blank=True)

src/backend/apps/event/models.py

@@ -34,7 +34,7 @@ class Event(BaseModel):
     last_updated = models.DateTimeField()
 
     # Schedule
-    schedule = models.JSONField(default={})
+    schedule = models.JSONField(default=dict)
 
     # Scheduled start and end
     start = models.DateTimeField(null=True)

src/backend/apps/feed/serializers.py

@@ -27,6 +27,8 @@ class WebcamFeedSerializer(serializers.Serializer):
     # Description
     camName = DriveBCField('name', source="*")
     caption = serializers.CharField(max_length=256, allow_blank=True, allow_null=True)
+    dbcMark = DriveBCField('dbc_mark', source="*")
+    credit = DriveBCField('credit', source="*")
 
     # Location
     region = WebcamRegionField(source="*")

src/backend/apps/feed/tasks.py

@@ -31,6 +31,7 @@ def populate_ferry_task():
 def publish_scheduled():
     call_command('publish_scheduled')
 
+
 @db_periodic_task(crontab(minute="*/5"))
 def populate_regional_weather_task():
     populate_all_regional_weather_data()

src/backend/apps/shared/tests.py

@@ -11,7 +11,7 @@
 
 from rest_framework.test import APIRequestFactory
 
-from src.backend.apps.shared.views import FeedbackView
+from apps.shared.views import FeedbackView
 
 logger = logging.getLogger(__name__)
 
@@ -52,4 +52,4 @@ def tearDown(self):
         Event.objects.all().delete()
         Ferry.objects.all().delete()
         RegionalWeather.objects.all().delete()
-    
+

src/backend/apps/weather/tests/test_regional_weather_populate.py

@@ -10,7 +10,7 @@
     populate_all_regional_weather_data,
 )
 from apps.weather.tests.test_data.regional_weather_parsed_feed import json_feed
-from src.backend.apps.feed.client import FeedClient
+from apps.feed.client import FeedClient
 from unittest import mock
 
 class TestRegionalWeatherModel(BaseTest):
@@ -36,7 +36,7 @@ def test_populate_regional_weather_function(self):
         regional_weather_one = RegionalWeather.objects.get(code="s0000341")
         assert regional_weather_one.location_latitude == \
                "58.66N"
-        
+
     def test_populate_regional_weather_function_with_existing_data(self):
         RegionalWeather.objects.create(
             code="s0000341",
@@ -48,7 +48,7 @@ def test_populate_regional_weather_function_with_existing_data(self):
         assert regional_weather_one.location_latitude == \
                "58.66N"
 
-    @patch('src.backend.apps.feed.client.FeedClient.get_regional_weather_list')
+    @patch('apps.feed.client.FeedClient.get_regional_weather_list')
     def test_populate_and_update_regional_weather(self, mock_requests_get):
         mock_requests_get.side_effect = [
             MockResponse(self.mock_regional_weather_feed_result, status_code=200),
@@ -62,7 +62,7 @@ def test_populate_and_update_regional_weather(self, mock_requests_get):
             populate_regional_weather_from_data(regional_weather_data)
         weather_list_length = len(response)
         assert weather_list_length == 2
-        
+
     def test_populate_all_regional_weather_data(self):
         with mock.patch('requests.post') as mock_post, mock.patch('requests.get') as mock_get:
             # Mock the response for requests.post
@@ -80,6 +80,6 @@ def test_populate_all_regional_weather_data(self):
                 # Mock the response for the second requests.get (weather data for a specific area codes)
                 mock.Mock(json=lambda: self.mock_regional_weather_feed_result_weather_one),
             ]
-            
+
             populate_all_regional_weather_data()
             assert len(RegionalWeather.objects.all()) == 1

src/backend/apps/webcam/management/commands/update_webcam.py

@@ -0,0 +1,13 @@
+from apps.webcam.tasks import update_single_webcam_data
+from django.core.management.base import BaseCommand
+
+from apps.webcam.models import Webcam
+
+class Command(BaseCommand):
+
+    def add_arguments(self , parser):
+        parser.add_argument('id' , nargs='?' , type=int)
+
+    def handle(self, id, *args, **options):
+        cam = Webcam.objects.get(id=id)
+        update_single_webcam_data(cam)

src/backend/apps/webcam/tasks.py

@@ -1,7 +1,20 @@
 import datetime
+import io
 import logging
+from math import floor
+import os
+from pprint import pprint
+import re
+import urllib.request
+from zoneinfo import ZoneInfo
+
+from django.conf import settings
+from PIL import Image, ImageDraw, ImageFont
+from PIL.ExifTags import TAGS
+from pathlib import Path
 
 import pytz
+
 from apps.feed.client import FeedClient
 from apps.webcam.enums import CAMERA_DIFF_FIELDS
 from apps.webcam.models import Webcam
@@ -11,6 +24,10 @@
 
 logger = logging.getLogger(__name__)
 
+APP_DIR = Path(__file__).resolve().parent
+FONT = ImageFont.truetype(f'{APP_DIR}/static/BCSans.otf', size=14)
+CAMS_DIR = f'{settings.SRC_DIR}/images/webcams'
+
 
 def populate_webcam_from_data(webcam_data):
     webcam_id = webcam_data.get("id")
@@ -44,6 +61,7 @@ def update_single_webcam_data(webcam):
             webcam_serializer = WebcamSerializer(webcam, data=webcam_data)
             webcam_serializer.is_valid(raise_exception=True)
             webcam_serializer.save()
+            update_webcam_image(webcam_data)
             return
 
 
@@ -55,3 +73,70 @@ def update_all_webcam_data():
 
     # Rebuild cache
     WebcamAPI().set_list_data()
+
+
+def update_webcam_image(webcam):
+    '''
+    Retrieve the current cam image, stamp it and save it
+
+    Per JIRA ticket DBC22-1857
+
+    '''
+
+    try:
+        # retrieve source image into a file-like buffer
+        base_url = settings.DRIVEBC_WEBCAM_API_BASE_URL
+        if base_url[-1] == '/':
+            base_url = base_url[:-1]
+        endpoint = f'{base_url}/webcams/{webcam["id"]}/imageSource'
+
+        with urllib.request.urlopen(endpoint) as url:
+            image_data = io.BytesIO(url.read())
+
+        raw = Image.open(image_data)
+        width, height = raw.size
+        if width > 800:
+            ratio = 800 / width
+            width = 800
+            height = floor(height * ratio)
+            raw = raw.resize((width, height))
+
+        stamped = Image.new('RGB', (width, height + 18))
+        stamped.paste(raw)  # leaves 18 pixel black bar left at bottom
+
+        # add mark and timestamp to black bar
+        pen = ImageDraw.Draw(stamped)
+        mark = webcam.get('dbc_mark', '')
+        pen.text((3,  height + 14), mark, fill="white", anchor='ls', font=FONT)
+
+        lastmod = webcam.get('last_update_modified')
+        timestamp = 'Last modification time unavailable'
+        if lastmod is not None:
+            month = lastmod.strftime('%b')
+            day = lastmod.strftime('%d')
+            day = day[1:] if day[:1] == '0' else day  # strip leading zero
+            timestamp = f'{month} {day}, {lastmod.strftime("%Y %H:%M:%S %p %Z")}'
+        pen.text((width - 3,  height + 14), timestamp, fill="white", anchor='rs', font=FONT)
+
+        # save image in shared volume
+        filename = f'{CAMS_DIR}/{webcam["id"]}.jpg'
+        with open(filename, 'wb') as saved:
+            stamped.save(saved, 'jpeg', quality=95, exif=raw.getexif())
+
+        # Set the last modified time to the last modified time plus a timedelta
+        # calculated as 110% of the mean time between updates, minus the standard
+        # deviation.  If that can't be calculated, default to 5 minutes.  This is
+        # then used to set the expires header in nginx.
+        delta = 300  # 5 minutes
+        try:
+            mean = webcam.get('update_period_mean')
+            stddev = webcam.get('update_period_stddev', 0)
+            delta = (1.1 * mean) - stddev
+        except:
+            pass  # update period times not available
+        delta = datetime.timedelta(seconds=delta)
+        if lastmod is not None:
+            lastmod = floor((lastmod + delta).timestamp())  # POSIX timestamp
+            os.utime(filename, times=(lastmod, lastmod))
+    except Exception as e:
+        logger.error(e)

src/backend/apps/webcam/tests/test_webcam_update_images.py

@@ -0,0 +1,20 @@
+import datetime
+import json
+import zoneinfo
+from pathlib import Path
+from unittest.mock import patch
+
+from apps.shared import enums as shared_enums
+from apps.shared.enums import CacheKey
+from apps.shared.tests import BaseTest, MockResponse
+from apps.webcam.models import Webcam
+from apps.webcam.views import WebcamAPI
+from django.contrib.gis.geos import Point
+from django.core.cache import cache
+from rest_framework.test import APITestCase
+
+
+class TestWebcamUpdateImages(BaseTest):
+
+    def test_webcam_image_update(self):
+        pass

src/backend/config/settings/django.py

@@ -179,3 +179,5 @@
         "level": "WARNING",
     },
 }
+
+TEST_RUNNER = env("TEST_RUNNER", default="django.test.runner.DiscoverRunner")

src/backend/config/settings/third_party.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 
 import environ
@@ -32,6 +33,12 @@
 # Wagtail
 WAGTAIL_SITE_NAME = 'DriveBC'
 WAGTAILEMBEDS_RESPONSIVE_HTML = True
+WAGTAILADMIN_BASE_URL = "\\cms-admin"
 
 # reCAPTCHA
 DRF_RECAPTCHA_SECRET_KEY = env("DJANGO_RECAPTCHA_SECRET_KEY")
+
+# On windows, GDAL and GEOS require explicit paths to the dlls
+if os.name == "nt":
+    GEOS_LIBRARY_PATH = env("GEOS_LIBRARY_PATH")
+    GDAL_LIBRARY_PATH = env("GDAL_LIBRARY_PATH")

src/backend/config/test.py

@@ -0,0 +1,28 @@
+from argparse import ArgumentParser
+from typing import Any
+from django.db.backends.base.base import BaseDatabaseWrapper
+from django.test.runner import DiscoverRunner
+
+
+class DbcRunner(DiscoverRunner):
+    '''Custom test runner to allow for skipping database creation altogether.'''
+
+    def __init__(self, *args, **kwargs):
+        self.skipdb = kwargs["skipdb"]
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def add_arguments(cls, parser: ArgumentParser) -> None:
+        parser.add_argument("--skipdb", action="store_true", default=False,
+                            help="Don't create a test database")
+        DiscoverRunner.add_arguments(parser)
+
+    def setup_databases(self, **kwargs: Any) -> list[tuple[BaseDatabaseWrapper, str, bool]]:
+        if self.skipdb:
+            return
+        return super().setup_databases(**kwargs)
+
+    def teardown_databases(self, old_config: list[tuple[BaseDatabaseWrapper, str, bool]], **kwargs: Any) -> None:
+        if self.skipdb:
+            return
+        return super().teardown_databases(old_config, **kwargs)