Added checks for database size and record count. Added menu badge on …

…status errors. #1125

dsmr_backend/apps.py

@@ -52,3 +52,19 @@ def check_scheduled_processes(**kwargs):
         )
 
     return issues
+
+
+@receiver(request_status)
+def postgresql_check_database_size(**kwargs):  # pragma: nocover
+    import dsmr_backend.services.backend
+
+    pretty_size, bytes_size = dsmr_backend.services.backend.postgresql_total_database_size()
+
+    if bytes_size < settings.DSMRREADER_STATUS_WARN_OVER_EXCESSIVE_DATABASE_SIZE:
+        return
+
+    return MonitoringStatusIssue(
+        __name__,
+        _('Database growing large: {}, consider data cleanup (if not already enabled)').format(pretty_size),
+        timezone.now()
+    )

dsmr_backend/services/backend.py

@@ -1,3 +1,4 @@
+import logging
 from distutils.version import StrictVersion
 import datetime
 
@@ -7,6 +8,7 @@
 from django.db.models import Q
 from django.utils import timezone
 from django.core.cache import cache
+from django.db import connection
 
 from dsmr_backend import signals
 from dsmr_backend.dto import MonitoringStatusIssue
@@ -16,6 +18,9 @@
 from dsmr_weather.models.settings import WeatherSettings
 
 
+logger = logging.getLogger('dsmrreader')
+
+
 def get_capabilities(capability=None):
     """
     Returns the capabilities of the data tracked, such as whether the meter supports gas readings or
@@ -24,7 +29,7 @@ def get_capabilities(capability=None):
     Optionally returns a single capability when requested.
     """
     # Caching time should be limited, but enough to make it matter, as this call is used A LOT.
-    capabilities = cache.get('capabilities')
+    capabilities = cache.get(settings.DSMRREADER_CAPABILITIES_CACHE)
 
     if capabilities is None:
         capabilities = {
@@ -66,7 +71,7 @@ def get_capabilities(capability=None):
             capabilities['electricity_returned'] = False
 
         capabilities['any'] = any(capabilities.values())
-        cache.set('capabilities', capabilities)
+        cache.set(settings.DSMRREADER_CAPABILITIES_CACHE, capabilities)
 
     # Single selection.
     if capability is not None:
@@ -94,6 +99,16 @@ def is_timestamp_passed(timestamp):
     return timezone.now() >= timestamp
 
 
+def request_cached_monitoring_status():
+    cached_monitoring_status = cache.get(settings.DSMRREADER_MONITORING_CACHE)
+
+    if cached_monitoring_status is None:
+        # This will also update the cache.
+        return request_monitoring_status()
+
+    return cached_monitoring_status  # pragma: nocover
+
+
 def request_monitoring_status():
     """ Requests all apps to report any issues for monitoring. """
     responses = signals.request_status.send_robust(None)
@@ -103,6 +118,10 @@ def request_monitoring_status():
         if not current_response:
             continue
 
+        if isinstance(current_response, Exception):
+            logger.warning(current_response)
+            continue
+
         if not isinstance(current_response, (list, tuple)):
             current_response = [current_response]
 
@@ -112,6 +131,9 @@ def request_monitoring_status():
 
     issues = sorted(issues, key=lambda x: x.since, reverse=True)
 
+    # Always invalidate and update cache
+    cache.set(settings.DSMRREADER_MONITORING_CACHE, issues)
+
     return issues
 
 
@@ -139,3 +161,19 @@ def hours_in_day(day):
     # Unchanged
     else:
         return 24
+
+
+def postgresql_total_database_size():  # pragma: nocover
+    if connection.vendor != 'postgresql':
+        return
+
+    with connection.cursor() as cursor:
+        database_name = settings.DATABASES['default']['NAME']
+        size_sql = """
+        SELECT pg_catalog.pg_size_pretty(pg_catalog.pg_database_size(d.datname)) as pretty_size,
+               pg_catalog.pg_database_size(d.datname) as bytes_size
+        FROM pg_catalog.pg_database d
+        WHERE d.datname = %s;
+        """
+        cursor.execute(size_sql, [database_name])
+        return cursor.fetchone()

dsmr_datalogger/apps.py

@@ -38,3 +38,19 @@ def check_recent_readings(**kwargs):
         _('No recent readings received'),
         latest_reading.timestamp
     )
+
+
+@receiver(request_status)
+def check_reading_count(**kwargs):  # pragma: nocover
+    import dsmr_datalogger.services.datalogger
+
+    reading_count = dsmr_datalogger.services.datalogger.postgresql_approximate_reading_count()
+
+    if reading_count < settings.DSMRREADER_STATUS_WARN_OVER_EXCESSIVE_READING_COUNT:
+        return
+
+    return MonitoringStatusIssue(
+        __name__,
+        _('Approximately {} readings stored, consider data cleanup (if not already enabled)').format(reading_count),
+        timezone.now()
+    )

dsmr_datalogger/services/datalogger.py

@@ -2,6 +2,7 @@
 
 from django.db.models.expressions import F
 from django.utils import timezone
+from django.db import connection
 import serial
 
 from dsmr_datalogger.models.reading import DsmrReading
@@ -206,3 +207,17 @@ def _get_dsmrreader_mapping(version):
         })
 
     return mapping
+
+
+def postgresql_approximate_reading_count():  # pragma: nocover
+    if connection.vendor != 'postgresql':
+        return
+
+    # A live count is too slow on huge datasets, this is accurate enough:
+    with connection.cursor() as cursor:
+        cursor.execute(
+            'SELECT reltuples as approximate_row_count FROM pg_class WHERE relname = %s;',
+            [DsmrReading._meta.db_table]
+        )
+        reading_count = cursor.fetchone()[0]
+        return int(reading_count)

dsmr_frontend/context_processors/__init__.py

@@ -1,9 +1,12 @@
 from django.conf import settings
 
+import dsmr_backend.services.backend
+
 
 def version(request):
     return {
         'dsmr_version': settings.DSMRREADER_VERSION,
+        'request_cached_monitoring_status': dsmr_backend.services.backend.request_cached_monitoring_status(),
         'DSMRREADER_MAIN_BRANCH': settings.DSMRREADER_MAIN_BRANCH,
         'LANGUAGE_CODE': request.LANGUAGE_CODE,
     }

dsmr_frontend/templates/dsmr_frontend/base.html

@@ -103,7 +103,14 @@
                         </li>
                         <li>
                             <a href="{% url 'frontend:about' %}">
-                                <i class="fas fa-robot"></i> &nbsp; <small>{% translate "About &amp; support" %}</small>
+                                <i class="fas fa-robot"></i> &nbsp;
+                                <small>
+                                    {% translate "About &amp; support" %}
+
+                                    {% if request_cached_monitoring_status %}
+                                    &nbsp; <span class="badge bg-red">{{ request_cached_monitoring_status|length }}</span>
+                                    {% endif %}
+                                </small>
                             </a>
                         </li>
                         <li>

dsmrreader/config/defaults.py

@@ -62,6 +62,8 @@
 DSMRREADER_STATUS_READING_OFFSET_MINUTES = 30
 
 DSMRREADER_STATUS_MAX_UNPROCESSED_READINGS = 100
+DSMRREADER_STATUS_WARN_OVER_EXCESSIVE_READING_COUNT = 20 * 1000 * 1000  # In millions
+DSMRREADER_STATUS_WARN_OVER_EXCESSIVE_DATABASE_SIZE = 1 * 1000 * 1000 * 1000  # In GB
 
 # The cooldown period until the next status notification will be sent.
 DSMRREADER_STATUS_NOTIFICATION_COOLDOWN_HOURS = 12
@@ -79,3 +81,6 @@
 DSMRREADER_BUIENRADAR_API_URL = 'https://data.buienradar.nl/2.0/feed/json'
 
 DSMRREADER_DATALOGGER_MIN_SLEEP_FOR_RECONNECT = 1.0
+
+DSMRREADER_CAPABILITIES_CACHE = 'capabilities'
+DSMRREADER_MONITORING_CACHE = 'monitoring_status'

dsmrreader/config/test.py

@@ -4,6 +4,10 @@
 from dsmrreader.config.development import *
 
 
+# Cache may cause weird stuff during automated testing.
+for k in CACHES.keys():
+    CACHES[k]['TIMEOUT'] = 0
+
 # Never use this in production!
 SECRET_KEY = secrets.token_hex(64)
 

dsmrreader/locales/nl/LC_MESSAGES/django.po

@@ -75,6 +75,9 @@ msgstr "De database engine \"{}\" wordt niet actief ondersteund, sommige functie
 msgid "Process behind schedule: {}"
 msgstr "Proces loopt achter op schema: {}"
 
+msgid "Database growing large: {}, consider data cleanup (if not already enabled)"
+msgstr "Database groeit gestaag: {}, overweeg dataopschoning (indien nog niet ingeschakeld)"
+
 msgid "Resets the environment for development purposes. Not intended for production."
 msgstr "Reset de omgeving voor ontwikkelingsdoeleinden. Niet bedoeld voor gebruik in productie."
 
@@ -534,6 +537,9 @@ msgstr "Nog nooit een meting ontvangen"
 msgid "No recent readings received"
 msgstr "Geen recente metingen ontvangen"
 
+msgid "Approximately {} readings stored, consider data cleanup (if not already enabled)"
+msgstr "Ongeveer {} metingen opgeslagen, overweeg dataopschoning (indien nog niet ingeschakeld)"
+
 msgid "Performs an DSMR P1 telegram reading on the serial port."
 msgstr "Leest een DSMR P1 telegram uit van de seriële poort (meti."
 

tools/test-all.sh

@@ -15,21 +15,21 @@ echo "OK"
 
 
 DJANGO_DATABASE_HOST=127.0.0.1
-DJANGO_DATABASE_PORT=dsmrreader
+DJANGO_DATABASE_USER=dsmrreader
 DJANGO_DATABASE_PASSWORD=dsmrreader
 # Will be adjusted to 'test_*' by Django.
 DJANGO_DATABASE_NAME=dsmrreader
 
 export DJANGO_DATABASE_HOST
-export DJANGO_DATABASE_PORT
+export DJANGO_DATABASE_USER
 export DJANGO_DATABASE_PASSWORD
 export DJANGO_DATABASE_NAME
 
 
 echo ""
 DJANGO_DATABASE_ENGINE=django.db.backends.sqlite3
 export DJANGO_DATABASE_ENGINE
-echo "--- Testing: DJANGO_DATABASE_ENGINE"
+echo "--- Testing: $DJANGO_DATABASE_ENGINE"
 pytest --cov --cov-report=html --cov-report=term --ds=dsmrreader.config.test -n 2