diff --git a/dsmr_backend/apps.py b/dsmr_backend/apps.py
index 7b6d09124..504dc66b3 100644
--- a/dsmr_backend/apps.py
+++ b/dsmr_backend/apps.py
@@ -52,3 +52,19 @@ def check_scheduled_processes(**kwargs):
)
return issues
+
+
+@receiver(request_status)
+def postgresql_check_database_size(**kwargs): # pragma: nocover
+ import dsmr_backend.services.backend
+
+ pretty_size, bytes_size = dsmr_backend.services.backend.postgresql_total_database_size()
+
+ if bytes_size < settings.DSMRREADER_STATUS_WARN_OVER_EXCESSIVE_DATABASE_SIZE:
+ return
+
+ return MonitoringStatusIssue(
+ __name__,
+ _('Database growing large: {}, consider data cleanup (if not already enabled)').format(pretty_size),
+ timezone.now()
+ )
diff --git a/dsmr_backend/services/backend.py b/dsmr_backend/services/backend.py
index 7669900a1..f714ae37d 100644
--- a/dsmr_backend/services/backend.py
+++ b/dsmr_backend/services/backend.py
@@ -1,3 +1,4 @@
+import logging
from distutils.version import StrictVersion
import datetime
@@ -7,6 +8,7 @@
from django.db.models import Q
from django.utils import timezone
from django.core.cache import cache
+from django.db import connection
from dsmr_backend import signals
from dsmr_backend.dto import MonitoringStatusIssue
@@ -16,6 +18,9 @@
from dsmr_weather.models.settings import WeatherSettings
+logger = logging.getLogger('dsmrreader')
+
+
def get_capabilities(capability=None):
"""
Returns the capabilities of the data tracked, such as whether the meter supports gas readings or
@@ -24,7 +29,7 @@ def get_capabilities(capability=None):
Optionally returns a single capability when requested.
"""
# Caching time should be limited, but enough to make it matter, as this call is used A LOT.
- capabilities = cache.get('capabilities')
+ capabilities = cache.get(settings.DSMRREADER_CAPABILITIES_CACHE)
if capabilities is None:
capabilities = {
@@ -66,7 +71,7 @@ def get_capabilities(capability=None):
capabilities['electricity_returned'] = False
capabilities['any'] = any(capabilities.values())
- cache.set('capabilities', capabilities)
+ cache.set(settings.DSMRREADER_CAPABILITIES_CACHE, capabilities)
# Single selection.
if capability is not None:
@@ -94,6 +99,16 @@ def is_timestamp_passed(timestamp):
return timezone.now() >= timestamp
+def request_cached_monitoring_status():
+ cached_monitoring_status = cache.get(settings.DSMRREADER_MONITORING_CACHE)
+
+ if cached_monitoring_status is None:
+ # This will also update the cache.
+ return request_monitoring_status()
+
+ return cached_monitoring_status # pragma: nocover
+
+
def request_monitoring_status():
""" Requests all apps to report any issues for monitoring. """
responses = signals.request_status.send_robust(None)
@@ -103,6 +118,10 @@ def request_monitoring_status():
if not current_response:
continue
+ if isinstance(current_response, Exception):
+ logger.warning(current_response)
+ continue
+
if not isinstance(current_response, (list, tuple)):
current_response = [current_response]
@@ -112,6 +131,9 @@ def request_monitoring_status():
issues = sorted(issues, key=lambda x: x.since, reverse=True)
+ # Always invalidate and update cache
+ cache.set(settings.DSMRREADER_MONITORING_CACHE, issues)
+
return issues
@@ -139,3 +161,19 @@ def hours_in_day(day):
# Unchanged
else:
return 24
+
+
+def postgresql_total_database_size(): # pragma: nocover
+ if connection.vendor != 'postgresql':
+ return
+
+ with connection.cursor() as cursor:
+ database_name = settings.DATABASES['default']['NAME']
+ size_sql = """
+ SELECT pg_catalog.pg_size_pretty(pg_catalog.pg_database_size(d.datname)) as pretty_size,
+ pg_catalog.pg_database_size(d.datname) as bytes_size
+ FROM pg_catalog.pg_database d
+ WHERE d.datname = %s;
+ """
+ cursor.execute(size_sql, [database_name])
+ return cursor.fetchone()
diff --git a/dsmr_datalogger/apps.py b/dsmr_datalogger/apps.py
index a1ff11a42..cda572281 100644
--- a/dsmr_datalogger/apps.py
+++ b/dsmr_datalogger/apps.py
@@ -38,3 +38,19 @@ def check_recent_readings(**kwargs):
_('No recent readings received'),
latest_reading.timestamp
)
+
+
+@receiver(request_status)
+def check_reading_count(**kwargs): # pragma: nocover
+ import dsmr_datalogger.services.datalogger
+
+ reading_count = dsmr_datalogger.services.datalogger.postgresql_approximate_reading_count()
+
+ if reading_count < settings.DSMRREADER_STATUS_WARN_OVER_EXCESSIVE_READING_COUNT:
+ return
+
+ return MonitoringStatusIssue(
+ __name__,
+ _('Approximately {} readings stored, consider data cleanup (if not already enabled)').format(reading_count),
+ timezone.now()
+ )
diff --git a/dsmr_datalogger/services/datalogger.py b/dsmr_datalogger/services/datalogger.py
index 77ce57e07..f61813d4c 100644
--- a/dsmr_datalogger/services/datalogger.py
+++ b/dsmr_datalogger/services/datalogger.py
@@ -2,6 +2,7 @@
from django.db.models.expressions import F
from django.utils import timezone
+from django.db import connection
import serial
from dsmr_datalogger.models.reading import DsmrReading
@@ -206,3 +207,17 @@ def _get_dsmrreader_mapping(version):
})
return mapping
+
+
+def postgresql_approximate_reading_count(): # pragma: nocover
+ if connection.vendor != 'postgresql':
+ return
+
+ # A live count is too slow on huge datasets, this is accurate enough:
+ with connection.cursor() as cursor:
+ cursor.execute(
+ 'SELECT reltuples as approximate_row_count FROM pg_class WHERE relname = %s;',
+ [DsmrReading._meta.db_table]
+ )
+ reading_count = cursor.fetchone()[0]
+ return int(reading_count)
diff --git a/dsmr_frontend/context_processors/__init__.py b/dsmr_frontend/context_processors/__init__.py
index 547ca2205..4162fd94f 100644
--- a/dsmr_frontend/context_processors/__init__.py
+++ b/dsmr_frontend/context_processors/__init__.py
@@ -1,9 +1,12 @@
from django.conf import settings
+import dsmr_backend.services.backend
+
def version(request):
return {
'dsmr_version': settings.DSMRREADER_VERSION,
+ 'request_cached_monitoring_status': dsmr_backend.services.backend.request_cached_monitoring_status(),
'DSMRREADER_MAIN_BRANCH': settings.DSMRREADER_MAIN_BRANCH,
'LANGUAGE_CODE': request.LANGUAGE_CODE,
}
diff --git a/dsmr_frontend/templates/dsmr_frontend/base.html b/dsmr_frontend/templates/dsmr_frontend/base.html
index ed311cf98..e02c79ff6 100644
--- a/dsmr_frontend/templates/dsmr_frontend/base.html
+++ b/dsmr_frontend/templates/dsmr_frontend/base.html
@@ -103,7 +103,14 @@
- {% translate "About & support" %}
+
+
+ {% translate "About & support" %}
+
+ {% if request_cached_monitoring_status %}
+ {{ request_cached_monitoring_status|length }}
+ {% endif %}
+
diff --git a/dsmrreader/config/defaults.py b/dsmrreader/config/defaults.py
index 0ce17a7ac..048216eeb 100644
--- a/dsmrreader/config/defaults.py
+++ b/dsmrreader/config/defaults.py
@@ -62,6 +62,8 @@
DSMRREADER_STATUS_READING_OFFSET_MINUTES = 30
DSMRREADER_STATUS_MAX_UNPROCESSED_READINGS = 100
+DSMRREADER_STATUS_WARN_OVER_EXCESSIVE_READING_COUNT = 20 * 1000 * 1000 # In millions
+DSMRREADER_STATUS_WARN_OVER_EXCESSIVE_DATABASE_SIZE = 1 * 1000 * 1000 * 1000 # In GB
# The cooldown period until the next status notification will be sent.
DSMRREADER_STATUS_NOTIFICATION_COOLDOWN_HOURS = 12
@@ -79,3 +81,6 @@
DSMRREADER_BUIENRADAR_API_URL = 'https://data.buienradar.nl/2.0/feed/json'
DSMRREADER_DATALOGGER_MIN_SLEEP_FOR_RECONNECT = 1.0
+
+DSMRREADER_CAPABILITIES_CACHE = 'capabilities'
+DSMRREADER_MONITORING_CACHE = 'monitoring_status'
diff --git a/dsmrreader/config/test.py b/dsmrreader/config/test.py
index ab3b46214..e5563a1c4 100644
--- a/dsmrreader/config/test.py
+++ b/dsmrreader/config/test.py
@@ -4,6 +4,10 @@
from dsmrreader.config.development import *
+# Cache may cause weird stuff during automated testing.
+for k in CACHES.keys():
+ CACHES[k]['TIMEOUT'] = 0
+
# Never use this in production!
SECRET_KEY = secrets.token_hex(64)
diff --git a/dsmrreader/locales/nl/LC_MESSAGES/django.mo b/dsmrreader/locales/nl/LC_MESSAGES/django.mo
index 928bdd591..5e4f67eaa 100644
Binary files a/dsmrreader/locales/nl/LC_MESSAGES/django.mo and b/dsmrreader/locales/nl/LC_MESSAGES/django.mo differ
diff --git a/dsmrreader/locales/nl/LC_MESSAGES/django.po b/dsmrreader/locales/nl/LC_MESSAGES/django.po
index 758042825..fe0cc872d 100644
--- a/dsmrreader/locales/nl/LC_MESSAGES/django.po
+++ b/dsmrreader/locales/nl/LC_MESSAGES/django.po
@@ -75,6 +75,9 @@ msgstr "De database engine \"{}\" wordt niet actief ondersteund, sommige functie
msgid "Process behind schedule: {}"
msgstr "Proces loopt achter op schema: {}"
+msgid "Database growing large: {}, consider data cleanup (if not already enabled)"
+msgstr "Database groeit gestaag: {}, overweeg dataopschoning (indien nog niet ingeschakeld)"
+
msgid "Resets the environment for development purposes. Not intended for production."
msgstr "Reset de omgeving voor ontwikkelingsdoeleinden. Niet bedoeld voor gebruik in productie."
@@ -534,6 +537,9 @@ msgstr "Nog nooit een meting ontvangen"
msgid "No recent readings received"
msgstr "Geen recente metingen ontvangen"
+msgid "Approximately {} readings stored, consider data cleanup (if not already enabled)"
+msgstr "Ongeveer {} metingen opgeslagen, overweeg dataopschoning (indien nog niet ingeschakeld)"
+
msgid "Performs an DSMR P1 telegram reading on the serial port."
msgstr "Leest een DSMR P1 telegram uit van de seriële poort (meti."
diff --git a/tools/test-all.sh b/tools/test-all.sh
index fc2a46613..d6a4e9dd0 100755
--- a/tools/test-all.sh
+++ b/tools/test-all.sh
@@ -15,13 +15,13 @@ echo "OK"
DJANGO_DATABASE_HOST=127.0.0.1
-DJANGO_DATABASE_PORT=dsmrreader
+DJANGO_DATABASE_USER=dsmrreader
DJANGO_DATABASE_PASSWORD=dsmrreader
# Will be adjusted to 'test_*' by Django.
DJANGO_DATABASE_NAME=dsmrreader
export DJANGO_DATABASE_HOST
-export DJANGO_DATABASE_PORT
+export DJANGO_DATABASE_USER
export DJANGO_DATABASE_PASSWORD
export DJANGO_DATABASE_NAME
@@ -29,7 +29,7 @@ export DJANGO_DATABASE_NAME
echo ""
DJANGO_DATABASE_ENGINE=django.db.backends.sqlite3
export DJANGO_DATABASE_ENGINE
-echo "--- Testing: DJANGO_DATABASE_ENGINE"
+echo "--- Testing: $DJANGO_DATABASE_ENGINE"
pytest --cov --cov-report=html --cov-report=term --ds=dsmrreader.config.test -n 2