Ability to detect bad measurements in transactions

The Grizzl-e charger sometimes records incorrect meter values, which
have the common feature of an impossibly large jump up from the previous
recorded value. This change adds fields to detect and track these incorrect
measurements.

.gitignore

@@ -2,3 +2,4 @@
 *.iml
 __pycache__
 /test_messages.txt
+.DS_Store

README.md

@@ -47,7 +47,7 @@ docker compose run --rm be poetry run python manage.py createsuperuser
 
 Requirements:
 * Docker
-* Python 3
+* Python 3.11+ w/ Poetry 1.7+
 
 ### Install services
 

be/ocpp/management/commands/correct_meter_values.py

@@ -0,0 +1,86 @@
+import csv
+import datetime
+import logging
+import sys
+
+import pytz
+from django.core.management.base import BaseCommand
+from ocpp.models import Transaction
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+ENERGY_MAX_JUMP = 10000
+DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+
+class Command(BaseCommand):
+    """Detect and correct unrealistic jumps between meter readings"""
+
+    def add_arguments(self, parser):
+        parser.add_argument("start", type=datetime.datetime.fromisoformat)
+        parser.add_argument("end", type=datetime.datetime.fromisoformat)
+        parser.add_argument("--tz", type=str, default="America/Vancouver")
+        parser.add_argument("--dry-run", action="store_true")
+
+    def handle(self, *args, **options):
+        csv_writer = csv.writer(sys.stdout)
+        tz = pytz.timezone(options["tz"])
+        options["start"] = tz.localize(options["start"])
+        options["end"] = tz.localize(options["end"])
+        transactions = Transaction.objects.filter(
+            stopped_at__gte=options["start"], stopped_at__lt=options["end"]
+        ).order_by("started_at")
+        csv_writer.writerow(
+            [
+                "timestamp",
+                "charge_point",
+                "transaction.id",
+                "meter_value.id",
+                "meter_value.prev",
+                "meter_value.cur",
+                "meter_value.delta",
+                "transaction.meter_stop",
+                "transaction.meter_correction",
+            ]
+        )
+        for transaction in transactions:
+            meter_correction = 0
+            report_rows = []
+            prev = None
+            for cur in transaction.metervalue_set.filter(
+                measurand="Energy.Active.Import.Register"
+            ).order_by("timestamp"):
+                if (
+                    prev
+                    and prev.value
+                    and cur.value
+                    and cur.value - prev.value > ENERGY_MAX_JUMP
+                ):
+                    delta_value = cur.value - prev.value
+                    report_rows.append(
+                        [
+                            cur.timestamp.astimezone(tz).strftime(DATETIME_FORMAT),
+                            transaction.charge_point,
+                            transaction.id,
+                            cur.id,
+                            round(prev.value, 2),
+                            round(cur.value, 2),
+                            round(delta_value, 2),
+                        ]
+                    )
+                    cur.is_incorrect = True
+                    if not options["dry_run"]:
+                        cur.save(update_fields=["is_incorrect"])
+                    meter_correction += delta_value
+                prev = cur
+            if meter_correction:
+                transaction.meter_correction = -meter_correction
+                for row in report_rows:
+                    row += [
+                        round(transaction.meter_stop, 2),
+                        round(transaction.meter_correction, 2),
+                    ]
+                    csv_writer.writerow(row)
+                if not options["dry_run"]:
+                    transaction.save(update_fields=["meter_correction"])

be/ocpp/management/commands/run_report.py

@@ -5,13 +5,12 @@
 
 import pytz
 from django.core.management.base import BaseCommand
-
 from ocpp.models import Transaction
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 
-DATETIME_FORMAT = "'%Y-%m-%d %H:%M:%S"
+DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
 
 
 class Command(BaseCommand):
@@ -36,7 +35,8 @@ def handle(self, *args, **options):
                     "charge_point",
                     "started_at",
                     "stopped_at",
-                    "meter (Wh)",
+                    "meter",
+                    "meter_correction",
                     "stop_reason",
                 ]
             )
@@ -48,8 +48,11 @@ def handle(self, *args, **options):
                         tx.started_at.astimezone(tz).strftime(DATETIME_FORMAT),
                         tx.stopped_at.astimezone(tz).strftime(DATETIME_FORMAT),
                         tx.meter_stop,
+                        tx.meter_correction,
                         tx.stop_reason,
                     ]
                 )
+        elif options["report_type"] == "message":
+            pass
         else:
             raise ValueError("Unknown report type")

be/ocpp/migrations/0007_metervalue_is_incorrect_transaction_meter_correction.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.0.1 on 2024-01-12 09:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ocpp", "0006_chargepoint_hw_imsi"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="metervalue",
+            name="is_incorrect",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="transaction",
+            name="meter_correction",
+            field=models.IntegerField(default=0),
+        ),
+    ]

be/ocpp/models/meter_value.py

@@ -1,7 +1,5 @@
 import dateutil.parser
-
 from django.db import models
-
 from ocpp.models.transaction import Transaction
 
 
@@ -16,6 +14,7 @@ class MeterValue(models.Model):
     unit = models.CharField(max_length=16)
     value = models.FloatField()
     is_final = models.BooleanField(default=False)
+    is_incorrect = models.BooleanField(default=False)
 
     @staticmethod
     def create_from_json(

be/ocpp/models/transaction.py

@@ -1,5 +1,4 @@
 from django.db import models
-
 from ocpp.models.charge_point import ChargePoint
 from ocpp.types.stop_reason import StopReason
 from ocpp.utils.date import utc_now
@@ -13,6 +12,7 @@ class Transaction(models.Model):
     id_tag = models.CharField(max_length=256)
     meter_start = models.IntegerField(default=0)
     meter_stop = models.IntegerField(default=0)
+    meter_correction = models.IntegerField(default=0)
     stop_reason = models.CharField(
         max_length=64, choices=StopReason.choices(), null=True, blank=True
     )

be/ocpp/scripts/pg_restore.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+pg_restore --clean -h 127.0.0.1 -v -U levity -d levity -j 2 $1

be/pyproject.toml

@@ -1,12 +1,11 @@
 [tool.poetry]
-name = "levity-be"
+name = "levity"
 version = "0.1.0"
 description = ""
-authors = ["Your Name <you@example.com>"]
-readme = "README.md"
+authors = ["Keith Grennan"]
 
 [tool.poetry.dependencies]
-python = "^3.10"
+python = "^3.11"
 pika = "^1.3.1"
 python-dateutil = "^2.8.2"
 dj-database-url = "^1.3.0"
@@ -17,6 +16,7 @@ prometheus-client = "^0.16.0"
 pyyaml = "6.0.1"
 pytz = "^2023.3"
 fluent-logger = "^0.10.0"
+django = "^5.0.1"
 
 
 [tool.poetry.group.dev.dependencies]

rabbitmq/Dockerfile

@@ -1,4 +1,4 @@
-FROM rabbitmq:3-alpine
+FROM rabbitmq:3.11-alpine
 RUN apk add curl
 RUN curl -L https://github.com/rabbitmq/rabbitmq-delayed-message-exchange/releases/download/3.11.1/rabbitmq_delayed_message_exchange-3.11.1.ez \
     -o /opt/rabbitmq/plugins/rabbitmq_delayed_message_exchange.ez