From 956bd4f71a47f70a6e75bc868e50fa47263abcfa Mon Sep 17 00:00:00 2001 From: Keith Grennan Date: Sun, 10 Mar 2024 18:19:43 -0700 Subject: [PATCH] Ability to detect bad measurements in transactions The Grizzl-e charger sometimes records incorrect meter values, which have the common feature of an impossibly large jump up from the previous recorded value. This change adds fields to detect and track these incorrect measurements. --- .gitignore | 1 + README.md | 2 +- .../commands/correct_meter_values.py | 86 +++++++++++++++++++ be/ocpp/management/commands/run_report.py | 9 +- ..._incorrect_transaction_meter_correction.py | 22 +++++ be/ocpp/models/meter_value.py | 3 +- be/ocpp/models/transaction.py | 2 +- be/ocpp/scripts/pg_restore.sh | 3 + be/poetry.lock | 62 +++++-------- be/pyproject.toml | 8 +- rabbitmq/Dockerfile | 2 +- 11 files changed, 148 insertions(+), 52 deletions(-) create mode 100644 be/ocpp/management/commands/correct_meter_values.py create mode 100644 be/ocpp/migrations/0007_metervalue_is_incorrect_transaction_meter_correction.py create mode 100644 be/ocpp/scripts/pg_restore.sh diff --git a/.gitignore b/.gitignore index 356baf8..0a7302a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.iml __pycache__ /test_messages.txt +.DS_Store diff --git a/README.md b/README.md index a6789c0..28524e9 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ docker compose run --rm be poetry run python manage.py createsuperuser Requirements: * Docker -* Python 3 +* Python 3.11+ w/ Poetry 1.7+ ### Install services diff --git a/be/ocpp/management/commands/correct_meter_values.py b/be/ocpp/management/commands/correct_meter_values.py new file mode 100644 index 0000000..e6eb9ac --- /dev/null +++ b/be/ocpp/management/commands/correct_meter_values.py @@ -0,0 +1,86 @@ +import csv +import datetime +import logging +import sys + +import pytz +from django.core.management.base import BaseCommand +from ocpp.models import Transaction + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +ENERGY_MAX_JUMP = 10000 +DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" + + +class Command(BaseCommand): + """Detect and correct unrealistic jumps between meter readings""" + + def add_arguments(self, parser): + parser.add_argument("start", type=datetime.datetime.fromisoformat) + parser.add_argument("end", type=datetime.datetime.fromisoformat) + parser.add_argument("--tz", type=str, default="America/Vancouver") + parser.add_argument("--dry-run", action="store_true") + + def handle(self, *args, **options): + csv_writer = csv.writer(sys.stdout) + tz = pytz.timezone(options["tz"]) + options["start"] = tz.localize(options["start"]) + options["end"] = tz.localize(options["end"]) + transactions = Transaction.objects.filter( + stopped_at__gte=options["start"], stopped_at__lt=options["end"] + ).order_by("started_at") + csv_writer.writerow( + [ + "timestamp", + "charge_point", + "transaction.id", + "meter_value.id", + "meter_value.prev", + "meter_value.cur", + "meter_value.delta", + "transaction.meter_stop", + "transaction.meter_correction", + ] + ) + for transaction in transactions: + meter_correction = 0 + report_rows = [] + prev = None + for cur in transaction.metervalue_set.filter( + measurand="Energy.Active.Import.Register" + ).order_by("timestamp"): + if ( + prev + and prev.value + and cur.value + and cur.value - prev.value > ENERGY_MAX_JUMP + ): + delta_value = cur.value - prev.value + report_rows.append( + [ + cur.timestamp.astimezone(tz).strftime(DATETIME_FORMAT), + transaction.charge_point, + transaction.id, + cur.id, + round(prev.value, 2), + round(cur.value, 2), + round(delta_value, 2), + ] + ) + cur.is_incorrect = True + if not options["dry_run"]: + cur.save(update_fields=["is_incorrect"]) + meter_correction += delta_value + prev = cur + if meter_correction: + transaction.meter_correction = -meter_correction + for row in report_rows: + row += [ + round(transaction.meter_stop, 2), + round(transaction.meter_correction, 2), + ] + csv_writer.writerow(row) + if not options["dry_run"]: + transaction.save(update_fields=["meter_correction"]) diff --git a/be/ocpp/management/commands/run_report.py b/be/ocpp/management/commands/run_report.py index 6032ee2..7b6ee45 100644 --- a/be/ocpp/management/commands/run_report.py +++ b/be/ocpp/management/commands/run_report.py @@ -5,13 +5,12 @@ import pytz from django.core.management.base import BaseCommand - from ocpp.models import Transaction logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) -DATETIME_FORMAT = "'%Y-%m-%d %H:%M:%S" +DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" class Command(BaseCommand): @@ -36,7 +35,8 @@ def handle(self, *args, **options): "charge_point", "started_at", "stopped_at", - "meter (Wh)", + "meter", + "meter_correction", "stop_reason", ] ) @@ -48,8 +48,11 @@ def handle(self, *args, **options): tx.started_at.astimezone(tz).strftime(DATETIME_FORMAT), tx.stopped_at.astimezone(tz).strftime(DATETIME_FORMAT), tx.meter_stop, + tx.meter_correction, tx.stop_reason, ] ) + elif options["report_type"] == "message": + pass else: raise ValueError("Unknown report type") diff --git a/be/ocpp/migrations/0007_metervalue_is_incorrect_transaction_meter_correction.py b/be/ocpp/migrations/0007_metervalue_is_incorrect_transaction_meter_correction.py new file mode 100644 index 0000000..257aee5 --- /dev/null +++ b/be/ocpp/migrations/0007_metervalue_is_incorrect_transaction_meter_correction.py @@ -0,0 +1,22 @@ +# Generated by Django 5.0.1 on 2024-01-12 09:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("ocpp", "0006_chargepoint_hw_imsi"), + ] + + operations = [ + migrations.AddField( + model_name="metervalue", + name="is_incorrect", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="transaction", + name="meter_correction", + field=models.IntegerField(default=0), + ), + ] diff --git a/be/ocpp/models/meter_value.py b/be/ocpp/models/meter_value.py index 80893b1..3ec85b3 100644 --- a/be/ocpp/models/meter_value.py +++ b/be/ocpp/models/meter_value.py @@ -1,7 +1,5 @@ import dateutil.parser - from django.db import models - from ocpp.models.transaction import Transaction @@ -16,6 +14,7 @@ class MeterValue(models.Model): unit = models.CharField(max_length=16) value = models.FloatField() is_final = models.BooleanField(default=False) + is_incorrect = models.BooleanField(default=False) @staticmethod def create_from_json( diff --git a/be/ocpp/models/transaction.py b/be/ocpp/models/transaction.py index ace3c1e..8f048b5 100644 --- a/be/ocpp/models/transaction.py +++ b/be/ocpp/models/transaction.py @@ -1,5 +1,4 @@ from django.db import models - from ocpp.models.charge_point import ChargePoint from ocpp.types.stop_reason import StopReason from ocpp.utils.date import utc_now @@ -13,6 +12,7 @@ class Transaction(models.Model): id_tag = models.CharField(max_length=256) meter_start = models.IntegerField(default=0) meter_stop = models.IntegerField(default=0) + meter_correction = models.IntegerField(default=0) stop_reason = models.CharField( max_length=64, choices=StopReason.choices(), null=True, blank=True ) diff --git a/be/ocpp/scripts/pg_restore.sh b/be/ocpp/scripts/pg_restore.sh new file mode 100644 index 0000000..136dacf --- /dev/null +++ b/be/ocpp/scripts/pg_restore.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +pg_restore --clean -h 127.0.0.1 -v -U levity -d levity -j 2 $1 \ No newline at end of file diff --git a/be/poetry.lock b/be/poetry.lock index 7ed880d..c27c2da 100644 --- a/be/poetry.lock +++ b/be/poetry.lock @@ -1,14 +1,14 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "asgiref" -version = "3.6.0" +version = "3.7.2" description = "ASGI specs, helper code, and adapters" optional = false python-versions = ">=3.7" files = [ - {file = "asgiref-3.6.0-py3-none-any.whl", hash = "sha256:71e68008da809b957b7ee4b43dbccff33d1b23519fb8344e33f049897077afac"}, - {file = "asgiref-3.6.0.tar.gz", hash = "sha256:9567dfe7bd8d3c8c892227827c41cce860b368104c3431da67a0c5a65a949506"}, + {file = "asgiref-3.7.2-py3-none-any.whl", hash = "sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e"}, + {file = "asgiref-3.7.2.tar.gz", hash = "sha256:9e0ce3aa93a819ba5b45120216b23878cf6e8525eb3848653452b4192b92afed"}, ] [package.extras] @@ -72,7 +72,6 @@ mypy-extensions = ">=0.4.3" packaging = ">=22.0" pathspec = ">=0.9.0" platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -155,38 +154,24 @@ typing-extensions = ">=3.10.0.0" [[package]] name = "django" -version = "4.1.7" +version = "5.0.1" description = "A high-level Python web framework that encourages rapid development and clean, pragmatic design." optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" files = [ - {file = "Django-4.1.7-py3-none-any.whl", hash = "sha256:f2f431e75adc40039ace496ad3b9f17227022e8b11566f4b363da44c7e44761e"}, - {file = "Django-4.1.7.tar.gz", hash = "sha256:44f714b81c5f190d9d2ddad01a532fe502fa01c4cb8faf1d081f4264ed15dcd8"}, + {file = "Django-5.0.1-py3-none-any.whl", hash = "sha256:f47a37a90b9bbe2c8ec360235192c7fddfdc832206fcf618bb849b39256affc1"}, + {file = "Django-5.0.1.tar.gz", hash = "sha256:8c8659665bc6e3a44fefe1ab0a291e5a3fb3979f9a8230be29de975e57e8f854"}, ] [package.dependencies] -asgiref = ">=3.5.2,<4" -sqlparse = ">=0.2.2" +asgiref = ">=3.7.0,<4" +sqlparse = ">=0.3.1" tzdata = {version = "*", markers = "sys_platform == \"win32\""} [package.extras] argon2 = ["argon2-cffi (>=19.1.0)"] bcrypt = ["bcrypt"] -[[package]] -name = "exceptiongroup" -version = "1.1.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, - {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, -] - -[package.extras] -test = ["pytest (>=6)"] - [[package]] name = "factory-boy" version = "3.2.1" @@ -661,11 +646,9 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] @@ -725,6 +708,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -732,8 +716,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -750,6 +741,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -757,6 +749,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -815,17 +808,6 @@ files = [ {file = "sqlparse-0.4.3.tar.gz", hash = "sha256:69ca804846bb114d2ec380e4360a8a340db83f0ccf3afceeb1404df028f57268"}, ] -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - [[package]] name = "typing-extensions" version = "4.5.0" @@ -870,5 +852,5 @@ test = ["covdefaults (>=2.2.2)", "coverage (>=7.1)", "coverage-enable-subprocess [metadata] lock-version = "2.0" -python-versions = "^3.10" -content-hash = "ac1f2ea1f1b43cac36922e5e7395f591635bd192512ef315b2010939fbdef086" +python-versions = "^3.11" +content-hash = "a8e83135fd1488acb8033e86071d54d18936673a096451c946fb0d5921557a05" diff --git a/be/pyproject.toml b/be/pyproject.toml index 6d18e6e..da9b898 100644 --- a/be/pyproject.toml +++ b/be/pyproject.toml @@ -1,12 +1,11 @@ [tool.poetry] -name = "levity-be" +name = "levity" version = "0.1.0" description = "" -authors = ["Your Name "] -readme = "README.md" +authors = ["Keith Grennan"] [tool.poetry.dependencies] -python = "^3.10" +python = "^3.11" pika = "^1.3.1" python-dateutil = "^2.8.2" dj-database-url = "^1.3.0" @@ -17,6 +16,7 @@ prometheus-client = "^0.16.0" pyyaml = "6.0.1" pytz = "^2023.3" fluent-logger = "^0.10.0" +django = "^5.0.1" [tool.poetry.group.dev.dependencies] diff --git a/rabbitmq/Dockerfile b/rabbitmq/Dockerfile index b97ce49..68a36e4 100644 --- a/rabbitmq/Dockerfile +++ b/rabbitmq/Dockerfile @@ -1,4 +1,4 @@ -FROM rabbitmq:3-alpine +FROM rabbitmq:3.11-alpine RUN apk add curl RUN curl -L https://github.com/rabbitmq/rabbitmq-delayed-message-exchange/releases/download/3.11.1/rabbitmq_delayed_message_exchange-3.11.1.ez \ -o /opt/rabbitmq/plugins/rabbitmq_delayed_message_exchange.ez