From eba215533f82500da18e1293884d6d7ee7e7b4e6 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Fri, 6 Sep 2024 16:15:29 +0200 Subject: [PATCH 01/18] django: fix resource serializer tests Signed-off-by: Jonas Remmert --- .../tests/test_post_composite_resource.py | 290 +++++++++--------- .../tests/test_post_single_resource.py | 25 +- 2 files changed, 153 insertions(+), 162 deletions(-) diff --git a/server/django/sensordata/tests/test_post_composite_resource.py b/server/django/sensordata/tests/test_post_composite_resource.py index 864bcd98..3689c8d0 100644 --- a/server/django/sensordata/tests/test_post_composite_resource.py +++ b/server/django/sensordata/tests/test_post_composite_resource.py @@ -11,185 +11,173 @@ ResourceType, ) -TEST_PAYLOAD = { - "ep" : "qemu_x86", +TEST_PAYLOAD = ''' +{ + "ep" : "urn:imei:100000000000000", "val" : { - "instances" : [ { - "kind" : "instance", - "resources" : [ { - "kind" : "singleResource", - "id" : 0, - "type" : "STRING", - "value" : "Zephyr" - }, { - "kind" : "singleResource", - "id" : 1, - "type" : "STRING", - "value" : "OMA-LWM2M Sample Client" - }, { - "kind" : "singleResource", - "id" : 2, - "type" : "STRING", - "value" : "345000123" - }, { - "kind" : "singleResource", - "id" : 3, - "type" : "STRING", - "value" : "1.0" - }, { - "kind" : "multiResource", - "values" : { - "0" : "1", - "1" : "5" - }, - "id" : 6, - "type" : "INTEGER" - }, { - "kind" : "multiResource", - "values" : { - "0" : "3800", - "1" : "5000" - }, - "id" : 7, - "type" : "INTEGER" - }, { - "kind" : "multiResource", - "values" : { - "0" : "125", - "1" : "900" - }, - "id" : 8, - "type" : "INTEGER" - }, { - "kind" : "singleResource", - "id" : 9, - "type" : "INTEGER", - "value" : "95" - }, { - "kind" : "singleResource", - "id" : 10, - "type" : "INTEGER", - "value" : "15" - }, { - "kind" : "multiResource", - "values" : { - "0" : "0" - }, - "id" : 11, - "type" : "INTEGER" - }, { - "kind" : "singleResource", - "id" : 13, - "type" : "TIME", - "value" : 2000 - }, { - "kind" : "singleResource", - "id" : 14, - "type" : "STRING", - "value" : "" - }, { - "kind" : "singleResource", - "id" : 15, - "type" : "STRING", - "value" : "" - }, { - "kind" : "singleResource", - "id" : 16, - "type" : "STRING", - "value" : "U" - }, { - "kind" : "singleResource", - "id" : 17, - "type" : "STRING", - "value" : "qemu_x86" - }, { - "kind" : "singleResource", - "id" : 18, - "type" : "STRING", - "value" : "1.0.1" - }, { - "kind" : "singleResource", - "id" : 19, - "type" : "STRING", - "value" : "" - }, { - "kind" : "singleResource", - "id" : 20, - "type" : "INTEGER", - "value" : "1" - }, { - "kind" : "singleResource", - "id" : 21, - "type" : "INTEGER", - "value" : "25" + "/3" : { + "instances" : [ { + "kind" : "instance", + "resources" : [ { + "kind" : "singleResource", + "id" : 0, + "type" : "STRING", + "value" : "Zephyr" + }, { + "kind" : "singleResource", + "id" : 1, + "type" : "STRING", + "value" : "OMA-LWM2M Sample Client" + }, { + "kind" : "singleResource", + "id" : 2, + "type" : "STRING", + "value" : "345000123" + }, { + "kind" : "singleResource", + "id" : 3, + "type" : "STRING", + "value" : "1.0" + }, { + "kind" : "multiResource", + "values" : { + "0" : "1", + "1" : "5" + }, + "id" : 6, + "type" : "INTEGER" + }, { + "kind" : "multiResource", + "values" : { + "0" : "3800", + "1" : "5000" + }, + "id" : 7, + "type" : "INTEGER" + }, { + "kind" : "multiResource", + "values" : { + "0" : "125", + "1" : "900" + }, + "id" : 8, + "type" : "INTEGER" + }, { + "kind" : "singleResource", + "id" : 9, + "type" : "INTEGER", + "value" : "95" + }, { + "kind" : "singleResource", + "id" : 10, + "type" : "INTEGER", + "value" : "15" + }, { + "kind" : "multiResource", + "values" : { + "0" : "0" + }, + "id" : 11, + "type" : "INTEGER" + }, { + "kind" : "singleResource", + "id" : 13, + "type" : "TIME", + "value" : 0 + }, { + "kind" : "singleResource", + "id" : 14, + "type" : "STRING", + "value" : "" + }, { + "kind" : "singleResource", + "id" : 15, + "type" : "STRING", + "value" : "" + }, { + "kind" : "singleResource", + "id" : 16, + "type" : "STRING", + "value" : "U" + }, { + "kind" : "singleResource", + "id" : 17, + "type" : "STRING", + "value" : "native_sim" + }, { + "kind" : "singleResource", + "id" : 18, + "type" : "STRING", + "value" : "1.0.1" + }, { + "kind" : "singleResource", + "id" : 19, + "type" : "STRING", + "value" : "" + }, { + "kind" : "singleResource", + "id" : 20, + "type" : "INTEGER", + "value" : "1" + }, { + "kind" : "singleResource", + "id" : 21, + "type" : "INTEGER", + "value" : "25" + } ], + "id" : 0 } ], - "id" : 0 - } ], - "kind" : "obj", - "id" : 3 + "kind" : "obj", + "id" : 3 + } } } +''' + class SensorDataTests(TestCase): def setUp(self): # Ensure that all ResourceTypes object exist ResourceType.objects.create(object_id=3, resource_id=0, - name="Manufacturer", data_type="str_value") - + name="Manufacturer", data_type=ResourceType.STRING) ResourceType.objects.create(object_id=3, resource_id=1, - name="Model Number", data_type="str_value") - + name="Model Number", data_type=ResourceType.STRING) ResourceType.objects.create(object_id=3, resource_id=2, - name="Serial Number", data_type="str_value") - + name="Serial Number", data_type=ResourceType.STRING) ResourceType.objects.create(object_id=3, resource_id=3, - name="Firmware Version", data_type="str_value") - + name="Firmware Version", data_type=ResourceType.STRING) ResourceType.objects.create(object_id=3, resource_id=6, - name="Supported Objects", data_type="int_value") - + name="Supported Objects", data_type=ResourceType.INTEGER) ResourceType.objects.create(object_id=3, resource_id=7, - name="Supported Resources", data_type="int_value") - + name="Supported Resources", data_type=ResourceType.INTEGER) ResourceType.objects.create(object_id=3, resource_id=8, - name="Manufacturer ID", data_type="int_value") - + name="Manufacturer ID", data_type=ResourceType.INTEGER) ResourceType.objects.create(object_id=3, resource_id=9, - name="Memory Free", data_type="int_value") - + name="Memory Free", data_type=ResourceType.INTEGER) ResourceType.objects.create(object_id=3, resource_id=10, - name="Error Code", data_type="int_value") - + name="Error Code", data_type=ResourceType.INTEGER) ResourceType.objects.create(object_id=3, resource_id=11, - name="Current Time", data_type="int_value") - + name="Current Time", data_type=ResourceType.INTEGER) ResourceType.objects.create(object_id=3, resource_id=13, - name="UTC Offset", data_type="int_value") - + name="UTC Offset", data_type=ResourceType.INTEGER) ResourceType.objects.create(object_id=3, resource_id=14, - name="Timezone", data_type="str_value") - + name="Timezone", data_type=ResourceType.STRING) ResourceType.objects.create(object_id=3, resource_id=15, - name="Supported Binding and Modes", data_type="str_value") - + name="Supported Binding and Modes", + data_type=ResourceType.STRING) ResourceType.objects.create(object_id=3, resource_id=16, - name="Device Type", data_type="str_value") - + name="Device Type", data_type=ResourceType.STRING) ResourceType.objects.create(object_id=3, resource_id=17, - name="Hardware Version", data_type="str_value") - + name="Hardware Version", data_type=ResourceType.STRING) ResourceType.objects.create(object_id=3, resource_id=18, - name="Software Version", data_type="str_value") - + name="Software Version", data_type=ResourceType.STRING) ResourceType.objects.create(object_id=3, resource_id=19, - name="Battery Level", data_type="str_value") - + name="Battery Level", data_type=ResourceType.STRING) ResourceType.objects.create(object_id=3, resource_id=20, - name="Battery Status", data_type="int_value") - + name="Battery Status", data_type=ResourceType.INTEGER) ResourceType.objects.create(object_id=3, resource_id=21, - name="Memory Total", data_type="int_value") + name="Memory Total", data_type=ResourceType.INTEGER) def test_post_composite_resource(self): diff --git a/server/django/sensordata/tests/test_post_single_resource.py b/server/django/sensordata/tests/test_post_single_resource.py index 83784fd8..09ed0426 100644 --- a/server/django/sensordata/tests/test_post_single_resource.py +++ b/server/django/sensordata/tests/test_post_single_resource.py @@ -7,12 +7,14 @@ from django.urls import reverse from rest_framework import status from django.test import TestCase +import json from sensordata.models import ( ResourceType, Resource, ) -TEST_PAYLOAD = { +TEST_PAYLOAD = ''' +{ "ep": "qemu_x86", "obj_id": 3303, "val": { @@ -22,15 +24,17 @@ "value": "24.899181214836236" } } +''' class SensorDataTests(TestCase): def setUp(self): # Ensure that the ResourceType object exists ResourceType.objects.create(object_id=3303, resource_id=5700, - name="temperature", data_type="float_value") + name="temperature", data_type=ResourceType.FLOAT) def test_post_single_resource(self): + PL_DICT = json.loads(TEST_PAYLOAD) url = reverse('post-single-resource') response = self.client.post(url, TEST_PAYLOAD, content_type='application/json') @@ -40,15 +44,14 @@ def test_post_single_resource(self): # Query the database to retrieve the created object created_object = Resource.objects.get( - resource_type__object_id = TEST_PAYLOAD["obj_id"], - resource_type__resource_id = TEST_PAYLOAD["val"]["id"], - endpoint__endpoint= TEST_PAYLOAD["ep"] + resource_type__object_id = PL_DICT["obj_id"], + resource_type__resource_id = PL_DICT["val"]["id"], + endpoint__endpoint= PL_DICT["ep"] ) # Compare the retrieved object's attributes to the expected values - self.assertEqual(created_object.endpoint.endpoint, TEST_PAYLOAD["ep"]) - self.assertEqual(created_object.resource_type.object_id, TEST_PAYLOAD["obj_id"]) - self.assertEqual(created_object.resource_type.resource_id, TEST_PAYLOAD["val"]["id"]) - - self.assertEqual(created_object.resource_type.data_type, "float_value") - self.assertEqual(created_object.float_value, float(TEST_PAYLOAD["val"]["value"])) + self.assertEqual(created_object.endpoint.endpoint, PL_DICT["ep"]) + self.assertEqual(created_object.resource_type.object_id, PL_DICT["obj_id"]) + self.assertEqual(created_object.resource_type.resource_id, PL_DICT["val"]["id"]) + self.assertEqual(created_object.resource_type.data_type, ResourceType.FLOAT) + self.assertEqual(created_object.float_value, float(PL_DICT["val"]["value"])) From f398d04d92dc9e5dfcdbe9e0b627105262eca250 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Fri, 6 Sep 2024 16:16:08 +0200 Subject: [PATCH 02/18] django: implement timestamped serializer When using the LWM2M SEND operation, leshan uses a different format, called timestamped. In order to process SEND requests, this commit adds a new timestamped serializer. An Endpoint can send multiple resources with the SEND request at once. In case multiple endpoints are send, django summarizes all of them (regardless of the object) and links them to a new event. The name of the event is the object of the first resource. In the example payload below, the event name would be "3303". Example SEND Operation: { "ep" : "urn:imei:100000000000000", "val" : { "empty" : false, "timestamps" : [ null ], "nodes" : { "/3303/0/5700" : { "kind" : "singleResource", "id" : 5700, "type" : "FLOAT", "value" : "21.13306311567618" }, "/3304/0/5700" : { "kind" : "singleResource", "id" : 5700, "type" : "FLOAT", "value" : "54.92426540118741" } } } } Signed-off-by: Jonas Remmert --- .../timestamped_resource_serializer.py | 44 +++++++++++++++ .../tests/test_post_timestamped_resource.py | 56 +++++++++++++++++++ server/django/sensordata/urls.py | 8 ++- server/django/sensordata/views.py | 16 ++++++ 4 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 server/django/sensordata/serializers/timestamped_resource_serializer.py create mode 100644 server/django/sensordata/tests/test_post_timestamped_resource.py diff --git a/server/django/sensordata/serializers/timestamped_resource_serializer.py b/server/django/sensordata/serializers/timestamped_resource_serializer.py new file mode 100644 index 00000000..e2b7288b --- /dev/null +++ b/server/django/sensordata/serializers/timestamped_resource_serializer.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2024 Jonas Remmert +# +# SPDX-License-Identifier: Apache-2.0 +# + +from rest_framework import serializers +from .base import HandleResourceMixin, ResourceDataSerializer +from ..models import Endpoint +import logging + +logger = logging.getLogger(__name__) + + +class TsValueSerializer(serializers.Serializer): + nodes = serializers.DictField( + child=ResourceDataSerializer(), + help_text="Dictionary of resource paths (e.g., '/3303/0/5700') to resource data" + ) + + +class TimestampedResourceSerializer(HandleResourceMixin, serializers.Serializer): + ep = serializers.CharField(max_length=255, help_text="Unique LwM2M Endpoint") + val = TsValueSerializer() + + def create(self, validated_data): + ep = validated_data['ep'] + val = validated_data['val'] + + endpoint, _ = Endpoint.objects.get_or_create(endpoint=ep) + + nodes = val.get('nodes', {}) + + # If multiple resources are present, create and assign them to an + # event. Take the first object id as the event type. + if len(nodes) > 1: + obj_id = int(list(nodes.keys())[0].split('/')[1]) + self.create_event(endpoint, obj_id) + + for path, resource in nodes.items(): + obj_id = int(path.split('/')[1]) + self.handle_resource(endpoint, obj_id, resource) + + return endpoint diff --git a/server/django/sensordata/tests/test_post_timestamped_resource.py b/server/django/sensordata/tests/test_post_timestamped_resource.py new file mode 100644 index 00000000..55af8eea --- /dev/null +++ b/server/django/sensordata/tests/test_post_timestamped_resource.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2024 Jonas Remmert +# +# SPDX-License-Identifier: Apache-2.0 +# + +from django.urls import reverse +from rest_framework import status +from django.test import TestCase +from sensordata.models import ( + ResourceType, +) + +TEST_PAYLOAD = ''' +{ + "ep" : "urn:imei:100000000000000", + "val" : { + "empty" : false, + "timestamps" : [ null ], + "nodes" : { + "/3303/0/5700" : { + "kind" : "singleResource", + "id" : 5700, + "type" : "FLOAT", + "value" : "22.19110046564394" + }, + "/3304/0/5700" : { + "kind" : "singleResource", + "id" : 5700, + "type" : "FLOAT", + "value" : "52.237076014801175" + } + } + } +} +''' + + +class SensorDataTests(TestCase): + + def setUp(self): + # Ensure that all ResourceTypes object exist + ResourceType.objects.create(object_id=3303, resource_id=5700, + name="temperature", data_type=ResourceType.FLOAT) + ResourceType.objects.create(object_id=3304, resource_id=5700, + name="humidity", data_type=ResourceType.FLOAT) + + + def test_post_composite_resource(self): + + url = reverse('post-timestamped-resource') + + response = self.client.post(url, TEST_PAYLOAD, content_type='application/json') + + # Check that the response is 201 Created + self.assertEqual(response.status_code, status.HTTP_201_CREATED) diff --git a/server/django/sensordata/urls.py b/server/django/sensordata/urls.py index 11fb2d1e..b348bd65 100644 --- a/server/django/sensordata/urls.py +++ b/server/django/sensordata/urls.py @@ -5,10 +5,16 @@ # from django.urls import path -from .views import PostSingleResourceView, PostCompositeResourceView, ResourceDataView +from .views import ( + PostSingleResourceView, + PostCompositeResourceView, + PostTimestampedResourceView, + ResourceDataView +) urlpatterns = [ path('data//', ResourceDataView.as_view(), name='resource-data'), path('resource/single', PostSingleResourceView.as_view(), name='post-single-resource'), path('resource/composite', PostCompositeResourceView.as_view(), name='post-composite-resource'), + path('resource/timestamped', PostTimestampedResourceView.as_view(), name='post-timestamped-resource'), ] diff --git a/server/django/sensordata/views.py b/server/django/sensordata/views.py index 041588a7..701ee11c 100644 --- a/server/django/sensordata/views.py +++ b/server/django/sensordata/views.py @@ -14,6 +14,7 @@ from rest_framework.exceptions import ValidationError from .serializers.single_resource_serializer import SingleResourceSerializer from .serializers.composite_resource_serializer import CompositeResourceSerializer +from .serializers.timestamped_resource_serializer import TimestampedResourceSerializer from .serializers.generic_resource_serializer import GenericResourceSerializer logger = logging.getLogger(__name__) @@ -48,6 +49,21 @@ def post(self, request): logger.error("Backtrace: %s", traceback.format_exc()) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) +class PostTimestampedResourceView(APIView): + serializer_class = TimestampedResourceSerializer + + def post(self, request): + serializer = TimestampedResourceSerializer(data=request.data, many=False) + try: + if serializer.is_valid(raise_exception=True): + serializer.save() + return Response(serializer.validated_data, status=status.HTTP_201_CREATED) + except ValidationError as e: + logger.error("Validation error: %s", e) + logger.error("Request data: %s", request.data) + logger.error("Backtrace: %s", traceback.format_exc()) + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + class ResourceDataView(ListAPIView): serializer_class = GenericResourceSerializer From 935267ef5c0c8dcd5600d96cc2a3f2e76095c575 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Fri, 6 Sep 2024 16:21:14 +0200 Subject: [PATCH 03/18] leshan: implement SEND request handling Signed-off-by: Jonas Remmert --- .../src/main/java/lwm2m/DataSenderRest.java | 1 + .../leshan/src/main/java/lwm2m/LeshanSvr.java | 23 +++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/server/leshan/src/main/java/lwm2m/DataSenderRest.java b/server/leshan/src/main/java/lwm2m/DataSenderRest.java index 8352a6b9..c046d574 100644 --- a/server/leshan/src/main/java/lwm2m/DataSenderRest.java +++ b/server/leshan/src/main/java/lwm2m/DataSenderRest.java @@ -30,6 +30,7 @@ public String getPath() { /* Static factory method for creating instance of ApiPath */ public static final ApiPath SINGLE_RES = new ApiPath("/leshan_api/resource/single"); public static final ApiPath COMPOSITE_RES = new ApiPath("/leshan_api/resource/composite"); + public static final ApiPath TIMESTAMPED_RES = new ApiPath("/leshan_api/resource/timestamped"); @Override public String toString() { diff --git a/server/leshan/src/main/java/lwm2m/LeshanSvr.java b/server/leshan/src/main/java/lwm2m/LeshanSvr.java index 8a668ff9..a52f21d4 100644 --- a/server/leshan/src/main/java/lwm2m/LeshanSvr.java +++ b/server/leshan/src/main/java/lwm2m/LeshanSvr.java @@ -99,6 +99,7 @@ public LeshanSvr() { server.getRegistrationService().addListener(new MyRegistrationListener(this)); server.getObservationService().addListener(new MyObservationListener(this)); + server.getSendService().addListener(new MySendListener(this)); } public static void main(String[] args) { @@ -219,10 +220,10 @@ private void onboardingDevice(Registration registration) { /* Subscribe to single resource instances as an example */ int[][] singleObjectLinks = { - {3303, 0, 5700}, /* Temperature Sensor */ - {3304, 0, 5700}, /* Humidity Sensor */ - {5, 0, 3}, /* Firmware Update State */ - {5, 0, 5} /* Firmware Update Result */ + //{3303, 0, 5700}, /* Temperature Sensor */ + //{3304, 0, 5700}, /* Humidity Sensor */ + //{5, 0, 3}, /* Firmware Update State */ + //{5, 0, 5} /* Firmware Update Result */ }; for (int[] link : singleObjectLinks) { @@ -242,7 +243,7 @@ private void onboardingDevice(Registration registration) { * 10300: Custom Object Instance */ int[][] compositeObjectLinks = { - {10300}, + //{10300}, }; for (int[] link : compositeObjectLinks) { @@ -358,6 +359,8 @@ public void onResponse(SingleObservation observation, Registration registration, ObserveResponse response) { mapper.enable(SerializationFeature.INDENT_OUTPUT); + log.trace("onResponse: " + response.getContent()); + if (registration != null) { ObjectNode node = mapper.createObjectNode(); node.put("ep", registration.getEndpoint()); @@ -372,7 +375,7 @@ public void onResponse(SingleObservation observation, public void onResponse(CompositeObservation observation, Registration registration, ObserveCompositeResponse response) { - log.trace("onResponse: " + response.getObservation()); + log.trace("onResponse: " + response.getContent()); mapper.enable(SerializationFeature.INDENT_OUTPUT); ObjectNode node = mapper.createObjectNode(); @@ -407,7 +410,13 @@ public void dataReceived(Registration registration, TimestampedLwM2mNodes data, SendRequest request) { log.trace("dataReceived from: " + registration.getEndpoint()); - log.trace("data: " + data); + + mapper.enable(SerializationFeature.INDENT_OUTPUT); + ObjectNode node = mapper.createObjectNode(); + node.put("ep", registration.getEndpoint()); + node.set("val", mapper.valueToTree(data)); + + dataSenderRest.sendData(ApiPath.TIMESTAMPED_RES, node); } @Override From c056da963e01f87b3e25b772cf0d08d00162b40a Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Fri, 6 Sep 2024 16:21:36 +0200 Subject: [PATCH 04/18] simulation: change from observe to send operation in lwm2m client Observe operations are currently problematic. The observations become invalid, in case the endpoint changes it's ip address, route or port. Once DTLS is activated, observe operations should work reliably. For sending temperature values or generally frequent sensor values, send operations are better suited anyways as each value is guaranteed to be send. With observe operations, only the most recent value is send. Signed-off-by: Jonas Remmert --- .../lwm2m_client/src/ca_self_signed.pem | 11 +++++++ simulation/lwm2m_client/src/firmware_update.c | 22 +++++++++++-- simulation/lwm2m_client/src/lwm2m-client.c | 8 +++-- simulation/lwm2m_client/src/modules.h | 7 ++-- simulation/lwm2m_client/src/temperature.c | 32 +++++++++++++++++-- 5 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 simulation/lwm2m_client/src/ca_self_signed.pem diff --git a/simulation/lwm2m_client/src/ca_self_signed.pem b/simulation/lwm2m_client/src/ca_self_signed.pem new file mode 100644 index 00000000..03e471ed --- /dev/null +++ b/simulation/lwm2m_client/src/ca_self_signed.pem @@ -0,0 +1,11 @@ +"-----BEGIN CERTIFICATE-----\n" +"MIIBiDCCAS2gAwIBAgIUahmOgRlKsTWIqmZqXPHvNSCYe/kwCgYIKoZIzj0EAwIw\n" +"GDEWMBQGA1UEAwwNZmxvd25leHVzLm9yZzAgFw0yNDA4MTkxOTA0MTZaGA8yMTI0\n" +"MDcyNjE5MDQxNlowGDEWMBQGA1UEAwwNZmxvd25leHVzLm9yZzBZMBMGByqGSM49\n" +"AgEGCCqGSM49AwEHA0IABOwM3H8ZPMyKFvs3w8iSkAlcdkRqHUc3dL5BxF9/f0iE\n" +"mDMrcOYMLlYV0wn0djS3gEbITpUgiOucu5TfNASZam6jUzBRMB0GA1UdDgQWBBSH\n" +"sSIh3pqlJ5BSPA1TI3XchUL1FjAfBgNVHSMEGDAWgBSHsSIh3pqlJ5BSPA1TI3Xc\n" +"hUL1FjAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0kAMEYCIQDBKynm1aF1\n" +"MiTbJ8Q59VdKGd8XzApMJHggjWmX1s+G/QIhAPxGbrubPdmS48IttV9zpHcpmmF8\n" +"shKnmUNTg/w5BWon\n" +"-----END CERTIFICATE-----\n" diff --git a/simulation/lwm2m_client/src/firmware_update.c b/simulation/lwm2m_client/src/firmware_update.c index 1edb95ba..712f898c 100644 --- a/simulation/lwm2m_client/src/firmware_update.c +++ b/simulation/lwm2m_client/src/firmware_update.c @@ -31,6 +31,8 @@ static char download_url[MAX_URL_LENGTH]; static char response[CONFIG_NET_BUF_DATA_SIZE]; unsigned int cur_bytes; +static struct lwm2m_ctx *client_ctx; + #define SSTRLEN(s) (sizeof(s) - 1) static ssize_t sendall(int sock, const void *buf, size_t len) @@ -328,6 +330,8 @@ static int firmware_update_cb(uint16_t obj_inst_id, * In reality, it should be set at function lwm2m_setup() */ lwm2m_firmware_set_update_result(RESULT_SUCCESS); + lwm2m_send_cb(client_ctx, &LWM2M_OBJ(5, 0, 3), 1, NULL); + lwm2m_send_cb(client_ctx, &LWM2M_OBJ(5, 0, 5), 1, NULL); return 0; } @@ -342,9 +346,13 @@ static int firmware_download_cb(uint16_t obj_inst_id, LOG_INF("Download Link: %s", data); lwm2m_firmware_set_update_state(STATE_DOWNLOADING); + lwm2m_send_cb(client_ctx, &LWM2M_OBJ(5, 0, 3), 1, NULL); + + /* download is a blocking method */ ret = download(CONFIG_DL_SERVER_HOSTNAME, data); if (ret == 0) { lwm2m_firmware_set_update_state(STATE_DOWNLOADED); + lwm2m_send_cb(client_ctx, &LWM2M_OBJ(5, 0, 3), 1, NULL); return 0; } @@ -353,14 +361,19 @@ static int firmware_download_cb(uint16_t obj_inst_id, case -ENOTSUP: case -EINVAL: lwm2m_firmware_set_update_result(RESULT_UNSUP_PROTO); + lwm2m_firmware_set_update_state(STATE_IDLE); break; case -ENOTCONN: lwm2m_firmware_set_update_result(RESULT_CONNECTION_LOST); + lwm2m_firmware_set_update_state(STATE_IDLE); break; default: lwm2m_firmware_set_update_result(RESULT_UPDATE_FAILED); + lwm2m_firmware_set_update_state(STATE_IDLE); } - lwm2m_firmware_set_update_state(STATE_IDLE); + + lwm2m_send_cb(client_ctx, &LWM2M_OBJ(5, 0, 3), 1, NULL); + lwm2m_send_cb(client_ctx, &LWM2M_OBJ(5, 0, 5), 1, NULL); return ret; } @@ -372,13 +385,18 @@ static int firmware_cancel_cb(const uint16_t obj_inst_id) lwm2m_firmware_set_update_state(STATE_IDLE); lwm2m_firmware_set_update_result(RESULT_UPDATE_FAILED); + lwm2m_send_cb(client_ctx, &LWM2M_OBJ(5, 0, 3), 1, NULL); + lwm2m_send_cb(client_ctx, &LWM2M_OBJ(5, 0, 5), 1, NULL); + return 0; } -void init_firmware_update(void) +void init_firmware_update(struct lwm2m_ctx *client) { int ret; + client_ctx = client; + #if defined(CONFIG_NET_SOCKETS_SOCKOPT_TLS) LOG_INF("Adding CA certificate"); for (int i = 0; i < ARRAY_SIZE(ca_certificates); i++) { diff --git a/simulation/lwm2m_client/src/lwm2m-client.c b/simulation/lwm2m_client/src/lwm2m-client.c index dedfb42d..f1a2f72b 100644 --- a/simulation/lwm2m_client/src/lwm2m-client.c +++ b/simulation/lwm2m_client/src/lwm2m-client.c @@ -51,7 +51,7 @@ static int mem_total = 25; static double min_range = 0.0; static double max_range = 100; -static struct lwm2m_ctx client_ctx = {0}; +struct lwm2m_ctx client_ctx = {0}; static const char *endpoint = (sizeof(CONFIG_LWM2M_APP_ID) > 1 ? CONFIG_LWM2M_APP_ID : CONFIG_BOARD); @@ -68,7 +68,9 @@ static struct net_mgmt_event_callback l4_cb; static struct net_mgmt_event_callback conn_cb; static struct net_mgmt_event_callback wifi_conn_cb; +#if defined(CONFIG_WIFI) && (CONFIG_WIFI == 1) static struct wifi_connect_req_params cnx_params; +#endif static K_SEM_DEFINE(network_connected_sem, 0, 1); @@ -183,11 +185,11 @@ static int lwm2m_setup(void) /* setup FIRMWARE object */ if (IS_ENABLED(CONFIG_LWM2M_FIRMWARE_UPDATE_OBJ_SUPPORT)) { - init_firmware_update(); + init_firmware_update(&client_ctx); } /* setup TEMP SENSOR object */ - init_temp_sensor(); + init_temp_sensor(&client_ctx); /* Set multiple TEMP SENSOR resource values in one function call. */ int err = lwm2m_set_bulk(temp_sensor_items, ARRAY_SIZE(temp_sensor_items)); diff --git a/simulation/lwm2m_client/src/modules.h b/simulation/lwm2m_client/src/modules.h index 4e9befcf..4e145da8 100644 --- a/simulation/lwm2m_client/src/modules.h +++ b/simulation/lwm2m_client/src/modules.h @@ -7,9 +7,12 @@ #ifndef _MODULES_H #define _MODULES_H +#include + int init_led_device(void); void init_timer_object(void); -void init_temp_sensor(void); -void init_firmware_update(void); +void init_temp_sensor(struct lwm2m_ctx *client); +void init_firmware_update(struct lwm2m_ctx *client); + #endif diff --git a/simulation/lwm2m_client/src/temperature.c b/simulation/lwm2m_client/src/temperature.c index 324178b3..f61a0554 100644 --- a/simulation/lwm2m_client/src/temperature.c +++ b/simulation/lwm2m_client/src/temperature.c @@ -18,11 +18,14 @@ LOG_MODULE_REGISTER(LOG_MODULE_NAME); #include static struct k_work_delayable temp_work; -#define PERIOD K_MINUTES(1) +#define PERIOD K_SECONDS(10) + +static struct lwm2m_ctx *client_ctx; static void temp_work_cb(struct k_work *work) { double t, h; + int ret; if (IS_ENABLED(CONFIG_SHT4X)) { const struct device *dev = DEVICE_DT_GET_ANY(sensirion_sht4x); @@ -52,12 +55,37 @@ static void temp_work_cb(struct k_work *work) lwm2m_set_f64(&LWM2M_OBJ(3303, 0, 5700), t); lwm2m_set_f64(&LWM2M_OBJ(3304, 0, 5700), h); + ret = lwm2m_send_cb(client_ctx, &LWM2M_OBJ(3303, 0, 5700), 1, NULL); + if (ret) { + LOG_ERR("lwm2m_send_cb, error: %d", ret); + } + ret = lwm2m_send_cb(client_ctx, &LWM2M_OBJ(3304, 0, 5700), 1, NULL); + if (ret) { + LOG_ERR("lwm2m_send_cb, error: %d", ret); + } + + /* As an alternative, you can send the values together. In that case + * flownexus assigns them to an event. The name of the event is the + * first object (3303 in this case). + * + * const struct lwm2m_obj_path path[2] = { + * LWM2M_OBJ(3303, 0, 5700), + * LWM2M_OBJ(3304, 0, 5700), + * }; + * ret = lwm2m_send_cb(client_ctx, path, 2, NULL); + * if (ret) { + * LOG_ERR("lwm2m_send_cb, error: %d", ret); + * } + */ + out: k_work_schedule(&temp_work, PERIOD); } -void init_temp_sensor(void) +void init_temp_sensor(struct lwm2m_ctx *client) { + client_ctx = client; + if ((lwm2m_create_object_inst(&LWM2M_OBJ(3303, 0)) == 0) && (lwm2m_create_object_inst(&LWM2M_OBJ(3304, 0)) == 0)) { k_work_init_delayable(&temp_work, temp_work_cb); From 845eea1d7d9280720e815b07fc3a0a4ce46384c9 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Sat, 7 Sep 2024 09:22:42 +0200 Subject: [PATCH 05/18] django: rework base serializer and fix fota bugs Signed-off-by: Jonas Remmert --- server/django/sensordata/admin.py | 3 +- server/django/sensordata/serializers/base.py | 159 +++++++++++-------- 2 files changed, 95 insertions(+), 67 deletions(-) diff --git a/server/django/sensordata/admin.py b/server/django/sensordata/admin.py index ce8229d8..b65b708f 100644 --- a/server/django/sensordata/admin.py +++ b/server/django/sensordata/admin.py @@ -138,8 +138,7 @@ def get_form(self, request, obj=None, **kwargs): @admin.register(FirmwareUpdate) class FirmwareUpdateAdmin(admin.ModelAdmin): list_display = ('endpoint', 'firmware', 'state', 'result', - 'timestamp_created', 'timestamp_updated', - 'send_uri_operation', 'execute_operation') + 'timestamp_created', 'timestamp_updated') search_fields = ('endpoint__endpoint', 'firmware__version', 'state', 'result') list_filter = ('state', 'result', 'timestamp_created', 'timestamp_updated') readonly_fields = ('timestamp_created', 'timestamp_updated', diff --git a/server/django/sensordata/serializers/base.py b/server/django/sensordata/serializers/base.py index d4d3be2e..841a1dd1 100644 --- a/server/django/sensordata/serializers/base.py +++ b/server/django/sensordata/serializers/base.py @@ -48,19 +48,19 @@ def __init__(self, *args, **kwargs): self.event = None - def create_event(self, endpoint, event_type): + def create_event(self, ep, event_type): """ Create an Event instance for the given endpoint and event type. If no event is created, the resource data won't be associated with any event. """ event_data = { - 'endpoint': endpoint, + 'endpoint': ep, 'event_type': event_type, } self.event = Event.objects.create(**event_data) - def handle_resource(self, endpoint, obj_id, res): + def handle_resource(self, ep, obj_id, res): # Some LwM2M Resources are currently unsupported, we can skip them for now. if res['kind'] == 'multiResource': logging.error(f"multiResource currently not supported, skipping...") @@ -68,14 +68,14 @@ def handle_resource(self, endpoint, obj_id, res): res_id = res['id'] # Fetch resource information from Database - resource_type = ResourceType.objects.get(object_id=obj_id, resource_id=res_id) - if not resource_type: + res_type = ResourceType.objects.get(object_id=obj_id, resource_id=res_id) + if not res_type: err = f"Resource type {obj_id}/{res_id} not found" raise serializers.ValidationError(err) # Validate that datatype is matching the resource type data_type = dict(ResourceType.TYPE_CHOICES).get(res['type']) - res_data_type = dict(ResourceType.TYPE_CHOICES).get(resource_type.data_type) + res_data_type = dict(ResourceType.TYPE_CHOICES).get(res_type.data_type) if not data_type: err = f"Unsupported data type '{res['type']}', skipping..." logger.error(err) @@ -86,10 +86,10 @@ def handle_resource(self, endpoint, obj_id, res): raise serializers.ValidationError(err) # Create the Resource instance based on value type - logger.debug(f"Adding resource_type: {resource_type}") + logger.debug(f"Adding resource_type: {res_type}") resource_data = { - 'endpoint': endpoint, - 'resource_type': resource_type, + 'endpoint': ep, + 'resource_type': res_type, data_type: res['value'] } @@ -101,69 +101,98 @@ def handle_resource(self, endpoint, obj_id, res): logger.debug(f"Added EventResource: {self.event} - {created_res}") # Update the registration status if the resource is a registration resource - if resource_type.name == 'ep_registered': - endpoint.registered = True - endpoint.save() + if res_type.name == 'ep_registered': + ep.registered = True + ep.save() return - elif resource_type.name == 'ep_unregistered': - endpoint.registered = False - endpoint.save() + elif res_type.name == 'ep_unregistered': + ep.registered = False + ep.save() return - elif resource_type.name == 'ep_registration_update': - process_pending_operations.delay(endpoint.endpoint) + elif res_type.name == 'ep_registration_update': + process_pending_operations.delay(ep.endpoint) return # Cond 1: Check for fota update after ep registration. # "Firmware Version - 3/0/3" Resource. # # Cond 2: Handle FOTA Update - elif ((resource_type.object_id == 3 and resource_type.resource_id == 3) or - resource_type.object_id == 5): - # There must be exactly one FirmwareUpdate object with - # result = 0 (RESULT_DEFAULT). - fw_queryset = FirmwareUpdate.objects.filter(endpoint=endpoint, - result=FirmwareUpdate.Result.RESULT_DEFAULT) - cnt = fw_queryset.count() - if cnt == 1: - fw_obj = fw_queryset.get() - elif cnt == 0: - if resource_type.resource_id == 3: - # This case can happen if the result is updated before the - # state. Can be ignored, as a result will automatically set the - # state to IDLE. - return - else: - err = "No active FirmwareUpdate object found for endpoint" - raise serializers.ValidationError(err) - else: - err = "Multiple active FirmwareUpdate objects found for endpoint" - logger.info(fw_queryset) - raise serializers.ValidationError(err) - - # Compare version with expected version and set download state/result - if resource_type.object_id == 3 and resource_type.resource_id == 3: - fw_obj.state = FirmwareUpdate.State.STATE_IDLE - expected_version = fw_obj.firmware.version - reported_version = res['value'] - if expected_version == reported_version: - fw_obj.result = FirmwareUpdate.Result.RESULT_SUCCESS - else: - fw_obj.result = FirmwareUpdate.Result.RESULT_UPDATE_FAILED - fw_obj.save() + elif ((res_type.object_id == 3 and res_type.resource_id == 3) or + res_type.object_id == 5): + self.handle_fota(ep, res_type, res['value']) + return + + + def handle_fota(self, ep, res_type, value): + # There must be exactly one FirmwareUpdate object with + # result = 0 (RESULT_DEFAULT). + fw_query = FirmwareUpdate.objects.filter(endpoint=ep, + result=FirmwareUpdate.Result.RESULT_DEFAULT) + + # Check for exactly one FirmwareUpdate object + if fw_query.count() == 0: + if res_type.object_id == 3 and res_type.resource_id == 3: + # Just a regular reboot, no FOTA update + return + err = "No FirmwareUpdate object found for endpoint" + raise serializers.ValidationError + if fw_query.count() != 1: + err = "Multiple active FirmwareUpdate objects found for endpoint" + logger.info(fw_query) + raise serializers.ValidationError(err) + + # Exactly one FirmwareUpdate object found + fw_obj = fw_query.get() + + # Device Rebooted + if res_type.object_id == 3 and res_type.resource_id == 3: + if (fw_obj.state == FirmwareUpdate.State.STATE_IDLE and \ + fw_obj.result == FirmwareUpdate.Result.RESULT_DEFAULT): + # Update hasn't been started yet return + expected_version = fw_obj.firmware.version + reported_version = value + if expected_version == reported_version: + fw_obj.result = FirmwareUpdate.Result.RESULT_SUCCESS + else: + fw_obj.result = FirmwareUpdate.Result.RESULT_UPDATE_FAILED + fw_obj.state = FirmwareUpdate.State.STATE_IDLE + self.abort_pending_fota_comms(fw_obj) + fw_obj.save() + return - if resource_type.resource_id == 3: - if int(res['value']) == FirmwareUpdate.State.STATE_DOWNLOADED: - # Create "Update" resource to execute the update, no payload - exec_resource = Resource.objects.create(endpoint = endpoint, - resource_type = ResourceType.objects.get(object_id = 5, resource_id = 2) - ) - exec_operation = EndpointOperation.objects.create(resource=exec_resource) - fw_obj.state = res['value'] - fw_obj.execute_operation = exec_operation - fw_obj.save() - process_pending_operations.delay(endpoint.endpoint) - elif resource_type.resource_id == 5: - fw_obj.result = res['value'] - fw_obj.state = FirmwareUpdate.State.STATE_IDLE - fw_obj.save() + # Update state changed + if res_type.resource_id == 3: + fw_obj.state = value + if int(value) == FirmwareUpdate.State.STATE_DOWNLOADED: + # Create "Update" resource to execute the update, no payload + exec_update = ResourceType.objects.get(object_id = 5, resource_id = 2) + exec_res = Resource.objects.create( endpoint = ep, resource_type = exec_update) + exec_operation = EndpointOperation.objects.create(resource=exec_res) + fw_obj.execute_operation = exec_operation + process_pending_operations.delay(ep.endpoint) + # Update Result changed (Success/Failure) + elif res_type.resource_id == 5: + fw_obj.result = value + # In cases (success/failure), the update process is finished. + fw_obj.state = FirmwareUpdate.State.STATE_IDLE + self.abort_pending_fota_comms(fw_obj) + else: + return + fw_obj.save() + + + # In case an update is finished (success/failure), abort any pending + # operations (send URI, execute update). All communications should usually + # be closed already. + def abort_pending_fota_comms(self, fw_obj): + # Abort all open operations + if fw_obj.execute_operation: + status = fw_obj.execute_operation.status + if status in (EndpointOperation.Status.QUEUED, EndpointOperation.Status.SENDING): + fw_obj.execute_operation.status = EndpointOperation.Status.FAILED + + if fw_obj.send_uri_operation: + status = fw_obj.send_uri_operation.status + if status in (EndpointOperation.Status.QUEUED, EndpointOperation.Status.SENDING): + fw_obj.send_uri_operation.status = EndpointOperation.Status.FAILED From 505d8bb0b72135e964e896e7f25b3b8ec2b03fa2 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Sat, 7 Sep 2024 12:57:34 +0200 Subject: [PATCH 06/18] django: update ep status with registration update Signed-off-by: Jonas Remmert --- server/django/sensordata/serializers/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/django/sensordata/serializers/base.py b/server/django/sensordata/serializers/base.py index 841a1dd1..8377251f 100644 --- a/server/django/sensordata/serializers/base.py +++ b/server/django/sensordata/serializers/base.py @@ -110,6 +110,8 @@ def handle_resource(self, ep, obj_id, res): ep.save() return elif res_type.name == 'ep_registration_update': + ep.registered = True + ep.save() process_pending_operations.delay(ep.endpoint) return From 663d5db0fbf5ff7c8f672b45730275cf491e867c Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Sat, 7 Sep 2024 19:45:03 +0200 Subject: [PATCH 07/18] django: check for pending operations during registration Signed-off-by: Jonas Remmert --- server/django/sensordata/serializers/base.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/server/django/sensordata/serializers/base.py b/server/django/sensordata/serializers/base.py index 8377251f..23479396 100644 --- a/server/django/sensordata/serializers/base.py +++ b/server/django/sensordata/serializers/base.py @@ -101,19 +101,15 @@ def handle_resource(self, ep, obj_id, res): logger.debug(f"Added EventResource: {self.event} - {created_res}") # Update the registration status if the resource is a registration resource - if res_type.name == 'ep_registered': + if res_type.name in ['ep_registered', 'ep_registration_update']: ep.registered = True ep.save() + process_pending_operations.delay(ep.endpoint) return elif res_type.name == 'ep_unregistered': ep.registered = False ep.save() return - elif res_type.name == 'ep_registration_update': - ep.registered = True - ep.save() - process_pending_operations.delay(ep.endpoint) - return # Cond 1: Check for fota update after ep registration. # "Firmware Version - 3/0/3" Resource. From dde92399b51bec50bdee66ddc1fc6f81cf7796a5 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Mon, 9 Sep 2024 15:23:11 +0200 Subject: [PATCH 08/18] docker-compose: restart redis automatically Signed-off-by: Jonas Remmert --- server/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/docker-compose.yml b/server/docker-compose.yml index b4154b5b..6d2a7c0c 100644 --- a/server/docker-compose.yml +++ b/server/docker-compose.yml @@ -23,6 +23,8 @@ services: - mynetwork ports: - "6379:6379" + # Restart the container unless it is explicitly stopped + restart: unless-stopped leshan: stop_grace_period: 1s From 84823cf5882e71408f274b602fc3e298c31e3a64 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Tue, 10 Sep 2024 11:59:10 +0200 Subject: [PATCH 09/18] redis: add logging to redis Redis showed some instabilities, this commit adds logging to the common logging files. In addition it sets the memory usage to max 20 MB and disables persistence. Signed-off-by: Jonas Remmert --- server/docker-compose.yml | 8 +++++++- server/redis/Dockerfile.redis | 12 ++++++++++++ server/redis/redis.conf | 11 +++++++++++ server/redis/redis_start.sh | 21 +++++++++++++++++++++ 4 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 server/redis/Dockerfile.redis create mode 100644 server/redis/redis.conf create mode 100755 server/redis/redis_start.sh diff --git a/server/docker-compose.yml b/server/docker-compose.yml index 6d2a7c0c..9840f97f 100644 --- a/server/docker-compose.yml +++ b/server/docker-compose.yml @@ -18,7 +18,13 @@ services: - CELERY_RESULT_BACKEND=redis://redis:6379/0 redis: - image: redis:7.4-alpine + stop_grace_period: 1s + build: + context: ./redis + dockerfile: Dockerfile.redis + volumes: + - ./redis:/redis + - ./logs:/redis/logs networks: - mynetwork ports: diff --git a/server/redis/Dockerfile.redis b/server/redis/Dockerfile.redis new file mode 100644 index 00000000..83c8b28d --- /dev/null +++ b/server/redis/Dockerfile.redis @@ -0,0 +1,12 @@ +FROM redis:7.4-alpine + +# Create and set the working directory in the container +WORKDIR /redis + +# Copy the content of the local src directory to the working directory +COPY . /redis/ + +# Give execution rights on the start-up script +RUN chmod +x /redis/redis_start.sh + +ENTRYPOINT ["/redis/redis_start.sh"] diff --git a/server/redis/redis.conf b/server/redis/redis.conf new file mode 100644 index 00000000..e6733127 --- /dev/null +++ b/server/redis/redis.conf @@ -0,0 +1,11 @@ +loglevel notice + +# Limit Redis to 20 MB of memory +maxmemory 20mb + +# Evict least recently used keys when memory limit is reached +maxmemory-policy allkeys-lru + +# Disable persistence and Disable append-only file persistence +save "" +appendonly no diff --git a/server/redis/redis_start.sh b/server/redis/redis_start.sh new file mode 100755 index 00000000..5284b6bc --- /dev/null +++ b/server/redis/redis_start.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +# Ensure logs directory exists +mkdir -p logs + +logfile="logs/redis_$(date +%Y-%m-%d_%H-%M-%S).log" +touch "$logfile" + +# Start Redis server +redis-server /redis/redis.conf 2>&1 | tee -a $logfile + +# Wait for Redis to start +until redis-cli ping; do + echo "Waiting for Redis to start..." + sleep 1 +done + +echo "Redis is running." + +# Wait for Redis to exit +wait From f1f17df527ff3d46912b3aa00e9ee0543393e7da Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Wed, 11 Sep 2024 09:48:25 +0200 Subject: [PATCH 10/18] docker-compose: disable exposure of redis port Signed-off-by: Jonas Remmert --- server/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/docker-compose.yml b/server/docker-compose.yml index 9840f97f..f08744b7 100644 --- a/server/docker-compose.yml +++ b/server/docker-compose.yml @@ -27,8 +27,6 @@ services: - ./logs:/redis/logs networks: - mynetwork - ports: - - "6379:6379" # Restart the container unless it is explicitly stopped restart: unless-stopped From 19b835a53d8a27097ae47fbaa933a6c5dc67b587 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Thu, 12 Sep 2024 21:56:21 +0200 Subject: [PATCH 11/18] django: allow to crate new resource types Signed-off-by: Jonas Remmert --- server/django/sensordata/admin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/django/sensordata/admin.py b/server/django/sensordata/admin.py index b65b708f..58624dea 100644 --- a/server/django/sensordata/admin.py +++ b/server/django/sensordata/admin.py @@ -42,11 +42,10 @@ def get_model_perms(self, request): class ResourceTypeAdmin(admin.ModelAdmin): list_display = ('object_id', 'resource_id', 'name', 'data_type') search_fields = ('object_id', 'resource_id', 'name') - readonly_fields = ('object_id', 'resource_id', 'name', 'data_type') def get_model_perms(self, request): return { - 'add': False, + 'add': True, 'change': False, 'delete': False, 'view': True, From c7cd61fd58bddd36cb4c7538c5c39a2651ed1508 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Thu, 12 Sep 2024 21:57:11 +0200 Subject: [PATCH 12/18] leshan: fix problem with timestamped data serializer Signed-off-by: Jonas Remmert --- server/leshan/pom.xml | 5 +++++ server/leshan/src/main/java/lwm2m/LeshanSvr.java | 2 ++ 2 files changed, 7 insertions(+) diff --git a/server/leshan/pom.xml b/server/leshan/pom.xml index 2d5e358f..d97fa733 100644 --- a/server/leshan/pom.xml +++ b/server/leshan/pom.xml @@ -63,6 +63,11 @@ 6.1.0 provided + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + 2.17.2 + diff --git a/server/leshan/src/main/java/lwm2m/LeshanSvr.java b/server/leshan/src/main/java/lwm2m/LeshanSvr.java index a52f21d4..d38158b0 100644 --- a/server/leshan/src/main/java/lwm2m/LeshanSvr.java +++ b/server/leshan/src/main/java/lwm2m/LeshanSvr.java @@ -56,6 +56,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -96,6 +97,7 @@ public LeshanSvr() { module.addSerializer(LwM2mNode.class, new JacksonLwM2mNodeSerializer()); module.addSerializer(Version.class, new JacksonVersionSerializer()); mapper.registerModule(module); + mapper.registerModule(new JavaTimeModule()); server.getRegistrationService().addListener(new MyRegistrationListener(this)); server.getObservationService().addListener(new MyObservationListener(this)); From a0ec3e25f9a22642e292ace637aa35a43cf09413 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Sat, 14 Sep 2024 09:25:54 +0200 Subject: [PATCH 13/18] django: allow to fill timestamp during create Signed-off-by: Jonas Remmert --- .../0003_alter_resource_timestamp_created.py | 18 ++++++++++++++++++ server/django/sensordata/models.py | 8 +++++++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 server/django/sensordata/migrations/0003_alter_resource_timestamp_created.py diff --git a/server/django/sensordata/migrations/0003_alter_resource_timestamp_created.py b/server/django/sensordata/migrations/0003_alter_resource_timestamp_created.py new file mode 100644 index 00000000..57ac3b8f --- /dev/null +++ b/server/django/sensordata/migrations/0003_alter_resource_timestamp_created.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1 on 2024-09-14 05:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sensordata', '0002_alter_firmware_binary'), + ] + + operations = [ + migrations.AlterField( + model_name='resource', + name='timestamp_created', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/server/django/sensordata/models.py b/server/django/sensordata/models.py index 5bffcd60..d175c959 100644 --- a/server/django/sensordata/models.py +++ b/server/django/sensordata/models.py @@ -7,6 +7,7 @@ from django.db import models from django.core.exceptions import ValidationError from django.db import transaction +from django.utils import timezone import os @@ -59,7 +60,12 @@ class Resource(models.Model): int_value = models.IntegerField(null=True, blank=True) float_value = models.FloatField(null=True, blank=True) str_value = models.CharField(max_length=512, null=True, blank=True) - timestamp_created = models.DateTimeField(auto_now_add=True, blank=True) + timestamp_created = models.DateTimeField(blank=True, null=True) + + def save(self, *args, **kwargs): + if not self.timestamp_created: + self.timestamp_created = timezone.now() + super().save(*args, **kwargs) def __str__(self): return f"{self.endpoint} - {self.resource_type} - {self.timestamp_created}" From 069f9096c833c55991b7c079066a7c56bb75a86e Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Sat, 14 Sep 2024 09:26:16 +0200 Subject: [PATCH 14/18] django: add alternative timestamp parser Signed-off-by: Jonas Remmert --- server/django/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/server/django/requirements.txt b/server/django/requirements.txt index 9e9ec78d..77b8f34b 100644 --- a/server/django/requirements.txt +++ b/server/django/requirements.txt @@ -5,6 +5,7 @@ djangorestframework==3.15.2 whitenoise==6.7.0 celery[redis]==5.4.0 gevent==24.2.1 +python-dateutil==2.9.0 # Generation of Entity-Relationship Diagrams graphviz==0.20.3 From a2643056280d0e7258ba22aa5fb962cd27822006 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Sat, 14 Sep 2024 09:27:12 +0200 Subject: [PATCH 15/18] squash: django: serializers: Signed-off-by: Jonas Remmert --- server/django/sensordata/serializers/base.py | 8 ++-- .../timestamped_resource_serializer.py | 40 ++++++++++++------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/server/django/sensordata/serializers/base.py b/server/django/sensordata/serializers/base.py index 23479396..474848fe 100644 --- a/server/django/sensordata/serializers/base.py +++ b/server/django/sensordata/serializers/base.py @@ -60,7 +60,7 @@ def create_event(self, ep, event_type): self.event = Event.objects.create(**event_data) - def handle_resource(self, ep, obj_id, res): + def handle_resource(self, ep, obj_id, res, ts=None): # Some LwM2M Resources are currently unsupported, we can skip them for now. if res['kind'] == 'multiResource': logging.error(f"multiResource currently not supported, skipping...") @@ -90,15 +90,15 @@ def handle_resource(self, ep, obj_id, res): resource_data = { 'endpoint': ep, 'resource_type': res_type, - data_type: res['value'] + data_type: res['value'], + **({'timestamp_created': ts} if ts is not None else {}) } - created_res = Resource.objects.create(**resource_data) # Create EventResource linking the event and the resource if self.event is not None: EventResource.objects.create(event=self.event, resource=created_res) - logger.debug(f"Added EventResource: {self.event} - {created_res}") + logger.debug(f"Added Resource to event: {self.event.event_type}") # Update the registration status if the resource is a registration resource if res_type.name in ['ep_registered', 'ep_registration_update']: diff --git a/server/django/sensordata/serializers/timestamped_resource_serializer.py b/server/django/sensordata/serializers/timestamped_resource_serializer.py index e2b7288b..615342f7 100644 --- a/server/django/sensordata/serializers/timestamped_resource_serializer.py +++ b/server/django/sensordata/serializers/timestamped_resource_serializer.py @@ -8,11 +8,12 @@ from .base import HandleResourceMixin, ResourceDataSerializer from ..models import Endpoint import logging +from dateutil import parser logger = logging.getLogger(__name__) -class TsValueSerializer(serializers.Serializer): +class NodeSerializer(serializers.Serializer): nodes = serializers.DictField( child=ResourceDataSerializer(), help_text="Dictionary of resource paths (e.g., '/3303/0/5700') to resource data" @@ -21,24 +22,35 @@ class TsValueSerializer(serializers.Serializer): class TimestampedResourceSerializer(HandleResourceMixin, serializers.Serializer): ep = serializers.CharField(max_length=255, help_text="Unique LwM2M Endpoint") - val = TsValueSerializer() + val = serializers.ListField( + child=serializers.DictField( + child=NodeSerializer(), + ) + ) + def create(self, validated_data): ep = validated_data['ep'] val = validated_data['val'] - + # Create an Event only once for the given endpoint + event_created = False endpoint, _ = Endpoint.objects.get_or_create(endpoint=ep) - nodes = val.get('nodes', {}) - - # If multiple resources are present, create and assign them to an - # event. Take the first object id as the event type. - if len(nodes) > 1: - obj_id = int(list(nodes.keys())[0].split('/')[1]) - self.create_event(endpoint, obj_id) - - for path, resource in nodes.items(): - obj_id = int(path.split('/')[1]) - self.handle_resource(endpoint, obj_id, resource) + for item in val: + for ts, data in item.items(): + nodes = data.get('nodes', {}) + + for path, resource in nodes.items(): + obj_id = int(path.split('/')[1]) + if len(nodes) > 1 or len(val) > 1: + if not event_created: + self.create_event(endpoint, obj_id) + event_created = True + + if ts == 'null': + self.handle_resource(endpoint, obj_id, resource) + else: + timestamp = parser.parse(ts) + self.handle_resource(endpoint, obj_id, resource, timestamp) return endpoint From 87231edecbdf7569b29f14eabdd37424f867e1a9 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Sat, 14 Sep 2024 09:32:05 +0200 Subject: [PATCH 16/18] squash: leshan: timestamped Signed-off-by: Jonas Remmert --- .../leshan/src/main/java/lwm2m/LeshanSvr.java | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/server/leshan/src/main/java/lwm2m/LeshanSvr.java b/server/leshan/src/main/java/lwm2m/LeshanSvr.java index d38158b0..5a30dac2 100644 --- a/server/leshan/src/main/java/lwm2m/LeshanSvr.java +++ b/server/leshan/src/main/java/lwm2m/LeshanSvr.java @@ -56,7 +56,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import java.time.Instant; +import java.util.Map; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -407,18 +410,36 @@ public MySendListener(LeshanSvr server) { this.server = server; } - @Override public void dataReceived(Registration registration, TimestampedLwM2mNodes data, SendRequest request) { + mapper.enable(SerializationFeature.INDENT_OUTPUT); + log.trace("dataReceived from: " + registration.getEndpoint()); - mapper.enable(SerializationFeature.INDENT_OUTPUT); - ObjectNode node = mapper.createObjectNode(); - node.put("ep", registration.getEndpoint()); - node.set("val", mapper.valueToTree(data)); + ObjectNode rootNode = mapper.createObjectNode(); + rootNode.put("ep", registration.getEndpoint()); + + ArrayNode valArray = mapper.createArrayNode(); + + for (Instant ts: data.getTimestamps()) { + ObjectNode tsNode = mapper.createObjectNode(); + + ObjectNode nodesNode = mapper.createObjectNode(); + Map nodesAtTimestamp = data.getNodesAt(ts); + if (nodesAtTimestamp != null) { + for (Map.Entry e : nodesAtTimestamp.entrySet()) { + ObjectNode nodeDetails = mapper.valueToTree(e.getValue()); + nodesNode.set(e.getKey().toString(), nodeDetails); + } + } + + tsNode.set("nodes", nodesNode); + valArray.addObject().set(ts == null ? "null" : ts.toString(), tsNode); + } - dataSenderRest.sendData(ApiPath.TIMESTAMPED_RES, node); + rootNode.set("val", valArray); + dataSenderRest.sendData(ApiPath.TIMESTAMPED_RES, rootNode); } @Override From 62707720c8be1bd279baa7ee343740e379c275ee Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Sat, 14 Sep 2024 09:32:58 +0200 Subject: [PATCH 17/18] squash: django: test: timestamped resource Signed-off-by: Jonas Remmert --- .../tests/test_post_timestamped_resource.py | 74 ++++++++++++++----- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/server/django/sensordata/tests/test_post_timestamped_resource.py b/server/django/sensordata/tests/test_post_timestamped_resource.py index 55af8eea..b7b6951c 100644 --- a/server/django/sensordata/tests/test_post_timestamped_resource.py +++ b/server/django/sensordata/tests/test_post_timestamped_resource.py @@ -13,25 +13,63 @@ TEST_PAYLOAD = ''' { - "ep" : "urn:imei:100000000000000", - "val" : { - "empty" : false, - "timestamps" : [ null ], - "nodes" : { - "/3303/0/5700" : { - "kind" : "singleResource", - "id" : 5700, - "type" : "FLOAT", - "value" : "22.19110046564394" - }, - "/3304/0/5700" : { - "kind" : "singleResource", - "id" : 5700, - "type" : "FLOAT", - "value" : "52.237076014801175" + "ep": "urn:imei:100000000000000", + "val": [ + { + "null": { + "nodes": { + "/3303/0/5700": { + "kind": "singleResource", + "id": 5700, + "type": "FLOAT", + "value": "20.145982801948204" + }, + "/3304/0/5700": { + "kind": "singleResource", + "id": 5700, + "type": "FLOAT", + "value": "51.200783720985235" + } + } + } + }, + { + "2024-09-13T19:06:17.032059888Z": { + "nodes": { + "/10300/0/0": { + "kind": "singleResource", + "id": 0, + "type": "INTEGER", + "value": "8" + } + } + } + }, + { + "2024-09-13T19:06:18.032059888Z": { + "nodes": { + "/10300/0/0": { + "kind": "singleResource", + "id": 0, + "type": "INTEGER", + "value": "9" + } + } + } + }, + { + "2024-09-13T19:06:19.032059888Z": { + "nodes": { + "/10300/0/0": { + "kind": "singleResource", + "id": 0, + "type": "INTEGER", + "value": "10" + } + } } } - } + ] } ''' @@ -44,6 +82,8 @@ def setUp(self): name="temperature", data_type=ResourceType.FLOAT) ResourceType.objects.create(object_id=3304, resource_id=5700, name="humidity", data_type=ResourceType.FLOAT) + ResourceType.objects.create(object_id=10300, resource_id=0, + name="humidity", data_type=ResourceType.INTEGER) def test_post_composite_resource(self): From 75e4c1e4ab9dcee9ce3173c13c41059cf8bd0029 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Sat, 14 Sep 2024 11:19:46 +0200 Subject: [PATCH 18/18] simulation: add time series test object to lwm2m client Signed-off-by: Jonas Remmert --- simulation/lwm2m_client/prj.conf | 8 ++ simulation/lwm2m_client/src/lwm2m-client.c | 3 + simulation/lwm2m_client/src/modules.h | 1 + .../lwm2m_client/src/time_series_object.c | 135 ++++++++++++++++++ 4 files changed, 147 insertions(+) create mode 100644 simulation/lwm2m_client/src/time_series_object.c diff --git a/simulation/lwm2m_client/prj.conf b/simulation/lwm2m_client/prj.conf index 200639b9..d73b022c 100644 --- a/simulation/lwm2m_client/prj.conf +++ b/simulation/lwm2m_client/prj.conf @@ -76,3 +76,11 @@ CONFIG_LWM2M_RW_SENML_CBOR_SUPPORT=y CONFIG_LWM2M_RW_JSON_SUPPORT=n CONFIG_LWM2M_RW_SENML_JSON_SUPPORT=n CONFIG_ZCBOR_CANONICAL=y + # Time series support +CONFIG_LWM2M_RW_SENML_CBOR_SUPPORT=y +CONFIG_LWM2M_RW_SENML_CBOR_RECORDS=100 +CONFIG_LWM2M_RESOURCE_DATA_CACHE_SUPPORT=y +CONFIG_LWM2M_COAP_BLOCK_TRANSFER=y +CONFIG_LWM2M_COAP_ENCODE_BUFFER_SIZE=4096 +CONFIG_POSIX_CLOCK=y +CONFIG_RING_BUFFER=y diff --git a/simulation/lwm2m_client/src/lwm2m-client.c b/simulation/lwm2m_client/src/lwm2m-client.c index f1a2f72b..b079b846 100644 --- a/simulation/lwm2m_client/src/lwm2m-client.c +++ b/simulation/lwm2m_client/src/lwm2m-client.c @@ -191,6 +191,9 @@ static int lwm2m_setup(void) /* setup TEMP SENSOR object */ init_temp_sensor(&client_ctx); + /* setup TIME SERIES object */ + init_time_series_obj(&client_ctx); + /* Set multiple TEMP SENSOR resource values in one function call. */ int err = lwm2m_set_bulk(temp_sensor_items, ARRAY_SIZE(temp_sensor_items)); diff --git a/simulation/lwm2m_client/src/modules.h b/simulation/lwm2m_client/src/modules.h index 4e145da8..48cb0797 100644 --- a/simulation/lwm2m_client/src/modules.h +++ b/simulation/lwm2m_client/src/modules.h @@ -14,5 +14,6 @@ void init_timer_object(void); void init_temp_sensor(struct lwm2m_ctx *client); void init_firmware_update(struct lwm2m_ctx *client); +void init_time_series_obj(struct lwm2m_ctx *client); #endif diff --git a/simulation/lwm2m_client/src/time_series_object.c b/simulation/lwm2m_client/src/time_series_object.c new file mode 100644 index 00000000..5c266d11 --- /dev/null +++ b/simulation/lwm2m_client/src/time_series_object.c @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2024 Jonas Remmert + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#define LOG_MODULE_NAME app_time_series_obj +#include +LOG_MODULE_REGISTER(LOG_MODULE_NAME); + +#include "modules.h" +#include +#include +#include + +#define N_SAMPLES 40 +#define PERIOD K_SECONDS(1) + +#define TIME_SERIES_VERSION_MAJOR 1 +#define TIME_SERIES_VERSION_MINOR 0 +#define LWM2M_TIME_SERIES_ID 10300 +#define LWM2M_TIME_SERIES_RAW_ID 0 +#define TIME_SERIES_MAX_ID 1 +/* + * Calculate resource instances as follows: + * start with EVENT_LOG_MAX_ID + * subtract EXEC resources (0) + */ +#define RESOURCE_INSTANCE_COUNT (TIME_SERIES_MAX_ID - 0) + +static struct k_work_delayable time_series_work; + +static struct lwm2m_ctx *client_ctx; + +/* resource state variables */ +static uint8_t time_series_raw; +/* Allocate data cache storage */ +static struct lwm2m_time_series_elem time_series_cache[N_SAMPLES]; + +static struct lwm2m_engine_obj lwm2m_time_series_obj; +static struct lwm2m_engine_obj_field fields[] = { + OBJ_FIELD_DATA(LWM2M_TIME_SERIES_RAW_ID, R, U8), +}; + +static struct lwm2m_engine_obj_inst inst; +static struct lwm2m_engine_res res[TIME_SERIES_MAX_ID]; +static struct lwm2m_engine_res_inst res_inst[RESOURCE_INSTANCE_COUNT]; + + +void time_series_send_cb(enum lwm2m_send_status status) +{ + if (status == 0) { + LOG_INF("Time series data sent successfully"); + } else { + LOG_ERR("Time series data send failed"); + } + + /* Once the previous message is out, add new data */ + k_work_schedule(&time_series_work, PERIOD); +} + +static void time_series_work_cb(struct k_work *work) +{ + int ret; + static int sample; + + /* Assign a test payload that counts from 1 - N_SAMPLES */ + lwm2m_set_u8(&LWM2M_OBJ(LWM2M_TIME_SERIES_ID, 0, + LWM2M_TIME_SERIES_RAW_ID), sample); + if (sample < N_SAMPLES) { + sample++; + k_work_schedule(&time_series_work, PERIOD); + LOG_INF("Sample: %d/%d", sample, N_SAMPLES); + return; + } + + LOG_INF("Sending time series data"); + ret = lwm2m_send_cb(client_ctx, &LWM2M_OBJ(LWM2M_TIME_SERIES_ID, 0, + LWM2M_TIME_SERIES_RAW_ID), 1, + time_series_send_cb); + if (ret) { + LOG_ERR("lwm2m_send_cb, error: %d", ret); + } + sample = 0; +} + +static struct lwm2m_engine_obj_inst *lwm2m_time_series_obj_create(uint16_t obj_inst_id) +{ + int i = 0, j = 0; + + init_res_instance(res_inst, ARRAY_SIZE(res_inst)); + + /* initialize instance resource data */ + INIT_OBJ_RES_DATA(LWM2M_TIME_SERIES_RAW_ID, res, i, res_inst, j, + &time_series_raw, sizeof(time_series_raw)); + + inst.resources = res; + inst.resource_count = i; + + LOG_INF("Created LwM2M Time Series instance: %d", obj_inst_id); + return &inst; +} + +void init_time_series_obj(struct lwm2m_ctx *client) +{ + client_ctx = client; + struct lwm2m_engine_obj_inst *obj_inst = NULL; + int ret = 0; + + /* initialize the Event Log field data */ + lwm2m_time_series_obj.obj_id = LWM2M_TIME_SERIES_ID; + lwm2m_time_series_obj.version_major = TIME_SERIES_VERSION_MAJOR; + lwm2m_time_series_obj.version_minor = TIME_SERIES_VERSION_MINOR; + lwm2m_time_series_obj.is_core = false; + lwm2m_time_series_obj.fields = fields; + lwm2m_time_series_obj.field_count = ARRAY_SIZE(fields); + lwm2m_time_series_obj.max_instance_count = 1U; + lwm2m_time_series_obj.create_cb = lwm2m_time_series_obj_create; + lwm2m_register_obj(&lwm2m_time_series_obj); + + /* auto create the first instance */ + ret = lwm2m_create_obj_inst(LWM2M_TIME_SERIES_ID, 0, &obj_inst); + if (ret < 0) { + LOG_ERR("Create LWM2M Pump Mon instance 0 error: %d", ret); + } + + ret = lwm2m_enable_cache(&LWM2M_OBJ(LWM2M_TIME_SERIES_ID, 0, LWM2M_TIME_SERIES_RAW_ID), + time_series_cache, ARRAY_SIZE(time_series_cache)); + if (ret < 0) { + LOG_ERR("Failed to enable cache for LWM2M Pump Mon instance 0: %d", ret); + } + + k_work_init_delayable(&time_series_work, time_series_work_cb); + k_work_schedule(&time_series_work, PERIOD); +}