Makes it possible to raise an exception if the checksum is invalid (#83)

* introduces optional error_if_checksum_invalid keyword (default=False)

* adds the possibility to raise an exception if the checksum is invalid

* version bump

docs/examples/single.rst

@@ -4,6 +4,9 @@ Decode a single AIS message
 
 You can decode AIVDM/AIVDO messages, as long as they are valid NMEA 0183 messages.
 
+Please note that invalid checksums are ignored. If you want to raise an error for
+invalid checksums set `error_if_checksum_invalid=True`.
+
 References
 ----------
 
@@ -76,3 +79,16 @@ Decode a stream of messages (e.g. a list or generator)::
     ]
     for msg in IterMessages(fake_stream):
         print(msg.decode())
+
+Note
+--------
+This library is often used for data analysis. This means that a researcher
+analyzes large amounts of AIS messages. Such message streams might contain
+thousands of messages with invalid checksums. Its up to the researcher to
+decide whether he/she wants to include such messages in his/her analysis.
+Raising an exception for every invalid checksum would both cause a
+performance degradation because handling of such exceptions is expensive
+and make it impossible to include such messages into the analysis.
+
+If you want to raise an error if the checksum of a message is invalid set
+the key word argument `error_if_checksum_invalid` to True.

pyais/__init__.py

@@ -4,7 +4,7 @@
 from pyais.decode import decode
 
 __license__ = 'MIT'
-__version__ = '2.1.4'
+__version__ = '2.2.0'
 __author__ = 'Leon Morten Richter'
 
 __all__ = (

pyais/decode.py

@@ -1,16 +1,23 @@
 import typing
 
-from pyais.exceptions import TooManyMessagesException, MissingMultipartMessageException
+from pyais.exceptions import (
+    TooManyMessagesException,
+    MissingMultipartMessageException,
+    InvalidNMEAChecksum
+)
 from pyais.messages import NMEAMessage, ANY_MESSAGE
 
 
-def _assemble_messages(*args: bytes) -> NMEAMessage:
+def _assemble_messages(*args: bytes, error_if_checksum_invalid: bool = False) -> NMEAMessage:
     # Convert bytes into NMEAMessage and remember fragment_count and fragment_numbers
     temp: typing.List[NMEAMessage] = []
     frags: typing.List[int] = []
     frag_cnt: int = 1
     for msg in args:
         nmea = NMEAMessage(msg)
+        if error_if_checksum_invalid and not nmea.is_valid:
+            raise InvalidNMEAChecksum(f'The checksum is invalid for message "{nmea.raw!r}"')
+
         temp.append(nmea)
         frags.append(nmea.frag_num)
         frag_cnt = nmea.fragment_count
@@ -29,7 +36,32 @@ def _assemble_messages(*args: bytes) -> NMEAMessage:
     return final
 
 
-def decode(*args: typing.Union[str, bytes]) -> ANY_MESSAGE:
+def decode(*args: typing.Union[str, bytes], error_if_checksum_invalid: bool = False) -> ANY_MESSAGE:
+    """
+    Decodes an AIS message.
+    For multi part messages all parts are required.
+
+    :param args: all parts of the AIS message to decode.
+    :param error_if_checksum_invalid: Raise an error if the checksum of
+                                      any part is invalid. (Default=False)
+    :returns: The decoded message
+    :raises InvalidNMEAChecksum: raised when the NMEA checksum is invalid.
+    :raises MissingMultipartMessageException: raised when there are missing parts for multi part messages.
+    :raises TooManyMessagesException: raised when more than one message is provided.
+                                      NOTE: multiple parts for the SAME message are allowed.
+
+    NOTE:
+        This library is often used for data analysis. This means that a researcher
+        analyzes large amounts of AIS messages. Such message streams might contain
+        thousands of messages with invalid checksums. Its up to the researcher to
+        decide whether he/she wants to include such messages in his/her analysis.
+        Raising an exception for every invalid checksum would both cause a
+        performance degradation because handling of such exceptions is expensive
+        and make it impossible to include such messages into the analysis.
+
+        If you want to raise an error if the checksum of a message is invalid set
+        the key word argument `error_if_checksum_invalid` to True.
+    """
     parts = tuple(msg.encode('utf-8') if isinstance(msg, str) else msg for msg in args)
-    nmea = _assemble_messages(*parts)
+    nmea = _assemble_messages(*parts, error_if_checksum_invalid=error_if_checksum_invalid)
     return nmea.decode()

pyais/exceptions.py

@@ -7,6 +7,10 @@ class InvalidNMEAMessageException(AISBaseException):
     pass
 
 
+class InvalidNMEAChecksum(AISBaseException):
+    """Invalid Checksum for the NMEA message"""
+
+
 class UnknownMessageException(AISBaseException):
     """Message not supported yet"""
     pass

pyais/messages.py

@@ -160,6 +160,7 @@ class NMEAMessage(object):
         'payload',
         'fill_bits',
         'checksum',
+        'is_valid',
         'bit_array'
     )
 
@@ -217,6 +218,9 @@ def __init__(self, raw: bytes) -> None:
         self.bit_array: bitarray = decode_into_bit_array(self.payload, self.fill_bits)
         self.ais_id: int = get_int(self.bit_array, 0, 6)
 
+        # Set the checksum valid field
+        self.is_valid = self.checksum == compute_checksum(self.raw)
+
     def __str__(self) -> str:
         return str(self.raw)
 
@@ -247,6 +251,7 @@ def asdict(self) -> Dict[str, Any]:
             'fill_bits': self.fill_bits,  # int
             'checksum': self.checksum,  # int
             'bit_array': self.bit_array.to01(),  # str
+            'is_valid': self.is_valid,  # bool
         }
 
     def decode_and_merge(self, enum_as_int: bool = False) -> Dict[str, Any]:
@@ -283,23 +288,22 @@ def assemble_from_iterable(cls, messages: Sequence["NMEAMessage"]) -> "NMEAMessa
         raw = b''
         data = b''
         bit_array = bitarray()
+        is_valid = True
 
         for i, msg in enumerate(sorted(messages, key=lambda m: m.frag_num)):
             if i > 0:
                 raw += b'\n'
             raw += msg.raw
             data += msg.payload
             bit_array += msg.bit_array
+            is_valid &= msg.is_valid
 
         messages[0].raw = raw
         messages[0].payload = data
         messages[0].bit_array = bit_array
+        messages[0].is_valid = is_valid
         return messages[0]
 
-    @property
-    def is_valid(self) -> bool:
-        return self.checksum == compute_checksum(self.raw)
-
     @property
     def is_single(self) -> bool:
         return not self.seq_id and self.frag_num == self.frag_cnt == 1

tests/test_decode.py

@@ -12,7 +12,7 @@
                              NavigationStatus, ShipType, StationType, SyncState,
                              TransmitMode)
 from pyais.decode import decode
-from pyais.exceptions import UnknownMessageException
+from pyais.exceptions import InvalidNMEAChecksum, UnknownMessageException
 from pyais.messages import (MSG_CLASS, MessageType5, MessageType6,
                             MessageType18, MessageType22Addressed,
                             MessageType22Broadcast, MessageType24PartA,
@@ -924,6 +924,7 @@ def test_decode_and_merge(self):
                 'ais_id': 21,
                 'assigned': None,
                 'channel': 'B',
+                'is_valid': True,
                 'checksum': 82,
                 'epfd': 7,
                 'fill_bits': 2,
@@ -961,6 +962,7 @@ def test_decode_and_merge(self):
                 'assigned': None,
                 'channel': 'B',
                 'checksum': 82,
+                'is_valid': True,
                 'epfd': EpfdType.Surveyed,
                 'fill_bits': 2,
                 'frag_cnt': 1,
@@ -1431,3 +1433,18 @@ def test_special_position_report(self):
         self.assertEqual(decoded.heading, 104)
         self.assertEqual(decoded.second, 41)
         self.assertEqual(decoded.raim, 0)
+
+    def test_decode_does_not_raise_an_error_if_error_if_checksum_invalid_is_false(self):
+        raw = b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*FF"
+        msg = decode(raw, error_if_checksum_invalid=False)
+        self.assertIsNotNone(msg,)
+
+    def test_decode_does_not_raise_an_error_by_default(self):
+        raw = b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*FF"
+        msg = decode(raw)
+        self.assertIsNotNone(msg)
+
+    def test_decode_does_raise_an_error_if_error_if_checksum_invalid_is_true(self):
+        raw = b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*FF"
+        with self.assertRaises(InvalidNMEAChecksum):
+            _ = decode(raw, error_if_checksum_invalid=True)

tests/test_nmea.py

@@ -1,7 +1,7 @@
 import unittest
-from pprint import pprint
 
 from bitarray import bitarray
+from pyais.decode import _assemble_messages
 
 from pyais.exceptions import InvalidNMEAMessageException
 from pyais.messages import NMEAMessage
@@ -131,7 +131,6 @@ def serializable(o: object):
         )
 
         actual = msg.asdict()
-        pprint(actual)
         self.assertEqual(expected, actual)
         self.assertEqual(1, actual["ais_id"])
         self.assertEqual("!AIVDM,1,1,,A,15Mj23P000G?q7fK>g:o7@1:0L3S,0*1B", actual["raw"])
@@ -196,3 +195,38 @@ def test_chk_to_int_with_missing_fill_bits(self):
         self.assertEqual(chk_to_int(b""), (0, -1))
         with self.assertRaises(ValueError):
             self.assertEqual(chk_to_int(b"*1B"), (0, 24))
+
+    def test_that_a_valid_checksum_is_correctly_identified(self):
+        raw = b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05"
+        msg = NMEAMessage(raw)
+        self.assertTrue(msg.is_valid)
+
+    def test_that_an_invalid_checksum_is_correctly_identified(self):
+        raw = b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*04"
+        msg = NMEAMessage(raw)
+        self.assertFalse(msg.is_valid)
+
+    def test_that_a_valid_checksum_is_correctly_identified_for_multi_part_msgs(self):
+        sentences = [
+            b"!AIVDM,2,1,4,A,55O0W7`00001L@gCWGA2uItLth@DqtL5@F22220j1h742t0Ht0000000,0*08",
+            b"!AIVDM,2,2,4,A,000000000000000,2*20",
+        ]
+        msg = _assemble_messages(*sentences)
+        self.assertTrue(msg.is_valid)
+
+    def test_that_an_invalid_checksum_is_correctly_identified_for_multi_part_msgs(self):
+        # The first sentence has an invalid checksum
+        sentences = [
+            b"!AIVDM,2,1,4,A,55O0W7`00001L@gCWGA2uItLth@DqtL5@F22220j1h742t0Ht0000000,0*09",
+            b"!AIVDM,2,2,4,A,000000000000000,2*20",
+        ]
+        msg = _assemble_messages(*sentences)
+        self.assertFalse(msg.is_valid)
+
+        # The second sentence has an invalid checksum
+        sentences = [
+            b"!AIVDM,2,1,4,A,55O0W7`00001L@gCWGA2uItLth@DqtL5@F22220j1h742t0Ht0000000,0*08",
+            b"!AIVDM,2,2,4,A,000000000000000,2*21",
+        ]
+        msg = _assemble_messages(*sentences)
+        self.assertFalse(msg.is_valid)