diff --git a/docs/examples/single.rst b/docs/examples/single.rst index 4d9a34c..54504d4 100644 --- a/docs/examples/single.rst +++ b/docs/examples/single.rst @@ -4,6 +4,9 @@ Decode a single AIS message You can decode AIVDM/AIVDO messages, as long as they are valid NMEA 0183 messages. +Please note that invalid checksums are ignored. If you want to raise an error for +invalid checksums set `error_if_checksum_invalid=True`. + References ---------- @@ -76,3 +79,16 @@ Decode a stream of messages (e.g. a list or generator):: ] for msg in IterMessages(fake_stream): print(msg.decode()) + +Note +-------- +This library is often used for data analysis. This means that a researcher +analyzes large amounts of AIS messages. Such message streams might contain +thousands of messages with invalid checksums. Its up to the researcher to +decide whether he/she wants to include such messages in his/her analysis. +Raising an exception for every invalid checksum would both cause a +performance degradation because handling of such exceptions is expensive +and make it impossible to include such messages into the analysis. + +If you want to raise an error if the checksum of a message is invalid set +the key word argument `error_if_checksum_invalid` to True. \ No newline at end of file diff --git a/pyais/__init__.py b/pyais/__init__.py index eed1712..9620903 100644 --- a/pyais/__init__.py +++ b/pyais/__init__.py @@ -4,7 +4,7 @@ from pyais.decode import decode __license__ = 'MIT' -__version__ = '2.1.4' +__version__ = '2.2.0' __author__ = 'Leon Morten Richter' __all__ = ( diff --git a/pyais/decode.py b/pyais/decode.py index d9d082f..4eed717 100644 --- a/pyais/decode.py +++ b/pyais/decode.py @@ -1,16 +1,23 @@ import typing -from pyais.exceptions import TooManyMessagesException, MissingMultipartMessageException +from pyais.exceptions import ( + TooManyMessagesException, + MissingMultipartMessageException, + InvalidNMEAChecksum +) from pyais.messages import NMEAMessage, ANY_MESSAGE -def _assemble_messages(*args: bytes) -> NMEAMessage: +def _assemble_messages(*args: bytes, error_if_checksum_invalid: bool = False) -> NMEAMessage: # Convert bytes into NMEAMessage and remember fragment_count and fragment_numbers temp: typing.List[NMEAMessage] = [] frags: typing.List[int] = [] frag_cnt: int = 1 for msg in args: nmea = NMEAMessage(msg) + if error_if_checksum_invalid and not nmea.is_valid: + raise InvalidNMEAChecksum(f'The checksum is invalid for message "{nmea.raw!r}"') + temp.append(nmea) frags.append(nmea.frag_num) frag_cnt = nmea.fragment_count @@ -29,7 +36,32 @@ def _assemble_messages(*args: bytes) -> NMEAMessage: return final -def decode(*args: typing.Union[str, bytes]) -> ANY_MESSAGE: +def decode(*args: typing.Union[str, bytes], error_if_checksum_invalid: bool = False) -> ANY_MESSAGE: + """ + Decodes an AIS message. + For multi part messages all parts are required. + + :param args: all parts of the AIS message to decode. + :param error_if_checksum_invalid: Raise an error if the checksum of + any part is invalid. (Default=False) + :returns: The decoded message + :raises InvalidNMEAChecksum: raised when the NMEA checksum is invalid. + :raises MissingMultipartMessageException: raised when there are missing parts for multi part messages. + :raises TooManyMessagesException: raised when more than one message is provided. + NOTE: multiple parts for the SAME message are allowed. + + NOTE: + This library is often used for data analysis. This means that a researcher + analyzes large amounts of AIS messages. Such message streams might contain + thousands of messages with invalid checksums. Its up to the researcher to + decide whether he/she wants to include such messages in his/her analysis. + Raising an exception for every invalid checksum would both cause a + performance degradation because handling of such exceptions is expensive + and make it impossible to include such messages into the analysis. + + If you want to raise an error if the checksum of a message is invalid set + the key word argument `error_if_checksum_invalid` to True. + """ parts = tuple(msg.encode('utf-8') if isinstance(msg, str) else msg for msg in args) - nmea = _assemble_messages(*parts) + nmea = _assemble_messages(*parts, error_if_checksum_invalid=error_if_checksum_invalid) return nmea.decode() diff --git a/pyais/exceptions.py b/pyais/exceptions.py index d4137b9..6f312e9 100644 --- a/pyais/exceptions.py +++ b/pyais/exceptions.py @@ -7,6 +7,10 @@ class InvalidNMEAMessageException(AISBaseException): pass +class InvalidNMEAChecksum(AISBaseException): + """Invalid Checksum for the NMEA message""" + + class UnknownMessageException(AISBaseException): """Message not supported yet""" pass diff --git a/pyais/messages.py b/pyais/messages.py index 0025c21..5354153 100644 --- a/pyais/messages.py +++ b/pyais/messages.py @@ -160,6 +160,7 @@ class NMEAMessage(object): 'payload', 'fill_bits', 'checksum', + 'is_valid', 'bit_array' ) @@ -217,6 +218,9 @@ def __init__(self, raw: bytes) -> None: self.bit_array: bitarray = decode_into_bit_array(self.payload, self.fill_bits) self.ais_id: int = get_int(self.bit_array, 0, 6) + # Set the checksum valid field + self.is_valid = self.checksum == compute_checksum(self.raw) + def __str__(self) -> str: return str(self.raw) @@ -247,6 +251,7 @@ def asdict(self) -> Dict[str, Any]: 'fill_bits': self.fill_bits, # int 'checksum': self.checksum, # int 'bit_array': self.bit_array.to01(), # str + 'is_valid': self.is_valid, # bool } def decode_and_merge(self, enum_as_int: bool = False) -> Dict[str, Any]: @@ -283,6 +288,7 @@ def assemble_from_iterable(cls, messages: Sequence["NMEAMessage"]) -> "NMEAMessa raw = b'' data = b'' bit_array = bitarray() + is_valid = True for i, msg in enumerate(sorted(messages, key=lambda m: m.frag_num)): if i > 0: @@ -290,16 +296,14 @@ def assemble_from_iterable(cls, messages: Sequence["NMEAMessage"]) -> "NMEAMessa raw += msg.raw data += msg.payload bit_array += msg.bit_array + is_valid &= msg.is_valid messages[0].raw = raw messages[0].payload = data messages[0].bit_array = bit_array + messages[0].is_valid = is_valid return messages[0] - @property - def is_valid(self) -> bool: - return self.checksum == compute_checksum(self.raw) - @property def is_single(self) -> bool: return not self.seq_id and self.frag_num == self.frag_cnt == 1 diff --git a/tests/test_decode.py b/tests/test_decode.py index 18a4122..d50e35c 100644 --- a/tests/test_decode.py +++ b/tests/test_decode.py @@ -12,7 +12,7 @@ NavigationStatus, ShipType, StationType, SyncState, TransmitMode) from pyais.decode import decode -from pyais.exceptions import UnknownMessageException +from pyais.exceptions import InvalidNMEAChecksum, UnknownMessageException from pyais.messages import (MSG_CLASS, MessageType5, MessageType6, MessageType18, MessageType22Addressed, MessageType22Broadcast, MessageType24PartA, @@ -924,6 +924,7 @@ def test_decode_and_merge(self): 'ais_id': 21, 'assigned': None, 'channel': 'B', + 'is_valid': True, 'checksum': 82, 'epfd': 7, 'fill_bits': 2, @@ -961,6 +962,7 @@ def test_decode_and_merge(self): 'assigned': None, 'channel': 'B', 'checksum': 82, + 'is_valid': True, 'epfd': EpfdType.Surveyed, 'fill_bits': 2, 'frag_cnt': 1, @@ -1431,3 +1433,18 @@ def test_special_position_report(self): self.assertEqual(decoded.heading, 104) self.assertEqual(decoded.second, 41) self.assertEqual(decoded.raim, 0) + + def test_decode_does_not_raise_an_error_if_error_if_checksum_invalid_is_false(self): + raw = b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*FF" + msg = decode(raw, error_if_checksum_invalid=False) + self.assertIsNotNone(msg,) + + def test_decode_does_not_raise_an_error_by_default(self): + raw = b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*FF" + msg = decode(raw) + self.assertIsNotNone(msg) + + def test_decode_does_raise_an_error_if_error_if_checksum_invalid_is_true(self): + raw = b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*FF" + with self.assertRaises(InvalidNMEAChecksum): + _ = decode(raw, error_if_checksum_invalid=True) diff --git a/tests/test_nmea.py b/tests/test_nmea.py index e274ed4..6a37827 100644 --- a/tests/test_nmea.py +++ b/tests/test_nmea.py @@ -1,7 +1,7 @@ import unittest -from pprint import pprint from bitarray import bitarray +from pyais.decode import _assemble_messages from pyais.exceptions import InvalidNMEAMessageException from pyais.messages import NMEAMessage @@ -131,7 +131,6 @@ def serializable(o: object): ) actual = msg.asdict() - pprint(actual) self.assertEqual(expected, actual) self.assertEqual(1, actual["ais_id"]) self.assertEqual("!AIVDM,1,1,,A,15Mj23P000G?q7fK>g:o7@1:0L3S,0*1B", actual["raw"]) @@ -196,3 +195,38 @@ def test_chk_to_int_with_missing_fill_bits(self): self.assertEqual(chk_to_int(b""), (0, -1)) with self.assertRaises(ValueError): self.assertEqual(chk_to_int(b"*1B"), (0, 24)) + + def test_that_a_valid_checksum_is_correctly_identified(self): + raw = b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*05" + msg = NMEAMessage(raw) + self.assertTrue(msg.is_valid) + + def test_that_an_invalid_checksum_is_correctly_identified(self): + raw = b"!AIVDM,1,1,,B,15NG6V0P01G?cFhE`R2IU?wn28R>,0*04" + msg = NMEAMessage(raw) + self.assertFalse(msg.is_valid) + + def test_that_a_valid_checksum_is_correctly_identified_for_multi_part_msgs(self): + sentences = [ + b"!AIVDM,2,1,4,A,55O0W7`00001L@gCWGA2uItLth@DqtL5@F22220j1h742t0Ht0000000,0*08", + b"!AIVDM,2,2,4,A,000000000000000,2*20", + ] + msg = _assemble_messages(*sentences) + self.assertTrue(msg.is_valid) + + def test_that_an_invalid_checksum_is_correctly_identified_for_multi_part_msgs(self): + # The first sentence has an invalid checksum + sentences = [ + b"!AIVDM,2,1,4,A,55O0W7`00001L@gCWGA2uItLth@DqtL5@F22220j1h742t0Ht0000000,0*09", + b"!AIVDM,2,2,4,A,000000000000000,2*20", + ] + msg = _assemble_messages(*sentences) + self.assertFalse(msg.is_valid) + + # The second sentence has an invalid checksum + sentences = [ + b"!AIVDM,2,1,4,A,55O0W7`00001L@gCWGA2uItLth@DqtL5@F22220j1h742t0Ht0000000,0*08", + b"!AIVDM,2,2,4,A,000000000000000,2*21", + ] + msg = _assemble_messages(*sentences) + self.assertFalse(msg.is_valid)