Merge pull request #76 from miniscope/error-tracebacks

preserve original traceback in a note

Author: sneakers-the-rat

src/noob/network/message.py

@@ -86,6 +86,12 @@ class AnnounceValue(TypedDict):
     nodes: dict[str, IdentifyValue]
 
 
+class ErrorValue(TypedDict):
+    err_type: type[Exception]
+    err_args: tuple
+    traceback: str
+
+
 class AnnounceMsg(Message):
     """Command node 'announces' identities of other peers and the events they emit"""
 
@@ -132,7 +138,7 @@ class ErrorMsg(Message):
     """An error occurred in one of the processing nodes"""
 
     type_: Literal[MessageType.error] = Field(MessageType.error, alias="type")
-    value: Picklable[Exception]
+    value: Picklable[ErrorValue]
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 

src/noob/runner/zmq.py

@@ -26,6 +26,7 @@
 import os
 import signal
 import threading
+import traceback
 from collections import defaultdict
 from collections.abc import Callable, Generator
 from dataclasses import dataclass, field
@@ -54,6 +55,7 @@
     AnnounceMsg,
     AnnounceValue,
     ErrorMsg,
+    ErrorValue,
     EventMsg,
     IdentifyMsg,
     IdentifyValue,
@@ -563,8 +565,20 @@ def on_stop(self, msg: StopMsg) -> None:
         os.kill(pid, signal.SIGTERM)
 
     def error(self, err: Exception) -> None:
-        self.logger.debug("Throwing error in main runner: %s", err)
-        msg = ErrorMsg(node_id=self.spec.id, value=err)
+        """
+        Capture the error and traceback context from an exception using
+        :class:`traceback.TracebackException` and send to command node to re-raise
+        """
+        tbexception = "\n".join(traceback.format_tb(err.__traceback__))
+        self.logger.debug("Throwing error in main runner: %s", tbexception)
+        msg = ErrorMsg(
+            node_id=self.spec.id,
+            value=ErrorValue(
+                err_type=type(err),
+                err_args=err.args,
+                traceback=tbexception,
+            ),
+        )
         self._dealer.send_multipart([msg.to_bytes()])
 
 
@@ -583,7 +597,7 @@ class ZMQRunner(TubeRunner):
     _running: EventType = field(default_factory=mp.Event)
     _return_node: Return | None = None
     _init_lock: threading.Lock = field(default_factory=threading.Lock)
-    _to_throw: Exception | None = None
+    _to_throw: ErrorValue | None = None
 
     @property
     def running(self) -> bool:
@@ -708,14 +722,25 @@ def _handle_error(self, msg: ErrorMsg) -> None:
         self.tube.scheduler.end_epoch(self._current_epoch)
 
     def _throw_error(self) -> None:
-        err = self._to_throw
-        if err is None:
+        errval = self._to_throw
+        if errval is None:
             return
+        # clear instance object and store locally, we aren't locked here.
         self._to_throw = None
         self._logger.debug(
             "Deinitializing before throwing error",
         )
         self.deinit()
+
+        # add the traceback as a note,
+        # sort of the best we can do without using tblib
+        err = errval["err_type"](*errval["err_args"])
+        tb_message = "\nError re-raised from node runner process\n\n"
+        tb_message += "Original traceback:\n"
+        tb_message += "-" * 20 + "\n"
+        tb_message += errval["traceback"]
+        err.add_note(tb_message)
+
         raise err
 
     def enable_node(self, node_id: str) -> None:

tests/test_runners/test_zmq.py

@@ -47,5 +47,15 @@ def test_error_reporting():
     """When a zmq runner node has an error, it sends it back to the command node"""
     tube = Tube.from_specification("testing-error")
     runner = ZMQRunner(tube)
-    with pytest.raises(ValueError, match="This node just emits errors"):
+    with pytest.raises(ValueError) as exc_info:
         runner.process()
+
+    assert exc_info.type is ValueError
+    # original error message
+    assert str(exc_info.value) == "This node just emits errors"
+    # additional information in the notes
+    note = exc_info.value.__notes__[0]
+    # notice that this error was raised in another process
+    assert "Error re-raised from node runner process" in note
+    # the location of the original exception
+    assert "testing/nodes.py" in note