Skip to content

Commit

Permalink
Refactor stop as an event loop
Browse files Browse the repository at this point in the history
  • Loading branch information
andchiind committed Feb 27, 2025
1 parent 39bbcb6 commit 67e11b1
Show file tree
Hide file tree
Showing 9 changed files with 131 additions and 65 deletions.
2 changes: 1 addition & 1 deletion src/isar/config/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def __init__(self) -> None:
# Determines the number of state transitions that are kept in the log
STATE_TRANSITIONS_LOG_LENGTH: int = Field(default=20)

# Number of attempts to initiate a task or mission before cancelling
# Number of attempts to initiate a mission before cancelling
INITIATE_FAILURE_COUNTER_LIMIT: int = Field(default=10)

# Number of attempts to request a task status in monitor before cancelling
Expand Down
2 changes: 2 additions & 0 deletions src/isar/models/communication/queues/events.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ def __init__(self) -> None:
self.mission_started: Queue = Queue(maxsize=1)
self.mission_failed: Queue = Queue(maxsize=1)
self.robot_status_changed: Queue = Queue(maxsize=1)
self.mission_failed_to_stop: Queue = Queue(maxsize=1)
self.mission_successfully_stopped: Queue = Queue(maxsize=1)


class SharedState:
Expand Down
23 changes: 23 additions & 0 deletions src/isar/robot/robot.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from isar.models.communication.queues.queue_utils import check_for_event
from isar.robot.robot_start_mission import RobotStartMissionThread
from isar.robot.robot_status import RobotStatusThread
from isar.robot.robot_stop_mission import RobotStopMissionThread
from isar.robot.robot_task_status import RobotTaskStatusThread
from robot_interface.robot_interface import RobotInterface

Expand All @@ -26,6 +27,7 @@ def __init__(
self.start_mission_thread: Optional[RobotStartMissionThread] = None
self.robot_status_thread: Optional[RobotStatusThread] = None
self.robot_task_status_thread: Optional[RobotTaskStatusThread] = None
self.stop_mission_thread_thread: Optional[RobotStopMissionThread] = None
self.signal_thread_quitting: Event = Event()

def stop(self) -> None:
Expand All @@ -42,6 +44,11 @@ def stop(self) -> None:
and self.robot_status_thread.is_alive()
):
self.start_mission_thread.join()
if (
self.stop_mission_thread_thread is not None
and self.stop_mission_thread_thread.is_alive()
):
self.stop_mission_thread_thread.join()
self.robot_status_thread = None
self.robot_task_status_thread = None
self.start_mission_thread = None
Expand Down Expand Up @@ -73,6 +80,18 @@ def _check_and_handle_task_status_request(self, event: Queue) -> None:
)
self.robot_task_status_thread.start()

def _check_and_handle_stop_mission(self, event: Queue) -> None:
if check_for_event(event):
if (
self.stop_mission_thread_thread is not None
and self.start_mission_thread.is_alive()
):
return
self.stop_mission_thread_thread = RobotStopMissionThread(
self.events, self.robot, self.signal_thread_quitting
)
self.stop_mission_thread_thread.start()

def run(self) -> None:
self.robot_status_thread = RobotStatusThread(
self.robot, self.signal_thread_quitting, self.shared_state
Expand All @@ -91,4 +110,8 @@ def run(self) -> None:
self.events.state_machine_events.task_status_request
)

self._check_and_handle_stop_mission(
self.events.state_machine_events.stop_mission
)

self.logger.info("Exiting robot service main thread")
3 changes: 0 additions & 3 deletions src/isar/robot/robot_start_mission.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
import logging
from threading import Event, Thread
from typing import Optional

from isar.config.settings import settings
from isar.models.communication.queues.events import Events
from isar.models.communication.queues.queue_utils import trigger_event
from isar.services.utilities.threaded_request import ThreadedRequest
from robot_interface.models.exceptions.robot_exceptions import (
ErrorMessage,
RobotException,
Expand All @@ -27,7 +25,6 @@ def __init__(
self.logger = logging.getLogger("robot")
self.events: Events = events
self.robot: RobotInterface = robot
self.start_mission_thread: Optional[ThreadedRequest] = None
self.signal_thread_quitting: Event = signal_thread_quitting
self.mission = mission
Thread.__init__(self, name="Robot start mission thread")
Expand Down
61 changes: 61 additions & 0 deletions src/isar/robot/robot_stop_mission.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import logging
import time
from threading import Event, Thread
from typing import Optional

from isar.config.settings import settings
from isar.models.communication.queues.events import Events
from isar.models.communication.queues.queue_utils import trigger_event
from robot_interface.models.exceptions.robot_exceptions import (
RobotActionException,
RobotException,
)
from robot_interface.robot_interface import RobotInterface


class RobotStopMissionThread(Thread):

def __init__(
self,
events: Events,
robot: RobotInterface,
signal_thread_quitting: Event,
):
self.logger = logging.getLogger("robot")
self.events: Events = events
self.robot: RobotInterface = robot
self.signal_thread_quitting: Event = signal_thread_quitting
Thread.__init__(self, name="Robot start mission thread")

def run(self) -> None:
retries = 0
error_description: Optional[str] = None
while retries < settings.STOP_ROBOT_ATTEMPTS_LIMIT:
if self.signal_thread_quitting.wait(0):
return

try:
self.robot.stop()
except (RobotActionException, RobotException) as e:
self.logger.warning(
f"\nFailed to stop robot because: {e.error_description}"
f"\nAttempting to stop the robot again"
)
retries += 1
error_description = e.error_description
time.sleep(settings.FSM_SLEEP_TIME)
continue

trigger_event(self.events.robot_service_events.mission_successfully_stopped)
return

error_message = (
f"\nFailed to stop the robot after {retries} attempts because: "
f"{error_description}"
f"\nBe aware that the robot may still be moving even though a stop has "
"been attempted"
)

trigger_event(
self.events.robot_service_events.mission_failed_to_stop, error_message
)
8 changes: 6 additions & 2 deletions src/isar/state_machine/state_machine.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@
set_mission_to_in_progress,
trigger_start_mission_or_task_event,
)
from isar.state_machine.transitions.stop import stop_mission
from isar.state_machine.transitions.stop import (
stop_mission_cleanup,
trigger_stop_mission_event,
)
from isar.state_machine.transitions.utils import def_transition
from robot_interface.models.exceptions.robot_exceptions import ErrorMessage
from robot_interface.models.mission.mission import Mission
Expand Down Expand Up @@ -135,6 +138,7 @@ def __init__(
self.paused_state,
],
"dest": self.stop_state,
"before": def_transition(self, trigger_stop_mission_event),
},
{
"trigger": "request_mission_start",
Expand Down Expand Up @@ -177,7 +181,7 @@ def __init__(
"trigger": "mission_stopped",
"source": self.stop_state,
"dest": self.idle_state,
"before": def_transition(self, stop_mission),
"before": def_transition(self, stop_mission_cleanup),
},
{
"trigger": "robot_turned_offline",
Expand Down
86 changes: 28 additions & 58 deletions src/isar/state_machine/states/stop.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,12 @@
import logging
import time
from typing import TYPE_CHECKING, Callable, Optional
from queue import Queue
from typing import TYPE_CHECKING, Optional

from transitions import State

from isar.services.utilities.threaded_request import (
ThreadedRequest,
ThreadedRequestNotFinishedError,
)
from robot_interface.models.exceptions.robot_exceptions import (
ErrorMessage,
RobotActionException,
RobotException,
)
from isar.models.communication.queues.queue_utils import check_for_event
from isar.services.utilities.threaded_request import ThreadedRequest

if TYPE_CHECKING:
from isar.state_machine.state_machine import StateMachine
Expand All @@ -23,6 +17,7 @@ def __init__(self, state_machine: "StateMachine") -> None:
super().__init__(name="stop", on_enter=self.start, on_exit=self.stop)
self.state_machine: "StateMachine" = state_machine
self.logger = logging.getLogger("state_machine")
self.events = self.state_machine.events
self.stop_thread: Optional[ThreadedRequest] = None
self._count_number_retries: int = 0

Expand All @@ -36,56 +31,31 @@ def stop(self) -> None:
self.stop_thread = None
self._count_number_retries = 0

def _run(self) -> None:
transition: Callable
while True:
if not self.stop_thread:
self.stop_thread = ThreadedRequest(self.state_machine.robot.stop)
self.stop_thread.start_thread(name="State Machine Stop Robot")

try:
self.stop_thread.get_output()
except ThreadedRequestNotFinishedError:
time.sleep(self.state_machine.sleep_time)
continue

except (RobotActionException, RobotException) as e:
if self.handle_stop_fail(
retry_limit=self.state_machine.stop_robot_attempts_limit,
error_message=ErrorMessage(
error_reason=e.error_reason,
error_description=e.error_description,
),
):
transition = self.state_machine.mission_stopped # type: ignore
break

self.logger.warning(
f"\nFailed to stop robot because: {e.error_description}"
f"\nAttempting to stop the robot again"
)

self.stop_thread = None
continue

transition = self.state_machine.mission_stopped # type: ignore
def _check_and_handle_failed_stop(self, event: Queue) -> bool:
error_message = check_for_event(event)
if error_message is not None:
self.logger.warning(error_message)
self.state_machine.mission_stopped() # type: ignore
return True
return False

break
def _check_and_handle_successful_stop(self, event: Queue) -> bool:
if check_for_event(event):
self.state_machine.mission_stopped() # type: ignore
return True
return False

transition()
def _run(self) -> None:
while True:

def handle_stop_fail(self, retry_limit: int, error_message: ErrorMessage) -> bool:
self._count_number_retries += 1
if self._count_number_retries > retry_limit:
self.state_machine.current_task.error_message = error_message
if self._check_and_handle_failed_stop(
self.events.robot_service_events.mission_failed_to_stop
):
break

self.logger.error(
f"\nFailed to stop the robot after {retry_limit} attempts because: "
f"{error_message.error_description}"
f"\nBe aware that the robot may still be moving even though a stop has "
"been attempted"
)
if self._check_and_handle_successful_stop(
self.events.robot_service_events.mission_successfully_stopped
):
break

return True
time.sleep(self.state_machine.sleep_time)
return False
time.sleep(self.state_machine.sleep_time)
9 changes: 8 additions & 1 deletion src/isar/state_machine/transitions/stop.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,20 @@
from typing import TYPE_CHECKING

from isar.models.communication.queues.queue_utils import trigger_event

if TYPE_CHECKING:
from isar.state_machine.state_machine import StateMachine

from isar.apis.models.models import ControlMissionResponse
from robot_interface.models.mission.status import MissionStatus, TaskStatus


def stop_mission(state_machine: "StateMachine") -> bool:
def trigger_stop_mission_event(state_machine: "StateMachine") -> bool:
trigger_event(state_machine.events.state_machine_events.stop_mission)
return True


def stop_mission_cleanup(state_machine: "StateMachine") -> bool:
if state_machine.current_mission is None:
state_machine._queue_empty_response()
state_machine.reset_state_machine()
Expand Down
2 changes: 2 additions & 0 deletions tests/isar/state_machine/test_state_machine.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,8 @@ def test_state_machine_with_successful_mission_stop(
time.sleep(3)
scheduling_utilities.stop_mission()

time.sleep(1)

assert state_machine_thread.state_machine.transitions_list == deque(
[States.Idle, States.Monitor, States.Stop, States.Idle]
)
Expand Down

0 comments on commit 67e11b1

Please sign in to comment.