From d3c99b7e86cb137931b9a2c2ca3f9e25eecdd672 Mon Sep 17 00:00:00 2001 From: Nikita Date: Wed, 26 Jun 2024 11:21:27 +0000 Subject: [PATCH 01/48] =?UTF-8?q?In=20node.py:=20expanded=20the=20log=20in?= =?UTF-8?q?formation=20for=20some=20values=20=E2=80=8B=E2=80=8Bto=20make?= =?UTF-8?q?=20it=20easier=20to=20analyze=20the=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In raft_log_replication.py: start to write test that log replication happens correctly if leadership changes In raft_node.py: add a couple of lines of code to close node after each test, so that after running pytest it does not crash new issue: after the leader changes, the new leader is not aware of the latest index of the neighboring nodes and begins to rewrite them, even though his nodes completely coincide with the neighboring node. --- cyraft/node.py | 35 +++- tests/raft_log_replication.py | 306 +++++++++++++++++++++++++++++++++- tests/raft_node.py | 123 +------------- 3 files changed, 342 insertions(+), 122 deletions(-) diff --git a/cyraft/node.py b/cyraft/node.py index 6fc3070..3c8741b 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -208,8 +208,11 @@ async def _serve_request_vote_impl( if request.term < self._term or (self._voted_for is not None): _logger.info( c["request_vote"] - + "Request vote request denied (term < self._term or already voted for another candidate in this term)" - + c["end_color"] + + "Request vote request denied (term (%d) < self._term (%d) or already voted for another candidate in this term (%s))" + + c["end_color"], + request.term, + self._term, + self._voted_for, ) return sirius_cyber_corp.RequestVote_1.Response(term=self._term, vote_granted=False) @@ -338,15 +341,18 @@ async def _serve_append_entries( self._voted_for = metadata.client_node_id self._change_state(RaftState.FOLLOWER) # this will reset the election timeout as well self._term = request.term # update term + return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=True) # Reply false if term < currentTerm (§5.1) if request.term < self._term: _logger.info( c["append_entries"] - + "Node ID: %d -- Append entries request denied (term < currentTerm)" + + "Node ID: %d -- Append entries request denied (term (%d) < currentTerm (%d))" + c["end_color"], self._node.id, + request.term, + self._term ) return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) @@ -617,6 +623,17 @@ def _change_state(self, new_state: RaftState) -> None: if self._state == RaftState.FOLLOWER: # Cancel the term timeout (if it exists), and schedule a new election timeout. + for remote_node_index, remote_next_index in enumerate(self._next_index): + _logger.info( + c["raft_logic"] + + "Node ID: %d -- Value of next index = %d and value of remote next index = %s for node %s" + + c["end_color"], + self._node.id, + self._commit_index, + remote_next_index, + self._cluster[remote_node_index], + ) + if hasattr(self, "_term_timer"): self._term_timer.cancel() # FOLLOWER should not have term timer self._reset_election_timeout() # FOLLOWER should have election timer @@ -629,6 +646,7 @@ def _change_state(self, new_state: RaftState) -> None: self._election_timer.cancel() elif self._state == RaftState.LEADER: assert self._prev_state == RaftState.CANDIDATE, "Invalid state change 3" + # Cancel the election timeout (if it exists), and schedule a new term timeout. if hasattr(self, "_election_timer"): self._election_timer.cancel() @@ -691,6 +709,17 @@ async def _on_term_timeout(self) -> None: if self._state != RaftState.LEADER: # if the node is no longer a leader, stop sending heartbeats break + + _logger.info( + c["raft_logic"] + + "Node ID: %d -- Value of next index = %d and value of remote next index = %s for node %s" + + c["end_color"], + self._node.id, + self._commit_index, + remote_next_index, + self._cluster[remote_node_index], + ) + if self._commit_index + 1 == remote_next_index: # remote log is up to date _logger.info( c["raft_logic"] diff --git a/tests/raft_log_replication.py b/tests/raft_log_replication.py index 9db773e..a3b403b 100644 --- a/tests/raft_log_replication.py +++ b/tests/raft_log_replication.py @@ -593,7 +593,309 @@ async def _unittest_raft_log_replication() -> None: assert raft_node_1._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" assert raft_node_1._log[4].entry.value == 13 + raft_node_1.close() + raft_node_2.close() + raft_node_3.close() + await asyncio.sleep(1) + +# ======================================================================================= +# ========== Test that log replication happens correctly if leadership changes ========== +# ======================================================================================= + async def _unittest_raft_leader_changes() -> None: - # TODO: Test that log replication happens correctly if leadership changes - pass + + logging.root.setLevel(logging.INFO) + + os.environ["UAVCAN__SRV__REQUEST_VOTE__ID"] = "1" + os.environ["UAVCAN__CLN__REQUEST_VOTE__ID"] = "1" + os.environ["UAVCAN__SRV__APPEND_ENTRIES__ID"] = "2" + os.environ["UAVCAN__CLN__APPEND_ENTRIES__ID"] = "2" + + TERM_TIMEOUT = 0.5 + ELECTION_TIMEOUT = 5 + + os.environ["UAVCAN__NODE__ID"] = "41" + raft_node_1 = RaftNode() + raft_node_1.term_timeout = TERM_TIMEOUT + raft_node_1.election_timeout = ELECTION_TIMEOUT + os.environ["UAVCAN__NODE__ID"] = "42" + raft_node_2 = RaftNode() + raft_node_2.term_timeout = TERM_TIMEOUT + raft_node_2.election_timeout = ELECTION_TIMEOUT + 1 + os.environ["UAVCAN__NODE__ID"] = "43" + raft_node_3 = RaftNode() + raft_node_3.term_timeout = TERM_TIMEOUT + raft_node_3.election_timeout = ELECTION_TIMEOUT + 1 + + # make all part of the same cluster + cluster = [raft_node_1._node.id, raft_node_2._node.id, raft_node_3._node.id] + raft_node_1.add_remote_node(cluster) + raft_node_2.add_remote_node(cluster) + raft_node_3.add_remote_node(cluster) + + # start all nodes + asyncio.create_task(raft_node_1.run()) + asyncio.create_task(raft_node_2.run()) + asyncio.create_task(raft_node_3.run()) + + # wait for the leader to be elected + await asyncio.sleep(ELECTION_TIMEOUT + 1) + + # check if the leader is elected + assert raft_node_1._state == RaftState.LEADER + assert raft_node_2._state == RaftState.FOLLOWER + assert raft_node_3._state == RaftState.FOLLOWER + assert raft_node_1._voted_for == 41 + assert raft_node_2._voted_for == 41 + assert raft_node_3._voted_for == 41 + + new_entries = [ + sirius_cyber_corp.LogEntry_1( + term=4, + entry=sirius_cyber_corp.Entry_1( + name=uavcan.primitive.String_1(value="top_1"), + value=7, + ), + ), + sirius_cyber_corp.LogEntry_1( + term=5, + entry=sirius_cyber_corp.Entry_1( + name=uavcan.primitive.String_1(value="top_2"), + value=8, + ), + ), + sirius_cyber_corp.LogEntry_1( + term=6, + entry=sirius_cyber_corp.Entry_1( + name=uavcan.primitive.String_1(value="top_3"), + value=9, + ), + ), + ] + + for index, new_entry in enumerate(new_entries): + request = sirius_cyber_corp.AppendEntries_1.Request( + term=raft_node_1._term, + prev_log_index=index, # prev_log_index: 0, 1, 2 + prev_log_term=raft_node_1._log[index].term, + log_entry=new_entry, + ) + metadata = pycyphal.presentation.ServiceRequestMetadata( + client_node_id=42, + timestamp=time.time(), + priority=0, + transfer_id=0, + ) + response = await raft_node_1._serve_append_entries(request, metadata) + assert response.success == True + _logger.info("=========Start=========") + # wait for the request to be replicated + await asyncio.sleep(TERM_TIMEOUT + 1) + + # check if the new entry is replicated in the leader node + assert len(raft_node_1._log) == 1 + 3 + assert raft_node_1._log[0].term == 0 + assert raft_node_1._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty + assert raft_node_1._log[0].entry.value == 0 + assert raft_node_1._log[1].term == 4 + assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_1._log[1].entry.value == 7 + assert raft_node_1._log[2].term == 5 + assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_1._log[2].entry.value == 8 + assert raft_node_1._log[3].term == 6 + assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_1._log[3].entry.value == 9 + assert raft_node_1._commit_index == 3 + + # check if the new entry is replicated in the follower nodes + assert len(raft_node_2._log) == 1 + 3 + assert raft_node_2._log[0].term == 0 + assert raft_node_2._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty + assert raft_node_2._log[0].entry.value == 0 + assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_2._log[1].entry.value == 7 + assert raft_node_2._log[2].term == 5 + assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_2._log[2].entry.value == 8 + assert raft_node_2._log[3].term == 6 + assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_2._log[3].entry.value == 9 + assert raft_node_2._commit_index == 3 + + assert len(raft_node_3._log) == 1 + 3 + assert raft_node_3._log[0].term == 0 + assert raft_node_3._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty + assert raft_node_3._log[0].entry.value == 0 + assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_3._log[1].entry.value == 7 + assert raft_node_3._log[2].term == 5 + assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_3._log[2].entry.value == 8 + assert raft_node_3._log[3].term == 6 + assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_3._log[3].entry.value == 9 + assert raft_node_3._commit_index == 3 + + await asyncio.sleep(TERM_TIMEOUT + 1) + _logger.info("================== TEST 1: change LEADER and check logs ==================") + + # New LEADER => raft_node_2 + + raft_node_1.election_timeout = ELECTION_TIMEOUT + 1 + raft_node_2.election_timeout = ELECTION_TIMEOUT + raft_node_3.election_timeout = ELECTION_TIMEOUT + 1 + + raft_node_1._change_state(RaftState.FOLLOWER) + raft_node_2._change_state(RaftState.FOLLOWER) + raft_node_3._change_state(RaftState.FOLLOWER) + raft_node_1._voted_for = None + + + await asyncio.sleep(ELECTION_TIMEOUT + 1) + + assert raft_node_1._state == RaftState.FOLLOWER + assert raft_node_1._term == 1, "received heartbeat from LEADER" + assert raft_node_1._voted_for == 42 + + assert raft_node_2._state == RaftState.LEADER + assert raft_node_2._term == 1 + assert raft_node_2._voted_for == 42 + + assert raft_node_3._state == RaftState.FOLLOWER + assert raft_node_3._term == 1, "received heartbeat from LEADER" + assert raft_node_3._voted_for == 42 + + # check that all logs are saved from previous LEADER + assert len(raft_node_1._log) == 1 + 3 + assert raft_node_1._log[0].term == 0 + assert raft_node_1._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty + assert raft_node_1._log[0].entry.value == 0 + assert raft_node_1._log[1].term == 4 + assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_1._log[1].entry.value == 7 + assert raft_node_1._log[2].term == 5 + assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_1._log[2].entry.value == 8 + assert raft_node_1._log[3].term == 6 + assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_1._log[3].entry.value == 9 + assert raft_node_1._commit_index == 3 + + assert len(raft_node_2._log) == 1 + 3 + assert raft_node_2._log[0].term == 0 + assert raft_node_2._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty + assert raft_node_2._log[0].entry.value == 0 + assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_2._log[1].entry.value == 7 + assert raft_node_2._log[2].term == 5 + assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_2._log[2].entry.value == 8 + assert raft_node_2._log[3].term == 6 + assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_2._log[3].entry.value == 9 + assert raft_node_2._commit_index == 3 + + assert len(raft_node_3._log) == 1 + 3 + assert raft_node_3._log[0].term == 0 + assert raft_node_3._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty + assert raft_node_3._log[0].entry.value == 0 + assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_3._log[1].entry.value == 7 + assert raft_node_3._log[2].term == 5 + assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_3._log[2].entry.value == 8 + assert raft_node_3._log[3].term == 6 + assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_3._log[3].entry.value == 9 + assert raft_node_3._commit_index == 3 + + _logger.info("================== TEST 2: append new log entry fron another LEADER ==================") + + new_entry = sirius_cyber_corp.LogEntry_1( + term=7, + entry=sirius_cyber_corp.Entry_1( + name=uavcan.primitive.String_1(value="top_4"), + value=13, + ), + ) + request = sirius_cyber_corp.AppendEntries_1.Request( + term=raft_node_2._term, + prev_log_index=3, # index of top_3 + prev_log_term=raft_node_2._log[3].term, + log_entry=new_entry, + ) + metadata = pycyphal.presentation.ServiceRequestMetadata( + client_node_id=43, + timestamp=time.time(), + priority=0, + transfer_id=0, + ) + + response = await raft_node_2._serve_append_entries(request, metadata) + _logger.info("=========END=========") + assert response.success == True + + await asyncio.sleep(TERM_TIMEOUT + 1) + + assert len(raft_node_2._log) == 1 + 4 + assert raft_node_2._log[0].term == 0 + assert raft_node_2._log[0].entry.value == 0 + assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_2._log[1].entry.value == 7 + assert raft_node_2._log[2].term == 5 + assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_2._log[2].entry.value == 8 + assert raft_node_2._log[3].term == 6 + assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_2._log[3].entry.value == 9 + assert raft_node_2._log[4].term == 7 + assert raft_node_2._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" + assert raft_node_2._log[4].entry.value == 13 + assert raft_node_2._commit_index == 4 + + assert len(raft_node_1._log) == 1 + 4 + assert raft_node_1._log[0].term == 0 + assert raft_node_1._log[0].entry.value == 0 + assert raft_node_1._log[1].term == 4 + assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_1._log[1].entry.value == 7 + assert raft_node_1._log[2].term == 5 + assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_1._log[2].entry.value == 8 + assert raft_node_1._log[3].term == 6 + assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_1._log[3].entry.value == 9 + assert raft_node_1._log[4].term == 7 + assert raft_node_1._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" + assert raft_node_1._log[4].entry.value == 13 + assert raft_node_1._commit_index == 4 + + assert len(raft_node_3._log) == 1 + 4 + assert raft_node_3._log[0].term == 0 + assert raft_node_3._log[0].entry.value == 0 + assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_3._log[1].entry.value == 7 + assert raft_node_3._log[2].term == 5 + assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_3._log[2].entry.value == 8 + assert raft_node_3._log[3].term == 6 + assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_3._log[3].entry.value == 9 + assert raft_node_3._log[4].term == 7 + assert raft_node_3._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" + assert raft_node_3._log[4].entry.value == 13 + assert raft_node_3._commit_index == 4 + + + raft_node_1.close() + raft_node_2.close() + raft_node_3.close() + await asyncio.sleep(1) \ No newline at end of file diff --git a/tests/raft_node.py b/tests/raft_node.py index 8ee51de..a185436 100644 --- a/tests/raft_node.py +++ b/tests/raft_node.py @@ -95,6 +95,8 @@ async def _unittest_raft_node_init() -> None: assert len(raft_node._request_vote_clients) == 1 assert len(raft_node._append_entries_clients) == 1 assert raft_node._next_index == [1] + raft_node.close() + await asyncio.sleep(1) # assert raft_node._match_index == [0] @@ -155,123 +157,6 @@ async def _unittest_raft_node_election_timeout() -> None: await asyncio.sleep(1) # give some time for the node to close -async def _unittest_raft_node_heartbeat() -> None: - """ - Test that the node does NOT convert to candidate if it receives a heartbeat message - """ - os.environ["UAVCAN__NODE__ID"] = "41" - raft_node = RaftNode() - ELECTION_TIMEOUT = 5 - TERM_TIMEOUT = 1 - raft_node.election_timeout = ELECTION_TIMEOUT - raft_node.term_timeout = TERM_TIMEOUT - raft_node._voted_for = 42 - - asyncio.create_task(raft_node.run()) - await asyncio.sleep(ELECTION_TIMEOUT * 0.90) # sleep until right before election timeout - - # send heartbeat - terms_passed = raft_node._term # leader's term is equal to the follower's term - await raft_node._serve_append_entries( - sirius_cyber_corp.AppendEntries_1.Request( - term=terms_passed, # leader's term - prev_log_index=0, # index of log entry immediately preceding new ones - prev_log_term=0, # term of prevLogIndex entry - leader_commit=0, # leader's commitIndex - log_entry=None, # log entries to store (empty for heartbeat) - ), - pycyphal.presentation.ServiceRequestMetadata( - client_node_id=42, # leader's node id - timestamp=time.time(), # leader's timestamp - priority=0, # leader's priority - transfer_id=0, # leader's transfer id - ), - ) - - # wait for heartbeat to be processed [election is reached but shouldn't become leader due to heartbeat] - await asyncio.sleep(ELECTION_TIMEOUT * 0.1 + 0.1) - assert raft_node._state == RaftState.FOLLOWER - assert raft_node._voted_for == 42 - - # send heartbeat again - # (this time leader has a higher term, we want to make sure that the follower's term is updated) - await asyncio.sleep(ELECTION_TIMEOUT * 0.90) # sleep until right before election timeout - terms_passed = raft_node._term + 5 # leader's term is higher than the follower's term - await raft_node._serve_append_entries( - sirius_cyber_corp.AppendEntries_1.Request( - term=terms_passed, # leader's term - prev_log_index=0, # index of log entry immediately preceding new ones - prev_log_term=0, # term of prevLogIndex entry - leader_commit=0, # leader's commitIndex - log_entry=None, # log entries to store (empty for heartbeat) - ), - pycyphal.presentation.ServiceRequestMetadata( - client_node_id=42, # leader's node id - timestamp=time.time(), # leader's timestamp - priority=0, # leader's priority - transfer_id=0, # leader's transfer id - ), - ) - - # wait for heartbeat to be processed [election is reached but shouldn't become leader due to heartbeat] - await asyncio.sleep(ELECTION_TIMEOUT * 0.1 + 0.1) - assert raft_node._state == RaftState.FOLLOWER - assert raft_node._voted_for == 42 - assert raft_node._term == terms_passed - - # send heartbeat again - # (this time from a different leader with a higher term, we want to make sure the follower switches leader and updates term) - await asyncio.sleep(ELECTION_TIMEOUT * 0.90) # sleep until right before election timeout - terms_passed = raft_node._term + 5 # leader's term is higher than the follower's term - await raft_node._serve_append_entries( - sirius_cyber_corp.AppendEntries_1.Request( - term=terms_passed, # leader's term - prev_log_index=0, # index of log entry immediately preceding new ones - prev_log_term=0, # term of prevLogIndex entry - leader_commit=0, # leader's commitIndex - log_entry=None, # log entries to store (empty for heartbeat) - ), - pycyphal.presentation.ServiceRequestMetadata( - client_node_id=43, # leader's node id (different from previous leader) - timestamp=time.time(), # leader's timestamp - priority=0, # leader's priority - transfer_id=0, # leader's transfer id - ), - ) - - # wait for heartbeat to be processed [election is reached but shouldn't become leader due to heartbeat] - await asyncio.sleep(ELECTION_TIMEOUT * 0.1 + 0.1) - assert raft_node._state == RaftState.FOLLOWER - assert raft_node._voted_for == 43 - assert raft_node._term == terms_passed - - # send heartbeat again - # (this time the leader's term is lower than the follower's term, we want to make sure the follower doesn't switch leader) - await asyncio.sleep(ELECTION_TIMEOUT * 0.90) # sleep until right before election timeout - terms_passed = raft_node._term - 1 # leader's term is lower than the follower's term - await raft_node._serve_append_entries( - sirius_cyber_corp.AppendEntries_1.Request( - term=terms_passed, # leader's term - prev_log_index=0, # index of log entry immediately preceding new ones - prev_log_term=0, # term of prevLogIndex entry - leader_commit=0, # leader's commitIndex - log_entry=None, # log entries to store (empty for heartbeat) - ), - pycyphal.presentation.ServiceRequestMetadata( - client_node_id=42, # old leader's node id, which has lower term - timestamp=time.time(), # leader's timestamp - priority=0, # leader's priority - transfer_id=0, # leader's transfer id - ), - ) - - ## test that the node converts to candidate after the election timeout [no valid heartbeat is received] - await asyncio.sleep(ELECTION_TIMEOUT * 0.1 + 0.1) - assert raft_node._prev_state == RaftState.CANDIDATE - - raft_node.close() - await asyncio.sleep(1) # fixes when just running this test, however not when "pytest /cyraft" is run - async def _unittest_raft_node_request_vote_rpc() -> None: """ @@ -332,6 +217,8 @@ async def _unittest_raft_node_request_vote_rpc() -> None: assert raft_node._voted_for == 42 assert response.vote_granted == True assert raft_node._term == request.term # follower node term is updated to candidate's term + raft_node.close() + await asyncio.sleep(1) async def _unittest_raft_node_start_election() -> None: @@ -780,3 +667,5 @@ async def _unittest_raft_node_append_entries_rpc() -> None: assert raft_node._log[3].entry.value == 12 assert raft_node._log[4].term == 10 assert raft_node._log[4].entry.value == 13 + raft_node.close() + await asyncio.sleep(1) From 1429deeacf6f432bc4a89526230cfb77a4f3414b Mon Sep 17 00:00:00 2001 From: Nikita Date: Fri, 28 Jun 2024 09:07:52 +0000 Subject: [PATCH 02/48] Added test _unittest_raft_leader_changes for log_replication.py --- README.md | 4 +- cyraft/node.py | 31 ++++---- tests/raft_log_replication.py | 128 +++++++++++++++++++++++++++++++--- tests/raft_node.py | 2 + 4 files changed, 137 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 45e4ad0..e81b27a 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,9 @@ This feature is significant because it enables Cyphal to serve as a communicatio - [ ] some warning need to be fixed here: - [ ] _unittest_raft_node_term_timeout - [ ] _unittest_raft_node_start_election - - [ ] `log_replication.py` + - [x] `log_replication.py` - [x] `_unittest_raft_log_replication` - - [ ] `_unittest_raft_leader_changes` + - [x] `_unittest_raft_leader_changes` - [ ] `leader_commit` needs to integrated/tested - [ ] Test using orchestration so there's 3 nodes running simultanously - [ ] use yakut to send AppendEntries requests diff --git a/cyraft/node.py b/cyraft/node.py index 3c8741b..5641de1 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -88,6 +88,8 @@ def __init__(self) -> None: ## Volatile state on leaders self._next_index: typing.List[int] = [] # index of the next log entry to send to that server + + # self._match_index: typing.List[int] = [] # index of highest log entry known to be replicated on server ######################################## @@ -367,15 +369,19 @@ async def _serve_append_entries( self._node.id, ) return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) - except IndexError: + except IndexError as e: _logger.info( - c["append_entries"] + "Node ID: %d -- Append entries request denied (log mismatch 2)" + c["end_color"], - self._node.id, - ) + c["append_entries"] + + "Node ID: %d -- Append entries request denied (log mismatch 2). IndexError: %s. " + + "prev_log_index: %d, log_length: %d" + + c["end_color"], + self._node.id, + str(e), + request.prev_log_index, + len(self._log)) return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) self._append_entries_processing(request) - return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=True) def _append_entries_processing( @@ -448,7 +454,9 @@ def _append_entries_processing( # Update current_term (if follower) (leaders will update their own term on timeout) if self._state == RaftState.FOLLOWER: + self._change_state(RaftState.FOLLOWER) # this will reset the election timeout as well self._term = request.log_entry[0].term + async def _send_heartbeat(self, remote_node_index: int) -> None: """ @@ -623,17 +631,6 @@ def _change_state(self, new_state: RaftState) -> None: if self._state == RaftState.FOLLOWER: # Cancel the term timeout (if it exists), and schedule a new election timeout. - for remote_node_index, remote_next_index in enumerate(self._next_index): - _logger.info( - c["raft_logic"] - + "Node ID: %d -- Value of next index = %d and value of remote next index = %s for node %s" - + c["end_color"], - self._node.id, - self._commit_index, - remote_next_index, - self._cluster[remote_node_index], - ) - if hasattr(self, "_term_timer"): self._term_timer.cancel() # FOLLOWER should not have term timer self._reset_election_timeout() # FOLLOWER should have election timer @@ -715,7 +712,7 @@ async def _on_term_timeout(self) -> None: + "Node ID: %d -- Value of next index = %d and value of remote next index = %s for node %s" + c["end_color"], self._node.id, - self._commit_index, + self._commit_index + 1, remote_next_index, self._cluster[remote_node_index], ) diff --git a/tests/raft_log_replication.py b/tests/raft_log_replication.py index a3b403b..7948eb0 100644 --- a/tests/raft_log_replication.py +++ b/tests/raft_log_replication.py @@ -602,6 +602,44 @@ async def _unittest_raft_log_replication() -> None: # ========== Test that log replication happens correctly if leadership changes ========== # ======================================================================================= +""" + Initially, all nodes have an empty log entry at index 0 with term 0. + ____________ + | 0 | Log index + | 0 | Log term + | empty <= 0 | Name <= value + |____________| + + Step 1: Append 3 Log Entries to LEADER node 41 + + ____________ ____________ ____________ ____________ + | 0 | 1 | 2 | 3 | Log index + | 0 | 4 | 5 | 6 | Log term + | empty <= 0 | top_1 <= 7 | top_2 <= 8 | top_3 <= 9 | Name <= value + |____________|____________|____________|____________| + + Step 2: Leadership Change to Node 42 and Add New Entry + ____________ ____________ ____________ ____________ _____________ + | 0 | 1 | 2 | 3 | 4 | Log index + | 0 | 4 | 5 | 6 | 7 | Log term + | empty <= 0 | top_1 <= 7 | top_2 <= 8 | top_3 <= 9 | top_4 <= 17 | Name <= value + |____________|____________|____________|____________|_____________| + + Step 3: Replace Log Entry 3 with a New Entry fom LEADER Node 42 + ____________ + | 3 | Log index + | 7 | Log term + | top_3 <= 10| Name <= value + |____________| + + Result: + ____________ ____________ ____________ _____________ + | 0 | 1 | 2 | 3 | Log index + | 0 | 4 | 5 | 7 | Log term + | empty <= 0 | top_1 <= 7 | top_2 <= 8 | top_3 <= 10 | Name <= value + |____________|____________|____________|_____________| + +""" async def _unittest_raft_leader_changes() -> None: @@ -682,14 +720,14 @@ async def _unittest_raft_leader_changes() -> None: log_entry=new_entry, ) metadata = pycyphal.presentation.ServiceRequestMetadata( - client_node_id=42, + client_node_id=41, timestamp=time.time(), priority=0, transfer_id=0, ) response = await raft_node_1._serve_append_entries(request, metadata) assert response.success == True - _logger.info("=========Start=========") + # wait for the request to be replicated await asyncio.sleep(TERM_TIMEOUT + 1) @@ -741,7 +779,7 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_3._commit_index == 3 await asyncio.sleep(TERM_TIMEOUT + 1) - _logger.info("================== TEST 1: change LEADER and check logs ==================") + _logger.info("================== TEST 1: Change LEADER and check logs ==================") # New LEADER => raft_node_2 @@ -755,18 +793,18 @@ async def _unittest_raft_leader_changes() -> None: raft_node_1._voted_for = None - await asyncio.sleep(ELECTION_TIMEOUT + 1) + await asyncio.sleep(2*ELECTION_TIMEOUT + 1) assert raft_node_1._state == RaftState.FOLLOWER - assert raft_node_1._term == 1, "received heartbeat from LEADER" + assert raft_node_1._term == 20, "received heartbeat from LEADER" assert raft_node_1._voted_for == 42 assert raft_node_2._state == RaftState.LEADER - assert raft_node_2._term == 1 + assert raft_node_2._term == 20 assert raft_node_2._voted_for == 42 assert raft_node_3._state == RaftState.FOLLOWER - assert raft_node_3._term == 1, "received heartbeat from LEADER" + assert raft_node_3._term == 20, "received heartbeat from LEADER" assert raft_node_3._voted_for == 42 # check that all logs are saved from previous LEADER @@ -815,7 +853,7 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_3._log[3].entry.value == 9 assert raft_node_3._commit_index == 3 - _logger.info("================== TEST 2: append new log entry fron another LEADER ==================") + _logger.info("================== TEST 2: Append new log entry fron another LEADER ==================") new_entry = sirius_cyber_corp.LogEntry_1( term=7, @@ -838,7 +876,7 @@ async def _unittest_raft_leader_changes() -> None: ) response = await raft_node_2._serve_append_entries(request, metadata) - _logger.info("=========END=========") + assert response.success == True await asyncio.sleep(TERM_TIMEOUT + 1) @@ -894,6 +932,78 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_3._log[4].entry.value == 13 assert raft_node_3._commit_index == 4 + _logger.info("================== TEST 3: Replace log entries 1 and 4 with new entries from another LEADER ==================") + + new_entry = sirius_cyber_corp.LogEntry_1( + term=7, + entry=sirius_cyber_corp.Entry_1( + name=uavcan.primitive.String_1(value="top_3"), + value=10, + ), + ) + request = sirius_cyber_corp.AppendEntries_1.Request( + term=raft_node_1._term, + prev_log_index=2, # index of top_2 + prev_log_term=raft_node_1._log[2].term, + log_entry=new_entry, + ) + metadata = pycyphal.presentation.ServiceRequestMetadata( + client_node_id=42, + timestamp=time.time(), + priority=0, + transfer_id=0, + ) + response = await raft_node_2._serve_append_entries(request, metadata) + assert response.success == True + + await asyncio.sleep(TERM_TIMEOUT + 1) + + assert len(raft_node_2._log) == 1 + 3 + assert raft_node_2._log[0].term == 0 + assert raft_node_2._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty + assert raft_node_2._log[0].entry.value == 0 + assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_2._log[1].entry.value == 7 + assert raft_node_2._log[2].term == 5 + assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_2._log[2].entry.value == 8 + assert raft_node_2._log[3].term == 7 + assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_2._log[3].entry.value == 10 + assert raft_node_2._commit_index == 3 + + # check if the new entry is replicated in the follower nodes + assert len(raft_node_1._log) == 1 + 3 + assert raft_node_1._log[0].term == 0 + assert raft_node_1._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty + assert raft_node_1._log[0].entry.value == 0 + assert raft_node_1._log[1].term == 4 + assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_1._log[1].entry.value == 7 + assert raft_node_1._log[2].term == 5 + assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_1._log[2].entry.value == 8 + assert raft_node_1._log[3].term == 7 + assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_1._log[3].entry.value == 10 + assert raft_node_1._commit_index == 3 + + assert len(raft_node_3._log) == 1 + 3 + assert raft_node_3._log[0].term == 0 + assert raft_node_3._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty + assert raft_node_3._log[0].entry.value == 0 + assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_3._log[1].entry.value == 7 + assert raft_node_3._log[2].term == 5 + assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_3._log[2].entry.value == 8 + assert raft_node_3._log[3].term == 7 + assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_3._log[3].entry.value == 10 + assert raft_node_3._commit_index == 3 + raft_node_1.close() raft_node_2.close() diff --git a/tests/raft_node.py b/tests/raft_node.py index a185436..5cc2c0e 100644 --- a/tests/raft_node.py +++ b/tests/raft_node.py @@ -385,6 +385,8 @@ async def _unittest_raft_node_append_entries_rpc() -> None: assert raft_node._term == 6 assert raft_node._voted_for == 42 + + assert len(raft_node._log) == 1 + 3 assert raft_node._log[0].term == 0 assert raft_node._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty From 72a28f59bd9136bdeef304054066431f754b5c19 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Fri, 28 Jun 2024 15:55:41 +0300 Subject: [PATCH 03/48] Create cyraft-test.yml --- .github/workflows/cyraft-test.yml | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/cyraft-test.yml diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml new file mode 100644 index 0000000..83769bc --- /dev/null +++ b/.github/workflows/cyraft-test.yml @@ -0,0 +1,43 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python application + +on: + push: + branches: [ "test_CI" ] + pull_request: + branches: [ "test_CI" ] + +permissions: + contents: read + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Set up Environment Variables + run: | + export CYPHAL_PATH="$HOME/cyraft/demo/custom_data_types:$HOME/cyraft/demo/public_regulated_data_types" + source my_env.sh + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pytest tests/ From 2c0344665be0562835c84006e0e500c7d0048be7 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Fri, 28 Jun 2024 16:17:35 +0300 Subject: [PATCH 04/48] Update cyraft-test.yml Add line "echo "PYTHONPATH=$PWD/cyraft" >> $GITHUB_ENV" --- .github/workflows/cyraft-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 83769bc..6eccf1b 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -32,6 +32,7 @@ jobs: run: | export CYPHAL_PATH="$HOME/cyraft/demo/custom_data_types:$HOME/cyraft/demo/public_regulated_data_types" source my_env.sh + echo "PYTHONPATH=$PWD/cyraft" >> $GITHUB_ENV - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names From 7e9a1a61195a7d945992b18f9094bdb43a5dd36b Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Fri, 28 Jun 2024 16:26:07 +0300 Subject: [PATCH 05/48] Update cyraft-test.yml Add lines: 1. "echo "PYTHONPATH=$PYTHONPATH:$HOME/cyraft/demo/custom_data_types:$HOME/cyraft/demo/public_regulated_data_types" >> $GITHUB_ENV" 2. "- name: Verify PYTHONPATH run: echo $PYTHONPATH - name: List directory for debugging run: ls -R" --- .github/workflows/cyraft-test.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 6eccf1b..83147b2 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -23,22 +23,31 @@ jobs: uses: actions/setup-python@v3 with: python-version: "3.10" + - name: Install dependencies run: | python -m pip install --upgrade pip pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Set up Environment Variables run: | - export CYPHAL_PATH="$HOME/cyraft/demo/custom_data_types:$HOME/cyraft/demo/public_regulated_data_types" + echo "PYTHONPATH=$PYTHONPATH:$HOME/cyraft/demo/custom_data_types:$HOME/cyraft/demo/public_regulated_data_types" >> $GITHUB_ENV source my_env.sh - echo "PYTHONPATH=$PWD/cyraft" >> $GITHUB_ENV + + - name: Verify PYTHONPATH + run: echo $PYTHONPATH + + - name: List directory for debugging + run: ls -R + - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest run: | pytest tests/ From 905bf5ba9d6f2f143844c99a39631df6d333a8a2 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Fri, 28 Jun 2024 16:42:50 +0300 Subject: [PATCH 06/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 83147b2..3d90b38 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -19,6 +19,7 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: @@ -32,15 +33,15 @@ jobs: - name: Set up Environment Variables run: | - echo "PYTHONPATH=$PYTHONPATH:$HOME/cyraft/demo/custom_data_types:$HOME/cyraft/demo/public_regulated_data_types" >> $GITHUB_ENV + echo "PYTHONPATH=$PYTHONPATH:$HOME/cyraft/demo/custom_data_types/sirius_cyber_corp" >> $GITHUB_ENV source my_env.sh - name: Verify PYTHONPATH run: echo $PYTHONPATH - name: List directory for debugging - run: ls -R - + run: ls -R $HOME/cyraft/demo/custom_data_types/sirius_cyber_corp + - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names @@ -49,5 +50,8 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest + env: + PYTHONPATH: ${{ env.PYTHONPATH }} run: | + echo "PYTHONPATH is set to $PYTHONPATH" pytest tests/ From c528b48b968b51076b48e7bf89310dcf30393975 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Fri, 28 Jun 2024 16:59:59 +0300 Subject: [PATCH 07/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 3d90b38..6e7099c 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -33,14 +33,13 @@ jobs: - name: Set up Environment Variables run: | - echo "PYTHONPATH=$PYTHONPATH:$HOME/cyraft/demo/custom_data_types/sirius_cyber_corp" >> $GITHUB_ENV - source my_env.sh + echo "PYTHONPATH=$PYTHONPATH:$HOME/work/cyraft/cyraft/demo/custom_data_types/sirius_cyber_corp" >> $GITHUB_ENV - name: Verify PYTHONPATH run: echo $PYTHONPATH - name: List directory for debugging - run: ls -R $HOME/cyraft/demo/custom_data_types/sirius_cyber_corp + run: ls -R $HOME/work/cyraft/cyraft/demo/custom_data_types/sirius_cyber_corp - name: Lint with flake8 run: | From 1fc0d54ac91a0bce81a041828adefccdfb53bbee Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Fri, 28 Jun 2024 17:07:39 +0300 Subject: [PATCH 08/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 6e7099c..ab17c48 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -34,6 +34,7 @@ jobs: - name: Set up Environment Variables run: | echo "PYTHONPATH=$PYTHONPATH:$HOME/work/cyraft/cyraft/demo/custom_data_types/sirius_cyber_corp" >> $GITHUB_ENV + echo "Updated PYTHONPATH to $PYTHONPATH" - name: Verify PYTHONPATH run: echo $PYTHONPATH @@ -49,8 +50,6 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest - env: - PYTHONPATH: ${{ env.PYTHONPATH }} run: | echo "PYTHONPATH is set to $PYTHONPATH" pytest tests/ From 7c4ea56cd55d787a24b4c55c48f73e83f7b2c3e7 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Fri, 28 Jun 2024 17:19:24 +0300 Subject: [PATCH 09/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index ab17c48..b2f9e24 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -33,14 +33,14 @@ jobs: - name: Set up Environment Variables run: | - echo "PYTHONPATH=$PYTHONPATH:$HOME/work/cyraft/cyraft/demo/custom_data_types/sirius_cyber_corp" >> $GITHUB_ENV - echo "Updated PYTHONPATH to $PYTHONPATH" + export CYPHAL_PATH="$HOME/work/cyraft/cyraft/demo/custom_data_types:$HOME/work/cyraft/cyraft/demo/public_regulated_data_types" + source my_env.sh - name: Verify PYTHONPATH - run: echo $PYTHONPATH + run: echo $CYPHAL_PATH - name: List directory for debugging - run: ls -R $HOME/work/cyraft/cyraft/demo/custom_data_types/sirius_cyber_corp + run: ls -R $HOME/work/cyraft/cyraft - name: Lint with flake8 run: | @@ -50,6 +50,4 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest - run: | - echo "PYTHONPATH is set to $PYTHONPATH" - pytest tests/ + run: pytest tests/ From aaad4e8a6a0623bff63604dae338748a7be31f8b Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Fri, 28 Jun 2024 17:25:34 +0300 Subject: [PATCH 10/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index b2f9e24..eb48dd6 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -33,10 +33,10 @@ jobs: - name: Set up Environment Variables run: | - export CYPHAL_PATH="$HOME/work/cyraft/cyraft/demo/custom_data_types:$HOME/work/cyraft/cyraft/demo/public_regulated_data_types" + echo "CYPHAL_PATH=$HOME/work/cyraft/cyraft/demo/custom_data_types:$HOME/work/cyraft/cyraft/demo/public_regulated_data_types" >> $GITHUB_ENV source my_env.sh - - name: Verify PYTHONPATH + - name: Verify CYPHAL_PATH run: echo $CYPHAL_PATH - name: List directory for debugging From 54e9002e09cb46190656648d3d956f6816859203 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:45:55 +0300 Subject: [PATCH 11/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index eb48dd6..c4ad7af 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -31,11 +31,13 @@ jobs: pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Set up Environment Variables + - name: Set up Environment Variable CYPHAL_PATH run: | echo "CYPHAL_PATH=$HOME/work/cyraft/cyraft/demo/custom_data_types:$HOME/work/cyraft/cyraft/demo/public_regulated_data_types" >> $GITHUB_ENV - source my_env.sh + - name: Run File my_env.sh + run: source my_env.sh + - name: Verify CYPHAL_PATH run: echo $CYPHAL_PATH From 1b8e6ae8610e5196d3dd0eda8162450d26141098 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:02:37 +0300 Subject: [PATCH 12/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index c4ad7af..96c2947 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -33,7 +33,7 @@ jobs: - name: Set up Environment Variable CYPHAL_PATH run: | - echo "CYPHAL_PATH=$HOME/work/cyraft/cyraft/demo/custom_data_types:$HOME/work/cyraft/cyraft/demo/public_regulated_data_types" >> $GITHUB_ENV + echo "CYPHAL_PATH=$GITHUB_WORKSPACE/demo/custom_data_types:$GITHUB_WORKSPACE/demo/public_regulated_data_types" >> $GITHUB_ENV - name: Run File my_env.sh run: source my_env.sh @@ -41,6 +41,14 @@ jobs: - name: Verify CYPHAL_PATH run: echo $CYPHAL_PATH + - name: List directory for debugging + run: ls -R $GITHUB_WORKSPACE/demo + + - name: Check DSDL directories + run: | + ls $GITHUB_WORKSPACE/demo/custom_data_types/sirius_cyber_corp + ls $GITHUB_WORKSPACE/demo/public_regulated_data_types/uavcan/primitive + - name: List directory for debugging run: ls -R $HOME/work/cyraft/cyraft From 244a78ad4ffdead24d2dcbe8f3acba5bb76218ad Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:09:22 +0300 Subject: [PATCH 13/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 96c2947..bcf1898 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -19,7 +19,9 @@ jobs: steps: - uses: actions/checkout@v4 - + with: + submodules: true # Initialize submodules + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: From bf2545fb23c7acd3d021a4999179449bf5e397cd Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:15:46 +0300 Subject: [PATCH 14/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index bcf1898..03a8e9d 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -37,8 +37,13 @@ jobs: run: | echo "CYPHAL_PATH=$GITHUB_WORKSPACE/demo/custom_data_types:$GITHUB_WORKSPACE/demo/public_regulated_data_types" >> $GITHUB_ENV - - name: Run File my_env.sh - run: source my_env.sh + - name: Source my_env.sh + run: | + source $GITHUB_WORKSPACE/cyraft/my_env.sh + echo "UAVCAN__NODE__ID=$UAVCAN__NODE__ID" + echo "UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE" + echo "UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID" + echo "UAVCAN__DIAGNOSTIC__SEVERITY=$UAVCAN__DIAGNOSTIC__SEVERITY" - name: Verify CYPHAL_PATH run: echo $CYPHAL_PATH From 25ead7e0f4b2771aa6cd07b8cdb58387da80baf8 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:16:43 +0300 Subject: [PATCH 15/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 03a8e9d..c6e5ab7 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -39,7 +39,7 @@ jobs: - name: Source my_env.sh run: | - source $GITHUB_WORKSPACE/cyraft/my_env.sh + source $GITHUB_WORKSPACE/my_env.sh echo "UAVCAN__NODE__ID=$UAVCAN__NODE__ID" echo "UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE" echo "UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID" From 2b7ebf982f7929ebd77e5756800040945ba56daa Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:26:54 +0300 Subject: [PATCH 16/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index c6e5ab7..709d41d 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -38,8 +38,10 @@ jobs: echo "CYPHAL_PATH=$GITHUB_WORKSPACE/demo/custom_data_types:$GITHUB_WORKSPACE/demo/public_regulated_data_types" >> $GITHUB_ENV - name: Source my_env.sh + run: source $GITHUB_WORKSPACE/my_env.sh + + - name: Debug environment variables run: | - source $GITHUB_WORKSPACE/my_env.sh echo "UAVCAN__NODE__ID=$UAVCAN__NODE__ID" echo "UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE" echo "UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID" From 6782c37a46bc0f4d696430ca2cfd123fc74351ab Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:29:59 +0300 Subject: [PATCH 17/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 709d41d..36ba8af 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -38,7 +38,8 @@ jobs: echo "CYPHAL_PATH=$GITHUB_WORKSPACE/demo/custom_data_types:$GITHUB_WORKSPACE/demo/public_regulated_data_types" >> $GITHUB_ENV - name: Source my_env.sh - run: source $GITHUB_WORKSPACE/my_env.sh + run: | + source $GITHUB_WORKSPACE/my_env.sh - name: Debug environment variables run: | From 24996bafc45dcac2fde823e3bc918fe16ad4383f Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:33:09 +0300 Subject: [PATCH 18/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 36ba8af..5928a2b 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -36,10 +36,17 @@ jobs: - name: Set up Environment Variable CYPHAL_PATH run: | echo "CYPHAL_PATH=$GITHUB_WORKSPACE/demo/custom_data_types:$GITHUB_WORKSPACE/demo/public_regulated_data_types" >> $GITHUB_ENV - + + - name: Verify CYPHAL_PATH + run: echo $CYPHAL_PATH + - name: Source my_env.sh run: | source $GITHUB_WORKSPACE/my_env.sh + echo "UAVCAN__NODE__ID=$UAVCAN__NODE__ID" + echo "UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE" + echo "UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID" + echo "UAVCAN__DIAGNOSTIC__SEVERITY=$UAVCAN__DIAGNOSTIC__SEVERITY" - name: Debug environment variables run: | @@ -47,10 +54,7 @@ jobs: echo "UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE" echo "UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID" echo "UAVCAN__DIAGNOSTIC__SEVERITY=$UAVCAN__DIAGNOSTIC__SEVERITY" - - - name: Verify CYPHAL_PATH - run: echo $CYPHAL_PATH - + - name: List directory for debugging run: ls -R $GITHUB_WORKSPACE/demo From 7c048056b6831e56bbc3c0bc6eff4985471567fb Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:42:58 +0300 Subject: [PATCH 19/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 5928a2b..5c3de54 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -40,14 +40,14 @@ jobs: - name: Verify CYPHAL_PATH run: echo $CYPHAL_PATH - - name: Source my_env.sh + - name: Source and export my_env.sh run: | source $GITHUB_WORKSPACE/my_env.sh - echo "UAVCAN__NODE__ID=$UAVCAN__NODE__ID" - echo "UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE" - echo "UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID" - echo "UAVCAN__DIAGNOSTIC__SEVERITY=$UAVCAN__DIAGNOSTIC__SEVERITY" - + export UAVCAN__NODE__ID=$UAVCAN__NODE__ID + export UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE + export UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID + export UAVCAN__DIAGNOSTIC__SEVERITY=$UAVCAN__DIAGNOSTIC__SEVERITY + - name: Debug environment variables run: | echo "UAVCAN__NODE__ID=$UAVCAN__NODE__ID" @@ -74,4 +74,9 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest + source $GITHUB_WORKSPACE/my_env.sh + echo "UAVCAN__NODE__ID=$UAVCAN__NODE__ID" + echo "UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE" + echo "UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID" + echo "UAVCAN__DIAGNOSTIC__SEVERITY=$UAVCAN__DIAGNOSTIC__SEVERITY" run: pytest tests/ From cc3124fd5774d5f31929e4ab0e434c0060ec6176 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:50:22 +0300 Subject: [PATCH 20/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 5c3de54..ea16613 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -43,10 +43,10 @@ jobs: - name: Source and export my_env.sh run: | source $GITHUB_WORKSPACE/my_env.sh - export UAVCAN__NODE__ID=$UAVCAN__NODE__ID - export UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE - export UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID - export UAVCAN__DIAGNOSTIC__SEVERITY=$UAVCAN__DIAGNOSTIC__SEVERITY + echo "UAVCAN__NODE__ID=$UAVCAN__NODE__ID" >> $GITHUB_ENV + echo "UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE" >> $GITHUB_ENV + echo "UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID" >> $GITHUB_ENV + echo "UAVCAN__DIAGNOSTIC__SEVERITY=$UAVCAN__DIAGNOSTIC__SEVERITY" >> $GITHUB_ENV - name: Debug environment variables run: | From 97185c4d017d16873d94511df3aed4576a3bf170 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:01:10 +0300 Subject: [PATCH 21/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index ea16613..7b48176 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python -name: Python application +name: Cyraft application on: push: From ec89abdc3f5eca000bf912b5554bc673ab485beb Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:04:14 +0300 Subject: [PATCH 22/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 7b48176..b61cc06 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -5,9 +5,11 @@ name: Cyraft application on: push: - branches: [ "test_CI" ] + branches: + - ${{ github.ref }} pull_request: - branches: [ "test_CI" ] + branches: + - ${{ github.ref }} permissions: contents: read From dbf9b758e6533926bd581bd764dd7fb62a5ca97c Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:08:27 +0300 Subject: [PATCH 23/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index b61cc06..214319c 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -6,10 +6,10 @@ name: Cyraft application on: push: branches: - - ${{ github.ref }} + - ${{ github.event.push.branch }} pull_request: branches: - - ${{ github.ref }} + - ${{ github.event.pull_request.branch.ref }} permissions: contents: read From 158899357e1d9fd2fb3874405268c42c12f7cee6 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:09:40 +0300 Subject: [PATCH 24/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 214319c..78dd00c 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -10,6 +10,12 @@ on: pull_request: branches: - ${{ github.event.pull_request.branch.ref }} + workflow_dispatch: + inputs: + branch: + description: 'Branch to build' + required: true + default: ${{ github.event.push.branch || github.event.pull_request.branch.ref }} permissions: contents: read From eec4337bddc70aa6ca2de37bc89a8035c8e1767d Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:11:51 +0300 Subject: [PATCH 25/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 78dd00c..bef4483 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -6,16 +6,16 @@ name: Cyraft application on: push: branches: - - ${{ github.event.push.branch }} + - ${{ github.event.push.branchName }} pull_request: branches: - - ${{ github.event.pull_request.branch.ref }} + - ${{ github.event.pull_request.head.refName }} workflow_dispatch: inputs: branch: description: 'Branch to build' required: true - default: ${{ github.event.push.branch || github.event.pull_request.branch.ref }} + default: ${{ github.event.push.branchName || github.event.pull_request.head.refName }} permissions: contents: read From 5505bd33066902c56ae68aaa10c111cc96f95b19 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:13:49 +0300 Subject: [PATCH 26/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index bef4483..d35a6d8 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -6,16 +6,16 @@ name: Cyraft application on: push: branches: - - ${{ github.event.push.branchName }} + - '*' pull_request: branches: - - ${{ github.event.pull_request.head.refName }} + - '*' workflow_dispatch: inputs: branch: description: 'Branch to build' required: true - default: ${{ github.event.push.branchName || github.event.pull_request.head.refName }} + default: ${{ github.ref_name }} permissions: contents: read From f368816698794da56608a9ac25e46080adadc7e0 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:14:53 +0300 Subject: [PATCH 27/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index d35a6d8..2168a11 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -10,12 +10,6 @@ on: pull_request: branches: - '*' - workflow_dispatch: - inputs: - branch: - description: 'Branch to build' - required: true - default: ${{ github.ref_name }} permissions: contents: read From 8b8df67f98e253d58b1a9f5a24f0999dd1f44f69 Mon Sep 17 00:00:00 2001 From: Nikita Date: Mon, 1 Jul 2024 09:23:33 +0000 Subject: [PATCH 28/48] Fail tests for testing CI --- cyraft/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cyraft/node.py b/cyraft/node.py index 5641de1..e2b705c 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -341,7 +341,7 @@ async def _serve_append_entries( metadata.client_node_id, ) self._voted_for = metadata.client_node_id - self._change_state(RaftState.FOLLOWER) # this will reset the election timeout as well + #self._change_state(RaftState.FOLLOWER) # this will reset the election timeout as well self._term = request.term # update term return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=True) From 688a9bfb27ddb683695efa1f468c92ab257bcc2d Mon Sep 17 00:00:00 2001 From: Nikita Date: Mon, 1 Jul 2024 09:38:49 +0000 Subject: [PATCH 29/48] All tests pass for testing CI --- cyraft/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cyraft/node.py b/cyraft/node.py index e2b705c..5641de1 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -341,7 +341,7 @@ async def _serve_append_entries( metadata.client_node_id, ) self._voted_for = metadata.client_node_id - #self._change_state(RaftState.FOLLOWER) # this will reset the election timeout as well + self._change_state(RaftState.FOLLOWER) # this will reset the election timeout as well self._term = request.term # update term return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=True) From 71c27061f3ad7dd0612478075d4314e683db3d02 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:44:27 +0300 Subject: [PATCH 30/48] Update cyraft-test.yml --- .github/workflows/cyraft-test.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 2168a11..48fb7bb 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -65,9 +65,6 @@ jobs: ls $GITHUB_WORKSPACE/demo/custom_data_types/sirius_cyber_corp ls $GITHUB_WORKSPACE/demo/public_regulated_data_types/uavcan/primitive - - name: List directory for debugging - run: ls -R $HOME/work/cyraft/cyraft - - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names @@ -76,9 +73,4 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest - source $GITHUB_WORKSPACE/my_env.sh - echo "UAVCAN__NODE__ID=$UAVCAN__NODE__ID" - echo "UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE" - echo "UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID" - echo "UAVCAN__DIAGNOSTIC__SEVERITY=$UAVCAN__DIAGNOSTIC__SEVERITY" run: pytest tests/ From cf18a3721b15cf33a6f87c91a6d5bd8880d117a2 Mon Sep 17 00:00:00 2001 From: Nikita Date: Wed, 26 Jun 2024 11:21:27 +0000 Subject: [PATCH 31/48] =?UTF-8?q?In=20node.py:=20expanded=20the=20log=20in?= =?UTF-8?q?formation=20for=20some=20values=20=E2=80=8B=E2=80=8Bto=20make?= =?UTF-8?q?=20it=20easier=20to=20analyze=20the=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In raft_log_replication.py: start to write test that log replication happens correctly if leadership changes In raft_node.py: add a couple of lines of code to close node after each test, so that after running pytest it does not crash new issue: after the leader changes, the new leader is not aware of the latest index of the neighboring nodes and begins to rewrite them, even though his nodes completely coincide with the neighboring node. --- cyraft/node.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cyraft/node.py b/cyraft/node.py index 5641de1..bacee9f 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -631,6 +631,17 @@ def _change_state(self, new_state: RaftState) -> None: if self._state == RaftState.FOLLOWER: # Cancel the term timeout (if it exists), and schedule a new election timeout. + for remote_node_index, remote_next_index in enumerate(self._next_index): + _logger.info( + c["raft_logic"] + + "Node ID: %d -- Value of next index = %d and value of remote next index = %s for node %s" + + c["end_color"], + self._node.id, + self._commit_index, + remote_next_index, + self._cluster[remote_node_index], + ) + if hasattr(self, "_term_timer"): self._term_timer.cancel() # FOLLOWER should not have term timer self._reset_election_timeout() # FOLLOWER should have election timer From 1956d83c7bbbdfa42fcc7864c10bcb4a87977a4a Mon Sep 17 00:00:00 2001 From: Nikita Date: Fri, 28 Jun 2024 09:07:52 +0000 Subject: [PATCH 32/48] Added test _unittest_raft_leader_changes for log_replication.py --- cyraft/node.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cyraft/node.py b/cyraft/node.py index bacee9f..5641de1 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -631,17 +631,6 @@ def _change_state(self, new_state: RaftState) -> None: if self._state == RaftState.FOLLOWER: # Cancel the term timeout (if it exists), and schedule a new election timeout. - for remote_node_index, remote_next_index in enumerate(self._next_index): - _logger.info( - c["raft_logic"] - + "Node ID: %d -- Value of next index = %d and value of remote next index = %s for node %s" - + c["end_color"], - self._node.id, - self._commit_index, - remote_next_index, - self._cluster[remote_node_index], - ) - if hasattr(self, "_term_timer"): self._term_timer.cancel() # FOLLOWER should not have term timer self._reset_election_timeout() # FOLLOWER should have election timer From 17774d1a6e8bb65b54361eca4fd6cabc2e221260 Mon Sep 17 00:00:00 2001 From: Nikita Date: Mon, 1 Jul 2024 07:32:18 +0000 Subject: [PATCH 33/48] Fix small issues with tests --- cyraft/node.py | 4 +- tests/raft_log_replication.py | 7 ++- tests/raft_node.py | 115 ++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 5 deletions(-) diff --git a/cyraft/node.py b/cyraft/node.py index 5641de1..9787806 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -88,8 +88,6 @@ def __init__(self) -> None: ## Volatile state on leaders self._next_index: typing.List[int] = [] # index of the next log entry to send to that server - - # self._match_index: typing.List[int] = [] # index of highest log entry known to be replicated on server ######################################## @@ -454,7 +452,7 @@ def _append_entries_processing( # Update current_term (if follower) (leaders will update their own term on timeout) if self._state == RaftState.FOLLOWER: - self._change_state(RaftState.FOLLOWER) # this will reset the election timeout as well + self._reset_election_timeout() self._term = request.log_entry[0].term diff --git a/tests/raft_log_replication.py b/tests/raft_log_replication.py index 7948eb0..8cdf262 100644 --- a/tests/raft_log_replication.py +++ b/tests/raft_log_replication.py @@ -790,8 +790,11 @@ async def _unittest_raft_leader_changes() -> None: raft_node_1._change_state(RaftState.FOLLOWER) raft_node_2._change_state(RaftState.FOLLOWER) raft_node_3._change_state(RaftState.FOLLOWER) - raft_node_1._voted_for = None + # Need to reset the votes of each node so that node 41 doesn't win again. + raft_node_1._voted_for = None + raft_node_2._voted_for = None + raft_node_3._voted_for = None await asyncio.sleep(2*ELECTION_TIMEOUT + 1) @@ -932,7 +935,7 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_3._log[4].entry.value == 13 assert raft_node_3._commit_index == 4 - _logger.info("================== TEST 3: Replace log entries 1 and 4 with new entries from another LEADER ==================") + _logger.info("================== TEST 3: Replace log entry 3 with new entries from another LEADER ==================") new_entry = sirius_cyber_corp.LogEntry_1( term=7, diff --git a/tests/raft_node.py b/tests/raft_node.py index 5cc2c0e..5adf578 100644 --- a/tests/raft_node.py +++ b/tests/raft_node.py @@ -156,7 +156,122 @@ async def _unittest_raft_node_election_timeout() -> None: raft_node.close() await asyncio.sleep(1) # give some time for the node to close +async def _unittest_raft_node_heartbeat() -> None: + """ + Test that the node does NOT convert to candidate if it receives a heartbeat message + """ + os.environ["UAVCAN__NODE__ID"] = "41" + raft_node = RaftNode() + ELECTION_TIMEOUT = 5 + TERM_TIMEOUT = 1 + raft_node.election_timeout = ELECTION_TIMEOUT + raft_node.term_timeout = TERM_TIMEOUT + raft_node._voted_for = 42 + + asyncio.create_task(raft_node.run()) + await asyncio.sleep(ELECTION_TIMEOUT * 0.90) # sleep until right before election timeout + + # send heartbeat + terms_passed = raft_node._term # leader's term is equal to the follower's term + await raft_node._serve_append_entries( + sirius_cyber_corp.AppendEntries_1.Request( + term=terms_passed, # leader's term + prev_log_index=0, # index of log entry immediately preceding new ones + prev_log_term=0, # term of prevLogIndex entry + leader_commit=0, # leader's commitIndex + log_entry=None, # log entries to store (empty for heartbeat) + ), + pycyphal.presentation.ServiceRequestMetadata( + client_node_id=42, # leader's node id + timestamp=time.time(), # leader's timestamp + priority=0, # leader's priority + transfer_id=0, # leader's transfer id + ), + ) + + # wait for heartbeat to be processed [election is reached but shouldn't become leader due to heartbeat] + await asyncio.sleep(ELECTION_TIMEOUT * 0.1 + 0.1) + assert raft_node._state == RaftState.FOLLOWER + assert raft_node._voted_for == 42 + + # send heartbeat again + # (this time leader has a higher term, we want to make sure that the follower's term is updated) + await asyncio.sleep(ELECTION_TIMEOUT * 0.90) # sleep until right before election timeout + terms_passed = raft_node._term + 5 # leader's term is higher than the follower's term + await raft_node._serve_append_entries( + sirius_cyber_corp.AppendEntries_1.Request( + term=terms_passed, # leader's term + prev_log_index=0, # index of log entry immediately preceding new ones + prev_log_term=0, # term of prevLogIndex entry + leader_commit=0, # leader's commitIndex + log_entry=None, # log entries to store (empty for heartbeat) + ), + pycyphal.presentation.ServiceRequestMetadata( + client_node_id=42, # leader's node id + timestamp=time.time(), # leader's timestamp + priority=0, # leader's priority + transfer_id=0, # leader's transfer id + ), + ) + + # wait for heartbeat to be processed [election is reached but shouldn't become leader due to heartbeat] + await asyncio.sleep(ELECTION_TIMEOUT * 0.1 + 0.1) + assert raft_node._state == RaftState.FOLLOWER + assert raft_node._voted_for == 42 + assert raft_node._term == terms_passed + + # send heartbeat again + # (this time from a different leader with a higher term, we want to make sure the follower switches leader and updates term) + await asyncio.sleep(ELECTION_TIMEOUT * 0.90) # sleep until right before election timeout + terms_passed = raft_node._term + 5 # leader's term is higher than the follower's term + await raft_node._serve_append_entries( + sirius_cyber_corp.AppendEntries_1.Request( + term=terms_passed, # leader's term + prev_log_index=0, # index of log entry immediately preceding new ones + prev_log_term=0, # term of prevLogIndex entry + leader_commit=0, # leader's commitIndex + log_entry=None, # log entries to store (empty for heartbeat) + ), + pycyphal.presentation.ServiceRequestMetadata( + client_node_id=43, # leader's node id (different from previous leader) + timestamp=time.time(), # leader's timestamp + priority=0, # leader's priority + transfer_id=0, # leader's transfer id + ), + ) + # wait for heartbeat to be processed [election is reached but shouldn't become leader due to heartbeat] + await asyncio.sleep(ELECTION_TIMEOUT * 0.1 + 0.1) + assert raft_node._state == RaftState.FOLLOWER + assert raft_node._voted_for == 43 + assert raft_node._term == terms_passed + + # send heartbeat again + # (this time the leader's term is lower than the follower's term, we want to make sure the follower doesn't switch leader) + await asyncio.sleep(ELECTION_TIMEOUT * 0.90) # sleep until right before election timeout + terms_passed = raft_node._term - 1 # leader's term is lower than the follower's term + await raft_node._serve_append_entries( + sirius_cyber_corp.AppendEntries_1.Request( + term=terms_passed, # leader's term + prev_log_index=0, # index of log entry immediately preceding new ones + prev_log_term=0, # term of prevLogIndex entry + leader_commit=0, # leader's commitIndex + log_entry=None, # log entries to store (empty for heartbeat) + ), + pycyphal.presentation.ServiceRequestMetadata( + client_node_id=42, # old leader's node id, which has lower term + timestamp=time.time(), # leader's timestamp + priority=0, # leader's priority + transfer_id=0, # leader's transfer id + ), + ) + + ## test that the node converts to candidate after the election timeout [no valid heartbeat is received] + await asyncio.sleep(ELECTION_TIMEOUT * 0.1 + 0.1) + assert raft_node._prev_state == RaftState.CANDIDATE + + raft_node.close() + await asyncio.sleep(1) # fixes when just running this test, however not when "pytest /cyraft" is run async def _unittest_raft_node_request_vote_rpc() -> None: """ From ef7a57b07f5ab87ae30205c384cf4d72df1b6ce3 Mon Sep 17 00:00:00 2001 From: Nikita Date: Mon, 1 Jul 2024 09:57:00 +0000 Subject: [PATCH 34/48] Implement CI to the code --- .github/workflows/cyraft-test.yml | 44 ++++++++++++++++++++++++------- README.md | 2 +- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 48fb7bb..f277f17 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -1,76 +1,100 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +# Name of the workflow, which will be displayed in the GitHub Actions dashboard name: Cyraft application +# Trigger the workflow on push and pull request events for all branches on: push: branches: - - '*' + - '*' # This means the workflow will run on every push event, regardless of the branch pull_request: branches: - - '*' + - '*' # This means the workflow will run on every pull request event, regardless of the branch +# Set permissions to read contents, which allows the workflow to access the repository's code permissions: contents: read +# Define a job called "build", which will execute the steps defined below jobs: build: + # Run the job on an ubuntu-latest environment, which provides a fresh Ubuntu installation runs-on: ubuntu-latest + # Define the steps for the job, which will be executed in sequence steps: - uses: actions/checkout@v4 with: - submodules: true # Initialize submodules + submodules: true # Initialize submodules to ensure all code is available, including submodules in the repository + # Set up Python 3.10 environment, which will be used to run the tests and linting - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.10" # Specify the version of Python to use + # Install dependencies required for the project, including flake8 and pytest - name: Install dependencies run: | + # Upgrade pip to the latest version, to ensure we have the latest package manager python -m pip install --upgrade pip - pip install flake8 pytest + # Install flake8, which is a Python linter that checks for syntax errors and coding standards + pip install flake8 + # Install pytest, which is a testing framework for Python + pip install pytest + # If a requirements.txt file exists in the repository, install the dependencies specified in it if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + # Set up environment variable CYPHAL_PATH, which is used by the Cyraft application - name: Set up Environment Variable CYPHAL_PATH run: | + # Set CYPHAL_PATH to include custom_data_types and public_regulated_data_types directories echo "CYPHAL_PATH=$GITHUB_WORKSPACE/demo/custom_data_types:$GITHUB_WORKSPACE/demo/public_regulated_data_types" >> $GITHUB_ENV + # Verify CYPHAL_PATH is set correctly, by printing its value - name: Verify CYPHAL_PATH run: echo $CYPHAL_PATH + # Source and export environment variables from my_env.sh, which sets UAVCAN environment variables - name: Source and export my_env.sh run: | + # Source my_env.sh to set environment variables, such as UAVCAN__NODE__ID, UAVCAN__UDP__IFACE, etc. source $GITHUB_WORKSPACE/my_env.sh + # Export UAVCAN environment variables, so they can be used in subsequent steps echo "UAVCAN__NODE__ID=$UAVCAN__NODE__ID" >> $GITHUB_ENV echo "UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE" >> $GITHUB_ENV echo "UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID" >> $GITHUB_ENV echo "UAVCAN__DIAGNOSTIC__SEVERITY=$UAVCAN__DIAGNOSTIC__SEVERITY" >> $GITHUB_ENV + # Debug environment variables, by printing their values - name: Debug environment variables run: | + # Print UAVCAN environment variables for debugging purposes echo "UAVCAN__NODE__ID=$UAVCAN__NODE__ID" echo "UAVCAN__UDP__IFACE=$UAVCAN__UDP__IFACE" echo "UAVCAN__SRV__REQUEST_VOTE__ID=$UAVCAN__SRV__REQUEST_VOTE__ID" echo "UAVCAN__DIAGNOSTIC__SEVERITY=$UAVCAN__DIAGNOSTIC__SEVERITY" + # List directory for debugging, to verify the file structure - name: List directory for debugging run: ls -R $GITHUB_WORKSPACE/demo + # Check DSDL directories, to verify the existence of custom_data_types and public_regulated_data_types directories - name: Check DSDL directories run: | ls $GITHUB_WORKSPACE/demo/custom_data_types/sirius_cyber_corp ls $GITHUB_WORKSPACE/demo/public_regulated_data_types/uavcan/primitive + # Lint with flake8, to check for Python syntax errors and coding standards - name: Lint with flake8 run: | - # stop the build if there are Python syntax errors or undefined names + # Run flake8 to check for Python syntax errors and undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + # Run flake8 again with exit-zero to treat all errors as warnings flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + # Test with pytest, to run unit tests and verify the application's functionality - name: Test with pytest - run: pytest tests/ + run: pytest tests/ \ No newline at end of file diff --git a/README.md b/README.md index e81b27a..aad513c 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ This feature is significant because it enables Cyphal to serve as a communicatio - [ ] [Add name resolution service](https://github.com/OpenCyphal-Garage/cyraft/issues/3) - [ ] [Monitor the network for online nodes](https://github.com/OpenCyphal-Garage/cyraft/issues/4) - [ ] `.env-variables` and `my_env.sh` should be combined? - - [ ] Implement Github CI + - [x] Implement Github CI - [x] Refactor code into `cyraft` Questions: From 10bb90db483fb79d02ba4cc8dbbd9103afc44e3e Mon Sep 17 00:00:00 2001 From: Nikita Date: Wed, 3 Jul 2024 14:15:52 +0000 Subject: [PATCH 35/48] implemented the Black Code Style --- .github/workflows/cyraft-test.yml | 14 ++++---------- cyraft/node.py | 22 +++++++++++----------- demo/demo_cyraft.py | 2 +- pyproject.toml | 8 ++++++++ requirements.txt | 2 ++ tests/raft_leader_election.py | 2 +- tests/raft_log_replication.py | 23 +++++++++++++---------- tests/raft_node.py | 6 +++--- 8 files changed, 43 insertions(+), 36 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index f277f17..bb5c7d8 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -40,10 +40,6 @@ jobs: run: | # Upgrade pip to the latest version, to ensure we have the latest package manager python -m pip install --upgrade pip - # Install flake8, which is a Python linter that checks for syntax errors and coding standards - pip install flake8 - # Install pytest, which is a testing framework for Python - pip install pytest # If a requirements.txt file exists in the repository, install the dependencies specified in it if [ -f requirements.txt ]; then pip install -r requirements.txt; fi @@ -87,13 +83,11 @@ jobs: ls $GITHUB_WORKSPACE/demo/custom_data_types/sirius_cyber_corp ls $GITHUB_WORKSPACE/demo/public_regulated_data_types/uavcan/primitive - # Lint with flake8, to check for Python syntax errors and coding standards - - name: Lint with flake8 + # Lint with bleck, to check for Python syntax errors and coding standards + - name: Lint with black run: | - # Run flake8 to check for Python syntax errors and undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # Run flake8 again with exit-zero to treat all errors as warnings - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + # Run black to format the code + black . --check # Test with pytest, to run unit tests and verify the application's functionality - name: Test with pytest diff --git a/cyraft/node.py b/cyraft/node.py index 9787806..1d7c577 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -352,7 +352,7 @@ async def _serve_append_entries( + c["end_color"], self._node.id, request.term, - self._term + self._term, ) return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) @@ -369,14 +369,15 @@ async def _serve_append_entries( return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) except IndexError as e: _logger.info( - c["append_entries"] - + "Node ID: %d -- Append entries request denied (log mismatch 2). IndexError: %s. " - + "prev_log_index: %d, log_length: %d" - + c["end_color"], - self._node.id, - str(e), - request.prev_log_index, - len(self._log)) + c["append_entries"] + + "Node ID: %d -- Append entries request denied (log mismatch 2). IndexError: %s. " + + "prev_log_index: %d, log_length: %d" + + c["end_color"], + self._node.id, + str(e), + request.prev_log_index, + len(self._log), + ) return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) self._append_entries_processing(request) @@ -454,7 +455,6 @@ def _append_entries_processing( if self._state == RaftState.FOLLOWER: self._reset_election_timeout() self._term = request.log_entry[0].term - async def _send_heartbeat(self, remote_node_index: int) -> None: """ @@ -641,7 +641,7 @@ def _change_state(self, new_state: RaftState) -> None: self._election_timer.cancel() elif self._state == RaftState.LEADER: assert self._prev_state == RaftState.CANDIDATE, "Invalid state change 3" - + # Cancel the election timeout (if it exists), and schedule a new term timeout. if hasattr(self, "_election_timer"): self._election_timer.cancel() diff --git a/demo/demo_cyraft.py b/demo/demo_cyraft.py index 97eacea..9fc9af2 100644 --- a/demo/demo_cyraft.py +++ b/demo/demo_cyraft.py @@ -10,7 +10,7 @@ import sys # Get the absolute path of the parent directory -parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # Add the parent directory to the Python path sys.path.append(parent_dir) # This can be removed if setting PYTHONPATH (export PYTHONPATH=cyraft) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7166fc8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[tool.black] +line-length = 120 +target-version = ['py310'] +include = ''' +((cyraft|tests)/.*\.pyi?$) +| +(demo/[a-z0-9_]+\.py$) +''' \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2ac3377..20ce293 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,3 +32,5 @@ typing_extensions==4.12.2 urllib3==2.2.2 wrapt==1.16.0 yakut==0.13.0 +pylint~=2.6 +black==23.1.0 diff --git a/tests/raft_leader_election.py b/tests/raft_leader_election.py index 3cfd6b8..c9b9244 100644 --- a/tests/raft_leader_election.py +++ b/tests/raft_leader_election.py @@ -25,7 +25,7 @@ # Add parent directory to Python path # Get the absolute path of the parent directory -parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # Add the parent directory to the Python path sys.path.append(parent_dir) # This can be removed if setting PYTHONPATH (export PYTHONPATH=cyraft) diff --git a/tests/raft_log_replication.py b/tests/raft_log_replication.py index 8cdf262..b0d9a02 100644 --- a/tests/raft_log_replication.py +++ b/tests/raft_log_replication.py @@ -12,7 +12,7 @@ # Add parent directory to Python path # Get the absolute path of the parent directory -parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # Add the parent directory to the Python path sys.path.append(parent_dir) # This can be removed if setting PYTHONPATH (export PYTHONPATH=cyraft) @@ -598,6 +598,7 @@ async def _unittest_raft_log_replication() -> None: raft_node_3.close() await asyncio.sleep(1) + # ======================================================================================= # ========== Test that log replication happens correctly if leadership changes ========== # ======================================================================================= @@ -641,6 +642,7 @@ async def _unittest_raft_log_replication() -> None: """ + async def _unittest_raft_leader_changes() -> None: logging.root.setLevel(logging.INFO) @@ -777,7 +779,7 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_3._log[3].entry.value == 9 assert raft_node_3._commit_index == 3 - + await asyncio.sleep(TERM_TIMEOUT + 1) _logger.info("================== TEST 1: Change LEADER and check logs ==================") @@ -792,11 +794,11 @@ async def _unittest_raft_leader_changes() -> None: raft_node_3._change_state(RaftState.FOLLOWER) # Need to reset the votes of each node so that node 41 doesn't win again. - raft_node_1._voted_for = None - raft_node_2._voted_for = None - raft_node_3._voted_for = None + raft_node_1._voted_for = None + raft_node_2._voted_for = None + raft_node_3._voted_for = None - await asyncio.sleep(2*ELECTION_TIMEOUT + 1) + await asyncio.sleep(2 * ELECTION_TIMEOUT + 1) assert raft_node_1._state == RaftState.FOLLOWER assert raft_node_1._term == 20, "received heartbeat from LEADER" @@ -917,7 +919,7 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_1._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" assert raft_node_1._log[4].entry.value == 13 assert raft_node_1._commit_index == 4 - + assert len(raft_node_3._log) == 1 + 4 assert raft_node_3._log[0].term == 0 assert raft_node_3._log[0].entry.value == 0 @@ -935,7 +937,9 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_3._log[4].entry.value == 13 assert raft_node_3._commit_index == 4 - _logger.info("================== TEST 3: Replace log entry 3 with new entries from another LEADER ==================") + _logger.info( + "================== TEST 3: Replace log entry 3 with new entries from another LEADER ==================" + ) new_entry = sirius_cyber_corp.LogEntry_1( term=7, @@ -1007,8 +1011,7 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_3._log[3].entry.value == 10 assert raft_node_3._commit_index == 3 - raft_node_1.close() raft_node_2.close() raft_node_3.close() - await asyncio.sleep(1) \ No newline at end of file + await asyncio.sleep(1) diff --git a/tests/raft_node.py b/tests/raft_node.py index 5adf578..2bb58a2 100644 --- a/tests/raft_node.py +++ b/tests/raft_node.py @@ -12,7 +12,7 @@ # Add parent directory to Python path # Get the absolute path of the parent directory -parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # Add the parent directory to the Python path sys.path.append(parent_dir) # This can be removed if setting PYTHONPATH (export PYTHONPATH=cyraft) @@ -156,6 +156,7 @@ async def _unittest_raft_node_election_timeout() -> None: raft_node.close() await asyncio.sleep(1) # give some time for the node to close + async def _unittest_raft_node_heartbeat() -> None: """ Test that the node does NOT convert to candidate if it receives a heartbeat message @@ -273,6 +274,7 @@ async def _unittest_raft_node_heartbeat() -> None: raft_node.close() await asyncio.sleep(1) # fixes when just running this test, however not when "pytest /cyraft" is run + async def _unittest_raft_node_request_vote_rpc() -> None: """ Test the _serve_request_vote() method @@ -500,8 +502,6 @@ async def _unittest_raft_node_append_entries_rpc() -> None: assert raft_node._term == 6 assert raft_node._voted_for == 42 - - assert len(raft_node._log) == 1 + 3 assert raft_node._log[0].term == 0 assert raft_node._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty From c7ac6720d0fd37c46997b646bb6a2754534b7875 Mon Sep 17 00:00:00 2001 From: Nikita Date: Wed, 3 Jul 2024 14:31:28 +0000 Subject: [PATCH 36/48] Added command "black . -v" --- .github/workflows/cyraft-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index bb5c7d8..e4a5e30 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -87,6 +87,7 @@ jobs: - name: Lint with black run: | # Run black to format the code + black . -v black . --check # Test with pytest, to run unit tests and verify the application's functionality From 3da386438681c5f527bb0907f0070c9c373a85eb Mon Sep 17 00:00:00 2001 From: Nikita Date: Wed, 3 Jul 2024 14:39:03 +0000 Subject: [PATCH 37/48] Updated cyraft-test.yaml --- .github/workflows/cyraft-test.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index e4a5e30..4dc19f8 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -3,14 +3,12 @@ # Name of the workflow, which will be displayed in the GitHub Actions dashboard name: Cyraft application -# Trigger the workflow on push and pull request events for all branches -on: - push: - branches: - - '*' # This means the workflow will run on every push event, regardless of the branch - pull_request: - branches: - - '*' # This means the workflow will run on every pull request event, regardless of the branch +on: [ push, pull_request ] + +# Ensures that only one workflow is running at a time +concurrency: + group: ${{ github.workflow_sha }} + cancel-in-progress: true # Set permissions to read contents, which allows the workflow to access the repository's code permissions: From d4cf997df5d721e08ffd913cf5a078763fb9c654 Mon Sep 17 00:00:00 2001 From: Nikita Date: Wed, 3 Jul 2024 14:42:46 +0000 Subject: [PATCH 38/48] Deleted "black . -v" --- .github/workflows/cyraft-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 4dc19f8..b046927 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -85,7 +85,6 @@ jobs: - name: Lint with black run: | # Run black to format the code - black . -v black . --check # Test with pytest, to run unit tests and verify the application's functionality From a30d97a71fd306631202407ed48358343a932e8d Mon Sep 17 00:00:00 2001 From: Nikita Date: Wed, 3 Jul 2024 14:53:06 +0000 Subject: [PATCH 39/48] Rewrote line: "black . --check -v" --- .github/workflows/cyraft-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index b046927..9e48358 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -85,7 +85,7 @@ jobs: - name: Lint with black run: | # Run black to format the code - black . --check + black . --check -v # Test with pytest, to run unit tests and verify the application's functionality - name: Test with pytest From acd7a77a0916b32e61d6863f2ec0c18788f2d878 Mon Sep 17 00:00:00 2001 From: Nikita Date: Wed, 3 Jul 2024 14:58:29 +0000 Subject: [PATCH 40/48] Rewrote line: "black . --check --diff --color --verbose" --- .github/workflows/cyraft-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 9e48358..3dcb37c 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -85,7 +85,7 @@ jobs: - name: Lint with black run: | # Run black to format the code - black . --check -v + black . --check --diff --color --verbose # Test with pytest, to run unit tests and verify the application's functionality - name: Test with pytest From 0b07efb34a510f74724532ca56c3d7bfd982b4ca Mon Sep 17 00:00:00 2001 From: Nikita Date: Wed, 3 Jul 2024 15:02:48 +0000 Subject: [PATCH 41/48] Added flake8 for error lines detection --- .github/workflows/cyraft-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 3dcb37c..0d0dc02 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -38,6 +38,8 @@ jobs: run: | # Upgrade pip to the latest version, to ensure we have the latest package manager python -m pip install --upgrade pip + # Install flake8, which is a Python linter that checks for syntax errors and coding standards + pip install flake8 # If a requirements.txt file exists in the repository, install the dependencies specified in it if [ -f requirements.txt ]; then pip install -r requirements.txt; fi @@ -85,7 +87,7 @@ jobs: - name: Lint with black run: | # Run black to format the code - black . --check --diff --color --verbose + black . --check --diff --color --verbose && flake8 --select=E901,E999,F821,F822,F823 . # Test with pytest, to run unit tests and verify the application's functionality - name: Test with pytest From 987c79d8ddd575a577d08fa13322874a52c992c3 Mon Sep 17 00:00:00 2001 From: Nikita Date: Wed, 3 Jul 2024 15:08:05 +0000 Subject: [PATCH 42/48] Removed the extra line --- tests/raft_log_replication.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/raft_log_replication.py b/tests/raft_log_replication.py index b0d9a02..41bf376 100644 --- a/tests/raft_log_replication.py +++ b/tests/raft_log_replication.py @@ -644,7 +644,6 @@ async def _unittest_raft_log_replication() -> None: async def _unittest_raft_leader_changes() -> None: - logging.root.setLevel(logging.INFO) os.environ["UAVCAN__SRV__REQUEST_VOTE__ID"] = "1" From bc2573f556ec37468ccdbac26effffcd06dd5c00 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:07:26 +0300 Subject: [PATCH 43/48] Fix term: leader only increases term on election_timeout (#18) * Changed the logic of the terms, and also changed the tests * Update tests/raft_node.py Co-authored-by: maksimdrachov <39975120+maksimdrachov@users.noreply.github.com> * Made some minor changes --------- Co-authored-by: maksimdrachov <39975120+maksimdrachov@users.noreply.github.com> --- cyraft/node.py | 1 - tests/raft_leader_election.py | 32 +++---- tests/raft_log_replication.py | 153 +++++++++++++++++----------------- tests/raft_node.py | 15 ++-- 4 files changed, 100 insertions(+), 101 deletions(-) diff --git a/cyraft/node.py b/cyraft/node.py index 1d7c577..785df88 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -698,7 +698,6 @@ async def _on_term_timeout(self) -> None: self._term, ) assert self._state == RaftState.LEADER, "Only leaders have a term timeout" - self._term += 1 self._reset_term_timeout() for remote_node_index, remote_next_index in enumerate(self._next_index): if self._state != RaftState.LEADER: diff --git a/tests/raft_leader_election.py b/tests/raft_leader_election.py index c9b9244..c7a2de7 100644 --- a/tests/raft_leader_election.py +++ b/tests/raft_leader_election.py @@ -150,18 +150,18 @@ async def _unittest_raft_fsm_1() -> None: assert raft_node_1._prev_state == RaftState.CANDIDATE assert raft_node_1._state == RaftState.LEADER assert ( - raft_node_1._term == 2 - ), "+1 due to starting election, +1 due to term timeout (this last one is not guaranteed?)" + raft_node_1._term == 1 + ), "+1 due to starting election" assert raft_node_1._voted_for == 41 assert raft_node_2._prev_state == RaftState.FOLLOWER assert raft_node_2._state == RaftState.FOLLOWER - assert raft_node_2._term == 2, "received heartbeat from LEADER" + assert raft_node_2._term == 1, "received heartbeat from LEADER" assert raft_node_2._voted_for == 41 assert raft_node_3._prev_state == RaftState.FOLLOWER assert raft_node_3._state == RaftState.FOLLOWER - assert raft_node_3._term == 2, "received heartbeat from LEADER" + assert raft_node_3._term == 1, "received heartbeat from LEADER" assert raft_node_3._voted_for == 41 assert raft_node_1._term >= raft_node_2._term, "LEADER term should be higher (or equal) than FOLLOWER" @@ -173,17 +173,17 @@ async def _unittest_raft_fsm_1() -> None: assert raft_node_1._prev_state == RaftState.CANDIDATE assert raft_node_1._state == RaftState.LEADER - assert raft_node_1._term == 12, "+ 10 due to term timeout" + assert raft_node_1._term == 1 assert raft_node_1._voted_for == 41 assert raft_node_2._prev_state == RaftState.FOLLOWER assert raft_node_2._state == RaftState.FOLLOWER - assert raft_node_1._term == 12, "received heartbeat from LEADER" + assert raft_node_1._term == 1, "received heartbeat from LEADER" assert raft_node_2._voted_for == 41 assert raft_node_3._prev_state == RaftState.FOLLOWER assert raft_node_3._state == RaftState.FOLLOWER - assert raft_node_1._term == 12, "received heartbeat from LEADER" + assert raft_node_1._term == 1, "received heartbeat from LEADER" assert raft_node_3._voted_for == 41 assert raft_node_1._term >= raft_node_2._term @@ -246,17 +246,17 @@ async def _unittest_raft_fsm_2(): # assert raft_node_1._prev_state == RaftState.LEADER assert raft_node_1._state == RaftState.FOLLOWER - assert raft_node_1._term == 12, "received heartbeat from LEADER" + assert raft_node_1._term == 2, "received heartbeat from LEADER" assert raft_node_1._voted_for == 42 # assert raft_node_2._prev_state == RaftState.CANDIDATE assert raft_node_2._state == RaftState.LEADER - assert raft_node_2._term == 12, "+ 10 due to term timeout" + assert raft_node_2._term == 2, "+ 10 due to term timeout" assert raft_node_2._voted_for == 42 # assert raft_node_3._prev_state == RaftState.FOLLOWER assert raft_node_3._state == RaftState.FOLLOWER - assert raft_node_3._term == 12, "received heartbeat from LEADER" + assert raft_node_3._term == 2, "received heartbeat from LEADER" assert raft_node_3._voted_for == 42 assert raft_node_2._term >= raft_node_1._term @@ -320,17 +320,17 @@ async def _unittest_raft_fsm_3(): # assert raft_node_1._prev_state == RaftState.CANDIDATE assert raft_node_1._state == RaftState.FOLLOWER - assert raft_node_1._term == 11, "received heartbeat from LEADER" + assert raft_node_1._term == 1, "received heartbeat from LEADER" assert raft_node_1._voted_for == 42 # assert raft_node_2._prev_state == RaftState.CANDIDATE assert raft_node_2._state == RaftState.LEADER - assert raft_node_2._term == 11, "+ 10 due to term timeout" + assert raft_node_2._term == 1, "+ 10 due to term timeout" assert raft_node_2._voted_for == 42 # assert raft_node_3._prev_state == RaftState.FOLLOWER assert raft_node_3._state == RaftState.FOLLOWER - assert raft_node_3._term == 11, "received heartbeat from LEADER" + assert raft_node_3._term == 1, "received heartbeat from LEADER" assert raft_node_3._voted_for == 42 assert raft_node_2._term >= raft_node_1._term @@ -396,17 +396,17 @@ async def _unittest_raft_fsm_4(): assert raft_node_1._prev_state == RaftState.FOLLOWER assert raft_node_1._state == RaftState.FOLLOWER - assert raft_node_1._term == 12, "received heartbeat from LEADER" + assert raft_node_1._term == 2, "received heartbeat from LEADER" assert raft_node_1._voted_for == 42 assert raft_node_2._prev_state == RaftState.CANDIDATE assert raft_node_2._state == RaftState.LEADER - assert raft_node_2._term == 12, "+ 1 due to election timeout, + 10 due to term timeout" + assert raft_node_2._term == 2 assert raft_node_2._voted_for == 42 assert raft_node_3._prev_state == RaftState.FOLLOWER assert raft_node_3._state == RaftState.FOLLOWER - assert raft_node_3._term == 12, "received heartbeat from LEADER" + assert raft_node_3._term == 2, "received heartbeat from LEADER" assert raft_node_3._voted_for == 42 assert raft_node_2._term >= raft_node_1._term diff --git a/tests/raft_log_replication.py b/tests/raft_log_replication.py index 41bf376..83d1cce 100644 --- a/tests/raft_log_replication.py +++ b/tests/raft_log_replication.py @@ -30,56 +30,56 @@ Step 1: Append 3 log entries ____________ ____________ ____________ ____________ | 0 | 1 | 2 | 3 | Log index - | 0 | 4 | 5 | 6 | Log term + | 0 | 0 | 0 | 0 | Log term | empty <= 0 | top_1 <= 7 | top_2 <= 8 | top_3 <= 9 | Name <= value |____________|____________|____________|____________| Step 2: Replace log entry 3 with a new entry ____________ | 3 | Log index - | 7 | Log term + | 1 | Log term | top_3 <= 10| Name <= value |____________| Step 3: Replace log entries 2 and 3 with new entries ____________ ____________ | 2 | 3 | Log index - | 8 | 9 | Log term + | 1 | 1 | Log term | top_2 <= 11| top_3 <= 12| Name <= value |____________|____________| Step 4: Add an already existing log entry ____________ | 3 | Log index - | 9 | Log term + | 1 | Log term | top_3 <= 12| Name <= value |____________| Step 5: Add an additional log entry ____________ | 4 | Log index - | 10 | Log term + | 1 | Log term | top_4 <= 13| Name <= value |____________| Result: ____________ ____________ ____________ ____________ ____________ | 0 | 1 | 2 | 3 | 4 | Log index - | 0 | 4 | 8 | 9 | 10 | Log term + | 0 | 0 | 1 | 1 | 1 | Log term | empty <= 0 | top_1 <= 7 | top_2 <= 10| top_3 <= 11| top_4 <= 13| Name <= value |____________|____________|____________|____________|____________| Step 6: Try to append old log entry (term < currentTerm) ____________ | 4 | Log index - | 9 | Log term + | 0 | Log term | top_4 <= 14| Name <= value |____________| Step 7: Try to append valid log entry, however entry at prev_log_index term does not match ____________ | 4 | Log index - | 11 | Log term + | 1 | Log termы | top_4 <= 15| Name <= value |____________| """ @@ -134,21 +134,21 @@ async def _unittest_raft_log_replication() -> None: _logger.info("================== TEST 1: append 3 log entries ==================") new_entries = [ sirius_cyber_corp.LogEntry_1( - term=4, + term=0, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_1"), value=7, ), ), sirius_cyber_corp.LogEntry_1( - term=5, + term=0, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_2"), value=8, ), ), sirius_cyber_corp.LogEntry_1( - term=6, + term=0, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_3"), value=9, @@ -180,13 +180,13 @@ async def _unittest_raft_log_replication() -> None: assert raft_node_1._log[0].term == 0 assert raft_node_1._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_1._log[0].entry.value == 0 - assert raft_node_1._log[1].term == 4 + assert raft_node_1._log[1].term == 0 assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_1._log[1].entry.value == 7 - assert raft_node_1._log[2].term == 5 + assert raft_node_1._log[2].term == 0 assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_1._log[2].entry.value == 8 - assert raft_node_1._log[3].term == 6 + assert raft_node_1._log[3].term == 0 assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_1._log[3].entry.value == 9 assert raft_node_1._commit_index == 3 @@ -196,13 +196,13 @@ async def _unittest_raft_log_replication() -> None: assert raft_node_2._log[0].term == 0 assert raft_node_2._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_2._log[0].entry.value == 0 - assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].term == 0 assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_2._log[1].entry.value == 7 - assert raft_node_2._log[2].term == 5 + assert raft_node_2._log[2].term == 0 assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_2._log[2].entry.value == 8 - assert raft_node_2._log[3].term == 6 + assert raft_node_2._log[3].term == 0 assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_2._log[3].entry.value == 9 assert raft_node_2._commit_index == 3 @@ -210,13 +210,13 @@ async def _unittest_raft_log_replication() -> None: assert raft_node_3._log[0].term == 0 assert raft_node_3._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_3._log[0].entry.value == 0 - assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].term == 0 assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_3._log[1].entry.value == 7 - assert raft_node_3._log[2].term == 5 + assert raft_node_3._log[2].term == 0 assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_3._log[2].entry.value == 8 - assert raft_node_3._log[3].term == 6 + assert raft_node_3._log[3].term == 0 assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_3._log[3].entry.value == 9 assert raft_node_3._commit_index == 3 @@ -224,14 +224,14 @@ async def _unittest_raft_log_replication() -> None: _logger.info("================== TEST 2: Replace log entry 3 with a new entry ==================") new_entry = sirius_cyber_corp.LogEntry_1( - term=7, + term=1, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_3"), value=10, ), ) request = sirius_cyber_corp.AppendEntries_1.Request( - term=raft_node_1._term, + term=1, prev_log_index=2, # index of top_2 prev_log_term=raft_node_1._log[2].term, log_entry=new_entry, @@ -251,13 +251,13 @@ async def _unittest_raft_log_replication() -> None: assert raft_node_1._log[0].term == 0 assert raft_node_1._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_1._log[0].entry.value == 0 - assert raft_node_1._log[1].term == 4 + assert raft_node_1._log[1].term == 0 assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_1._log[1].entry.value == 7 - assert raft_node_1._log[2].term == 5 + assert raft_node_1._log[2].term == 0 assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_1._log[2].entry.value == 8 - assert raft_node_1._log[3].term == 7 + assert raft_node_1._log[3].term == 1 assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_1._log[3].entry.value == 10 assert raft_node_1._commit_index == 3 @@ -267,13 +267,13 @@ async def _unittest_raft_log_replication() -> None: assert raft_node_2._log[0].term == 0 assert raft_node_2._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_2._log[0].entry.value == 0 - assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].term == 0 assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_2._log[1].entry.value == 7 - assert raft_node_2._log[2].term == 5 + assert raft_node_2._log[2].term == 0 assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_2._log[2].entry.value == 8 - assert raft_node_2._log[3].term == 7 + assert raft_node_2._log[3].term == 1 assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_2._log[3].entry.value == 10 assert raft_node_2._commit_index == 3 @@ -281,13 +281,13 @@ async def _unittest_raft_log_replication() -> None: assert raft_node_3._log[0].term == 0 assert raft_node_3._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_3._log[0].entry.value == 0 - assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].term == 0 assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_3._log[1].entry.value == 7 - assert raft_node_3._log[2].term == 5 + assert raft_node_3._log[2].term == 0 assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_3._log[2].entry.value == 8 - assert raft_node_3._log[3].term == 7 + assert raft_node_3._log[3].term == 1 assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_3._log[3].entry.value == 10 assert raft_node_3._commit_index == 3 @@ -296,14 +296,14 @@ async def _unittest_raft_log_replication() -> None: new_entries = [ sirius_cyber_corp.LogEntry_1( - term=8, + term=1, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_2"), value=11, ), ), sirius_cyber_corp.LogEntry_1( - term=9, + term=1, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_3"), value=12, @@ -332,13 +332,13 @@ async def _unittest_raft_log_replication() -> None: assert len(raft_node_1._log) == 1 + 3 assert raft_node_1._log[0].term == 0 assert raft_node_1._log[0].entry.value == 0 - assert raft_node_1._log[1].term == 4 + assert raft_node_1._log[1].term == 0 assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_1._log[1].entry.value == 7 - assert raft_node_1._log[2].term == 8 + assert raft_node_1._log[2].term == 1 assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_1._log[2].entry.value == 11 - assert raft_node_1._log[3].term == 9 + assert raft_node_1._log[3].term == 1 assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_1._log[3].entry.value == 12 @@ -346,32 +346,32 @@ async def _unittest_raft_log_replication() -> None: assert len(raft_node_2._log) == 1 + 3 assert raft_node_2._log[0].term == 0 assert raft_node_2._log[0].entry.value == 0 - assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].term == 0 assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_2._log[1].entry.value == 7 - assert raft_node_2._log[2].term == 8 + assert raft_node_2._log[2].term == 1 assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_2._log[2].entry.value == 11 - assert raft_node_2._log[3].term == 9 + assert raft_node_2._log[3].term == 1 assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_2._log[3].entry.value == 12 assert len(raft_node_3._log) == 1 + 3 assert raft_node_3._log[0].term == 0 assert raft_node_3._log[0].entry.value == 0 - assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].term == 0 assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_3._log[1].entry.value == 7 - assert raft_node_3._log[2].term == 8 + assert raft_node_3._log[2].term == 1 assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_3._log[2].entry.value == 11 - assert raft_node_3._log[3].term == 9 + assert raft_node_3._log[3].term == 1 assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_3._log[3].entry.value == 12 _logger.info("================== TEST 4: Add an already existing log entry ==================") new_entry = sirius_cyber_corp.LogEntry_1( - term=9, + term=1, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_3"), value=12, @@ -398,13 +398,13 @@ async def _unittest_raft_log_replication() -> None: assert len(raft_node_1._log) == 1 + 3 assert raft_node_1._log[0].term == 0 assert raft_node_1._log[0].entry.value == 0 - assert raft_node_1._log[1].term == 4 + assert raft_node_1._log[1].term == 0 assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_1._log[1].entry.value == 7 - assert raft_node_1._log[2].term == 8 + assert raft_node_1._log[2].term == 1 assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_1._log[2].entry.value == 11 - assert raft_node_1._log[3].term == 9 + assert raft_node_1._log[3].term == 1 assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_1._log[3].entry.value == 12 @@ -412,32 +412,32 @@ async def _unittest_raft_log_replication() -> None: assert len(raft_node_2._log) == 1 + 3 assert raft_node_2._log[0].term == 0 assert raft_node_2._log[0].entry.value == 0 - assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].term == 0 assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_2._log[1].entry.value == 7 - assert raft_node_2._log[2].term == 8 + assert raft_node_2._log[2].term == 1 assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_2._log[2].entry.value == 11 - assert raft_node_2._log[3].term == 9 + assert raft_node_2._log[3].term == 1 assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_2._log[3].entry.value == 12 assert len(raft_node_3._log) == 1 + 3 assert raft_node_3._log[0].term == 0 assert raft_node_3._log[0].entry.value == 0 - assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].term == 0 assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_3._log[1].entry.value == 7 - assert raft_node_3._log[2].term == 8 + assert raft_node_3._log[2].term == 1 assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_3._log[2].entry.value == 11 - assert raft_node_3._log[3].term == 9 + assert raft_node_3._log[3].term == 1 assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_3._log[3].entry.value == 12 _logger.info("================== TEST 5: Add an additional log entry ==================") new_entry = sirius_cyber_corp.LogEntry_1( - term=10, + term=1, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_4"), value=13, @@ -463,23 +463,23 @@ async def _unittest_raft_log_replication() -> None: assert len(raft_node_1._log) == 1 + 4 assert raft_node_1._log[0].term == 0 assert raft_node_1._log[0].entry.value == 0 - assert raft_node_1._log[1].term == 4 + assert raft_node_1._log[1].term == 0 assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_1._log[1].entry.value == 7 - assert raft_node_1._log[2].term == 8 + assert raft_node_1._log[2].term == 1 assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_1._log[2].entry.value == 11 - assert raft_node_1._log[3].term == 9 + assert raft_node_1._log[3].term == 1 assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_1._log[3].entry.value == 12 - assert raft_node_1._log[4].term == 10 + assert raft_node_1._log[4].term == 1 assert raft_node_1._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" assert raft_node_1._log[4].entry.value == 13 _logger.info("================== TEST 6: Try to append old log entry (term < currentTerm) ==================") new_entry = sirius_cyber_corp.LogEntry_1( - term=9, + term=0, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_4"), value=14, @@ -505,47 +505,47 @@ async def _unittest_raft_log_replication() -> None: assert len(raft_node_1._log) == 1 + 4 assert raft_node_1._log[0].term == 0 assert raft_node_1._log[0].entry.value == 0 - assert raft_node_1._log[1].term == 4 + assert raft_node_1._log[1].term == 0 assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_1._log[1].entry.value == 7 - assert raft_node_1._log[2].term == 8 + assert raft_node_1._log[2].term == 1 assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_1._log[2].entry.value == 11 - assert raft_node_1._log[3].term == 9 + assert raft_node_1._log[3].term == 1 assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_1._log[3].entry.value == 12 - assert raft_node_1._log[4].term == 10 + assert raft_node_1._log[4].term == 1 assert raft_node_1._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" assert raft_node_1._log[4].entry.value == 13 assert len(raft_node_2._log) == 1 + 4 assert raft_node_2._log[0].term == 0 assert raft_node_2._log[0].entry.value == 0 - assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].term == 0 assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_2._log[1].entry.value == 7 - assert raft_node_2._log[2].term == 8 + assert raft_node_2._log[2].term == 1 assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_2._log[2].entry.value == 11 - assert raft_node_2._log[3].term == 9 + assert raft_node_2._log[3].term == 1 assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_2._log[3].entry.value == 12 - assert raft_node_2._log[4].term == 10 + assert raft_node_2._log[4].term == 1 assert raft_node_2._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" assert raft_node_2._log[4].entry.value == 13 assert len(raft_node_3._log) == 1 + 4 assert raft_node_3._log[0].term == 0 assert raft_node_3._log[0].entry.value == 0 - assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].term == 0 assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_3._log[1].entry.value == 7 - assert raft_node_3._log[2].term == 8 + assert raft_node_3._log[2].term == 1 assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_3._log[2].entry.value == 11 - assert raft_node_3._log[3].term == 9 + assert raft_node_3._log[3].term == 1 assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_3._log[3].entry.value == 12 - assert raft_node_3._log[4].term == 10 + assert raft_node_3._log[4].term == 1 assert raft_node_3._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" assert raft_node_3._log[4].entry.value == 13 @@ -554,9 +554,9 @@ async def _unittest_raft_log_replication() -> None: ) new_entry = sirius_cyber_corp.LogEntry_1( - term=11, + term=1, entry=sirius_cyber_corp.Entry_1( - name=uavcan.primitive.String_1(value="top_4"), + name=uavcan.primitive.String_1(value="top_5"), value=15, ), ) @@ -580,16 +580,16 @@ async def _unittest_raft_log_replication() -> None: assert len(raft_node_1._log) == 1 + 4 assert raft_node_1._log[0].term == 0 assert raft_node_1._log[0].entry.value == 0 - assert raft_node_1._log[1].term == 4 + assert raft_node_1._log[1].term == 0 assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_1._log[1].entry.value == 7 - assert raft_node_1._log[2].term == 8 + assert raft_node_1._log[2].term == 1 assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_1._log[2].entry.value == 11 - assert raft_node_1._log[3].term == 9 + assert raft_node_1._log[3].term == 1 assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_1._log[3].entry.value == 12 - assert raft_node_1._log[4].term == 10 + assert raft_node_1._log[4].term == 1 assert raft_node_1._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" assert raft_node_1._log[4].entry.value == 13 @@ -643,6 +643,7 @@ async def _unittest_raft_log_replication() -> None: """ + async def _unittest_raft_leader_changes() -> None: logging.root.setLevel(logging.INFO) diff --git a/tests/raft_node.py b/tests/raft_node.py index 2bb58a2..370e1b1 100644 --- a/tests/raft_node.py +++ b/tests/raft_node.py @@ -98,12 +98,13 @@ async def _unittest_raft_node_init() -> None: raft_node.close() await asyncio.sleep(1) # assert raft_node._match_index == [0] + raft_node.close() + await asyncio.sleep(1) async def _unittest_raft_node_term_timeout() -> None: """ - Test that the LEADER node term is increased upon term timeout - + Test that the LEADER node term is not increased by term timeout (this is only done by election timeout) Test that the CANDIDATE/FOLLOWER node term is not increased upon term timeout """ @@ -119,15 +120,15 @@ async def _unittest_raft_node_term_timeout() -> None: raft_node._change_state(RaftState.LEADER) # only leader can increase term await asyncio.sleep(TERM_TIMEOUT) # + 0.1 to make sure the timer has been reset - assert raft_node._term == 1 + assert raft_node._term == 0 # 0 because we have manually set the node to leader (instead of waiting for election) await asyncio.sleep(TERM_TIMEOUT) - assert raft_node._term == 2 + assert raft_node._term == 0 await asyncio.sleep(TERM_TIMEOUT) - assert raft_node._term == 3 + assert raft_node._term == 0 raft_node._change_state(RaftState.FOLLOWER) # follower should not increase term await asyncio.sleep(TERM_TIMEOUT) - assert raft_node._term == 3 + assert raft_node._term == 0 raft_node.close() await asyncio.sleep(1) # give some time for the node to close @@ -784,5 +785,3 @@ async def _unittest_raft_node_append_entries_rpc() -> None: assert raft_node._log[3].entry.value == 12 assert raft_node._log[4].term == 10 assert raft_node._log[4].entry.value == 13 - raft_node.close() - await asyncio.sleep(1) From 558f9ad6749e4bf8cc41e1912cf9e87142a5d8a6 Mon Sep 17 00:00:00 2001 From: Nikita Budovey <73550345+Parsifal22@users.noreply.github.com> Date: Mon, 29 Jul 2024 11:07:19 +0300 Subject: [PATCH 44/48] Complete with test _unittest_raft_leader_changes() (#11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Changed the logic of the terms, and also changed the tests * Update tests/raft_node.py Co-authored-by: maksimdrachov <39975120+maksimdrachov@users.noreply.github.com> * Made some minor changes * In node.py: expanded the log information for some values ​​to make it easier to analyze the code In raft_log_replication.py: start to write test that log replication happens correctly if leadership changes In raft_node.py: add a couple of lines of code to close node after each test, so that after running pytest it does not crash new issue: after the leader changes, the new leader is not aware of the latest index of the neighboring nodes and begins to rewrite them, even though his nodes completely coincide with the neighboring node. * Added test _unittest_raft_leader_changes for log_replication.py * In node.py: expanded the log information for some values ​​to make it easier to analyze the code In raft_log_replication.py: start to write test that log replication happens correctly if leadership changes In raft_node.py: add a couple of lines of code to close node after each test, so that after running pytest it does not crash new issue: after the leader changes, the new leader is not aware of the latest index of the neighboring nodes and begins to rewrite them, even though his nodes completely coincide with the neighboring node. * Added test _unittest_raft_leader_changes for log_replication.py * Adapted the test to term Changes * Returned the deleted fragment from the test _unittest_raft_log_replication() * Reformatted code * Returned the test _unittest_raft_node_heartbeat() * Added Black Formatter to the VSCode * Formatted tests * Changed raft_node.py tests --------- Co-authored-by: maksimdrachov <39975120+maksimdrachov@users.noreply.github.com> --- cyraft/node.py | 190 +++++++++++++++++------- tests/raft_leader_election.py | 4 +- tests/raft_log_replication.py | 268 +++++++++++++++++++--------------- tests/raft_node.py | 3 + 4 files changed, 288 insertions(+), 177 deletions(-) diff --git a/cyraft/node.py b/cyraft/node.py index 785df88..e84e73e 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -88,6 +88,7 @@ def __init__(self) -> None: ## Volatile state on leaders self._next_index: typing.List[int] = [] # index of the next log entry to send to that server + # self._match_index: typing.List[int] = [] # index of highest log entry known to be replicated on server ######################################## @@ -203,46 +204,48 @@ async def _serve_request_vote_impl( - the voted_for of the node the method will either grant or deny the vote. """ - # Reply false if term < self._term (§5.1) - # (or if already voted for another candidate in this term) - if request.term < self._term or (self._voted_for is not None): + + vote_granted = None + + """ + If request.term is higher than self.term, then: + 1. Switch to follower + 2. Update term to new term + 3. Reset voted_for + """ + if request.term > self._term: + self._change_state(RaftState.FOLLOWER) # Our term is stale, so we can't serve as leader + self._term = request.term + self._voted_for = None + + if request.term < self._term: + vote_granted = False _logger.info( - c["request_vote"] - + "Request vote request denied (term (%d) < self._term (%d) or already voted for another candidate in this term (%s))" - + c["end_color"], + c["request_vote"] + "Request vote request denied (term (%d) < self._term (%d)" + c["end_color"], request.term, self._term, - self._voted_for, ) - return sirius_cyber_corp.RequestVote_1.Response(term=self._term, vote_granted=False) - - # If voted_for is null or candidateId, and candidate’s log is at - # least as up-to-date as receiver’s log, grant vote (§5.2, §5.4) - elif self._voted_for is None or self._voted_for == client_node_id: - # log comparison - if self._log[request.last_log_index].term == request.last_log_term: - _logger.info(c["request_vote"] + "Request vote request granted" + c["end_color"]) + else: + vote_granted = (self._voted_for is None or self._voted_for == client_node_id) and self._log[ + request.last_log_index + ].term == request.last_log_term + + if vote_granted: + self._change_state( + RaftState.FOLLOWER + ) # Avoiding race condition when Candidate. This is necessary to avoid excessive elections self._voted_for = client_node_id - self._term = request.term - return sirius_cyber_corp.RequestVote_1.Response( - term=self._term, - vote_granted=True, - ) else: - _logger.info(c["request_vote"] + "Request vote request denied (failed log comparison)" + c["end_color"]) - return sirius_cyber_corp.RequestVote_1.Response(term=self._term, vote_granted=False) + _logger.info( + c["request_vote"] + + "Request vote request denied log is not up to date (self._log.term (%d) != request.last_log_term (%d) or already voted for another candidate in this term (%s))" + + c["end_color"], + self._log[request.last_log_index].term, + request.last_log_term, + self._voted_for, + ) - _logger.info( - c["request_vote"] + "Node ID: %d -- Request vote request denied (unknown reason)" + c["end_color"], - self._node.id, - ) - _logger.info( - c["request_vote"] + "Node ID: %d -- Current term: %d, Request term: %d" + c["end_color"], - self._node.id, - self._term, - request.term, - ) - assert False, "Should not reach here!" + return sirius_cyber_corp.RequestVote_1.Response(term=self._term, vote_granted=vote_granted) async def _start_election(self) -> None: """ @@ -253,7 +256,10 @@ async def _start_election(self) -> None: """ assert self._state == RaftState.CANDIDATE, "Election can only be started by a candidate" - _logger.info(c["raft_logic"] + "Node ID: %d -- Starting election" + c["end_color"], self._node.id) + _logger.info( + c["raft_logic"] + "Node ID: %d -- Starting election" + c["end_color"], + self._node.id, + ) # Increment currentTerm self._term += 1 # Vote for self @@ -294,10 +300,16 @@ async def _start_election(self) -> None: # If votes received from majority of servers: become leader if number_of_votes > number_of_nodes / 2: # int(5/2) = 2, int(3/2) = 1 - _logger.info(c["raft_logic"] + "Node ID: %d -- Became leader" + c["end_color"], self._node.id) + _logger.info( + c["raft_logic"] + "Node ID: %d -- Became leader" + c["end_color"], + self._node.id, + ) self._change_state(RaftState.LEADER) else: - _logger.info(c["raft_logic"] + "Node ID: %d -- Election failed" + c["end_color"], self._node.id) + _logger.info( + c["raft_logic"] + "Node ID: %d -- Election failed" + c["end_color"], + self._node.id, + ) # If election fails, revert to follower self._voted_for = None self._change_state(RaftState.FOLLOWER) @@ -331,7 +343,10 @@ async def _serve_append_entries( ) return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) else: # request.term >= self._term - _logger.info(c["append_entries"] + "Node ID: %d -- Heartbeat received" + c["end_color"], self._node.id) + _logger.info( + c["append_entries"] + "Node ID: %d -- Heartbeat received" + c["end_color"], + self._node.id, + ) if metadata.client_node_id != self._voted_for and request.term > self._term: _logger.info( c["append_entries"] + "Node ID: %d -- Heartbeat from new leader: %d" + c["end_color"], @@ -342,6 +357,7 @@ async def _serve_append_entries( self._change_state(RaftState.FOLLOWER) # this will reset the election timeout as well self._term = request.term # update term + return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=True) # Reply false if term < currentTerm (§5.1) @@ -349,10 +365,13 @@ async def _serve_append_entries( _logger.info( c["append_entries"] + "Node ID: %d -- Append entries request denied (term (%d) < currentTerm (%d))" + + "Node ID: %d -- Append entries request denied (term (%d) < currentTerm (%d))" + c["end_color"], self._node.id, request.term, self._term, + request.term, + self._term, ) return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) @@ -367,8 +386,13 @@ async def _serve_append_entries( self._node.id, ) return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) + except IndexError as e: except IndexError as e: _logger.info( + c["append_entries"] + + "Node ID: %d -- Append entries request denied (log mismatch 2). IndexError: %s. " + + "prev_log_index: %d, log_length: %d" + + c["end_color"], c["append_entries"] + "Node ID: %d -- Append entries request denied (log mismatch 2). IndexError: %s. " + "prev_log_index: %d, log_length: %d" @@ -377,6 +401,9 @@ async def _serve_append_entries( str(e), request.prev_log_index, len(self._log), + str(e), + request.prev_log_index, + len(self._log), ) return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) @@ -395,7 +422,11 @@ def _append_entries_processing( # If an existing entry conflicts with a new one (same index but different terms), # delete the existing entry and all that follow it (§5.3) new_index = request.prev_log_index + 1 - _logger.info(c["append_entries"] + "Node ID: %d -- new_index: %d" + c["end_color"], self._node.id, new_index) + _logger.info( + c["append_entries"] + "Node ID: %d -- new_index: %d" + c["end_color"], + self._node.id, + new_index, + ) for log_index, log_entry in enumerate(self._log[1:]): if ( log_index + 1 # index + 1 because we skip the first entry (self._log[1:]) @@ -421,13 +452,17 @@ def _append_entries_processing( and self._log[new_index].entry.value == request.log_entry[0].entry.value ): # log comparison can be done better, once this is fixed: https://github.com/OpenCyphal/pycyphal/issues/297 append_new_entry = False - _logger.info(c["append_entries"] + "Node ID: %d -- entry already exists" + c["end_color"], self._node.id) + _logger.info( + c["append_entries"] + "Node ID: %d -- entry already exists" + c["end_color"], + self._node.id, + ) # 2. If it does not exist, append it if append_new_entry: if new_index < len(self._log): _logger.info("new_index < len(self._log): %s", new_index < len(self._log)) _logger.info( - "self._log[new_index] == request.log_entry[0]: %s", self._log[new_index] == request.log_entry[0] + "self._log[new_index] == request.log_entry[0]: %s", + self._log[new_index] == request.log_entry[0], ) assert False self._log.append(request.log_entry[0]) @@ -453,7 +488,7 @@ def _append_entries_processing( # Update current_term (if follower) (leaders will update their own term on timeout) if self._state == RaftState.FOLLOWER: - self._reset_election_timeout() + self._change_state(RaftState.FOLLOWER) # this will reset the election timeout as well self._term = request.log_entry[0].term async def _send_heartbeat(self, remote_node_index: int) -> None: @@ -492,7 +527,17 @@ async def _send_heartbeat(self, remote_node_index: int) -> None: request.prev_log_term, ) assert len(request.log_entry) == 0, "Heartbeat should not have a log entry" - response = await remote_client(request) # metadata is filled out by the client + try: + response = await remote_client(request) # metadata is filled out by the client + except pycyphal.presentation._port._error.PortClosedError: + _logger.info( + c["raft_logic"] + + "Node ID: %d -- Failed to send append entries request to remote node %d (port closed)" + + c["end_color"], + self._node.id, + self._cluster[remote_node_index], + ) + return if response: if response.success: _logger.info( @@ -558,7 +603,17 @@ async def _send_append_entry(self, remote_node_index: int): log_entry=self._log[remote_next_index], ) assert len(request.log_entry) == 1, "Append entry should have a (single) log entry" - response = await remote_client(request) # metadata is filled out by the client + try: + response = await remote_client(request) # metadata is filled out by the client + except pycyphal.presentation._port._error.PortClosedError: + _logger.info( + c["raft_logic"] + + "Node ID: %d -- Failed to send append entries request to remote node %d (port closed)" + + c["end_color"], + self._node.id, + self._cluster[remote_node_index], + ) + return if response: if response.success: _logger.info( @@ -625,7 +680,12 @@ def _change_state(self, new_state: RaftState) -> None: self._prev_state = self._state self._state = new_state - _logger.info("Node ID: %d -- Changing state from %s to %s", self._node.id, self._prev_state, self._state) + _logger.info( + "Node ID: %d -- Changing state from %s to %s", + self._node.id, + self._prev_state, + self._state, + ) if self._state == RaftState.FOLLOWER: # Cancel the term timeout (if it exists), and schedule a new election timeout. @@ -642,6 +702,7 @@ def _change_state(self, new_state: RaftState) -> None: elif self._state == RaftState.LEADER: assert self._prev_state == RaftState.CANDIDATE, "Invalid state change 3" + # Cancel the election timeout (if it exists), and schedule a new term timeout. if hasattr(self, "_election_timer"): self._election_timer.cancel() @@ -654,13 +715,16 @@ def _reset_election_timeout(self) -> None: If a follower receives a heartbeat from the leader, it should reset its election timeout. """ assert self._state == RaftState.FOLLOWER, "Only followers should reset the election timeout" - _logger.info(c["raft_logic"] + "Node ID: %d -- Resetting election timeout" + c["end_color"], self._node.id) + _logger.info( + c["raft_logic"] + "Node ID: %d -- Resetting election timeout" + c["end_color"], + self._node.id, + ) loop = asyncio.get_event_loop() - self._next_election_timeout = loop.time() + self._election_timeout if hasattr(self, "_election_timer"): self._election_timer.cancel() - self._election_timer = loop.call_at( - self._next_election_timeout, lambda: asyncio.create_task(self._on_election_timeout()) + self._election_timer = loop.call_later( + self._election_timeout, + lambda: asyncio.create_task(self._on_election_timeout()), ) def _reset_term_timeout(self) -> None: @@ -668,12 +732,14 @@ def _reset_term_timeout(self) -> None: Once a term timeout is reached, another term callback is scheduled. """ assert self._state == RaftState.LEADER, "Only leaders should reset the term timeout" - _logger.info(c["raft_logic"] + "Node ID: %d -- Resetting term timeout" + c["end_color"], self._node.id) + _logger.info( + c["raft_logic"] + "Node ID: %d -- Resetting term timeout" + c["end_color"], + self._node.id, + ) loop = asyncio.get_event_loop() - self._next_term_timeout = loop.time() + self._term_timeout if hasattr(self, "_term_timer"): self._term_timer.cancel() - self._term_timer = loop.call_at(self._next_term_timeout, lambda: asyncio.create_task(self._on_term_timeout())) + self._term_timer = loop.call_later(self._term_timeout, lambda: asyncio.create_task(self._on_term_timeout())) async def _on_election_timeout(self) -> None: """ @@ -681,7 +747,10 @@ async def _on_election_timeout(self) -> None: The node starts an election and then restarts the election timeout. """ assert self._state == RaftState.FOLLOWER, "Only followers have an election timeout" - _logger.info(c["raft_logic"] + "Node ID: %d -- Election timeout reached" + c["end_color"], self._node.id) + _logger.info( + c["raft_logic"] + "Node ID: %d -- Election timeout reached" + c["end_color"], + self._node.id, + ) self._change_state(RaftState.CANDIDATE) await self._start_election() @@ -714,6 +783,17 @@ async def _on_term_timeout(self) -> None: self._cluster[remote_node_index], ) + + _logger.info( + c["raft_logic"] + + "Node ID: %d -- Value of next index = %d and value of remote next index = %s for node %s" + + c["end_color"], + self._node.id, + self._commit_index + 1, + remote_next_index, + self._cluster[remote_node_index], + ) + if self._commit_index + 1 == remote_next_index: # remote log is up to date _logger.info( c["raft_logic"] @@ -744,14 +824,16 @@ async def run(self) -> None: if self._state == RaftState.FOLLOWER: self._next_election_timeout = loop.time() + self._election_timeout self._term_timer = loop.call_at( - self._next_election_timeout, lambda: asyncio.create_task(self._on_election_timeout()) + self._next_election_timeout, + lambda: asyncio.create_task(self._on_election_timeout()), ) # Schedule term timeout (only for leader) if self._state == RaftState.LEADER: self._next_term_timeout = loop.time() + self._term_timeout self._term_timer = loop.call_at( - self._next_term_timeout, lambda: asyncio.create_task(self._on_term_timeout()) + self._next_term_timeout, + lambda: asyncio.create_task(self._on_term_timeout()), ) def close(self) -> None: diff --git a/tests/raft_leader_election.py b/tests/raft_leader_election.py index c7a2de7..25fe69d 100644 --- a/tests/raft_leader_election.py +++ b/tests/raft_leader_election.py @@ -149,9 +149,7 @@ async def _unittest_raft_fsm_1() -> None: assert raft_node_1._prev_state == RaftState.CANDIDATE assert raft_node_1._state == RaftState.LEADER - assert ( - raft_node_1._term == 1 - ), "+1 due to starting election" + assert raft_node_1._term == 1, "+1 due to starting election" assert raft_node_1._voted_for == 41 assert raft_node_2._prev_state == RaftState.FOLLOWER diff --git a/tests/raft_log_replication.py b/tests/raft_log_replication.py index 83d1cce..c1b450c 100644 --- a/tests/raft_log_replication.py +++ b/tests/raft_log_replication.py @@ -13,6 +13,7 @@ # Add parent directory to Python path # Get the absolute path of the parent directory parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # Add the parent directory to the Python path sys.path.append(parent_dir) # This can be removed if setting PYTHONPATH (export PYTHONPATH=cyraft) @@ -599,11 +600,11 @@ async def _unittest_raft_log_replication() -> None: await asyncio.sleep(1) -# ======================================================================================= -# ========== Test that log replication happens correctly if leadership changes ========== -# ======================================================================================= - """ +======================================================================================= +========== Test that log replication happens correctly if leadership changes ========== +======================================================================================= + Initially, all nodes have an empty log entry at index 0 with term 0. ____________ | 0 | Log index @@ -615,36 +616,36 @@ async def _unittest_raft_log_replication() -> None: ____________ ____________ ____________ ____________ | 0 | 1 | 2 | 3 | Log index - | 0 | 4 | 5 | 6 | Log term + | 0 | 1 | 1 | 1 | Log term | empty <= 0 | top_1 <= 7 | top_2 <= 8 | top_3 <= 9 | Name <= value |____________|____________|____________|____________| Step 2: Leadership Change to Node 42 and Add New Entry ____________ ____________ ____________ ____________ _____________ | 0 | 1 | 2 | 3 | 4 | Log index - | 0 | 4 | 5 | 6 | 7 | Log term + | 0 | 1 | 1 | 1 | 2 | Log term | empty <= 0 | top_1 <= 7 | top_2 <= 8 | top_3 <= 9 | top_4 <= 17 | Name <= value |____________|____________|____________|____________|_____________| - - Step 3: Replace Log Entry 3 with a New Entry fom LEADER Node 42 + + Step 3: Replace Log Entry 3 with a New Entry from LEADER Node 42 ____________ | 3 | Log index - | 7 | Log term + | 2 | Log term | top_3 <= 10| Name <= value |____________| Result: ____________ ____________ ____________ _____________ | 0 | 1 | 2 | 3 | Log index - | 0 | 4 | 5 | 7 | Log term + | 0 | 1 | 1 | 2 | Log term | empty <= 0 | top_1 <= 7 | top_2 <= 8 | top_3 <= 10 | Name <= value |____________|____________|____________|_____________| """ - async def _unittest_raft_leader_changes() -> None: + logging.root.setLevel(logging.INFO) os.environ["UAVCAN__SRV__REQUEST_VOTE__ID"] = "1" @@ -666,18 +667,31 @@ async def _unittest_raft_leader_changes() -> None: os.environ["UAVCAN__NODE__ID"] = "43" raft_node_3 = RaftNode() raft_node_3.term_timeout = TERM_TIMEOUT - raft_node_3.election_timeout = ELECTION_TIMEOUT + 1 + raft_node_3.election_timeout = ELECTION_TIMEOUT + 2 + os.environ["UAVCAN__NODE__ID"] = "44" + raft_node_4 = RaftNode() + raft_node_4.term_timeout = TERM_TIMEOUT + raft_node_4.election_timeout = ELECTION_TIMEOUT + 2.1 # make all part of the same cluster - cluster = [raft_node_1._node.id, raft_node_2._node.id, raft_node_3._node.id] + cluster = [ + raft_node_1._node.id, + raft_node_2._node.id, + raft_node_3._node.id, + raft_node_4._node.id, + ] raft_node_1.add_remote_node(cluster) raft_node_2.add_remote_node(cluster) raft_node_3.add_remote_node(cluster) + raft_node_4.add_remote_node(cluster) # start all nodes asyncio.create_task(raft_node_1.run()) asyncio.create_task(raft_node_2.run()) asyncio.create_task(raft_node_3.run()) + asyncio.create_task(raft_node_4.run()) + + _logger.info("================== TEST 1: Append 3 Log Entries to LEADER node 41 ==================") # wait for the leader to be elected await asyncio.sleep(ELECTION_TIMEOUT + 1) @@ -689,24 +703,25 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_1._voted_for == 41 assert raft_node_2._voted_for == 41 assert raft_node_3._voted_for == 41 + assert raft_node_4._voted_for == 41 new_entries = [ sirius_cyber_corp.LogEntry_1( - term=4, + term=1, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_1"), value=7, ), ), sirius_cyber_corp.LogEntry_1( - term=5, + term=1, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_2"), value=8, ), ), sirius_cyber_corp.LogEntry_1( - term=6, + term=1, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_3"), value=9, @@ -738,13 +753,13 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_1._log[0].term == 0 assert raft_node_1._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_1._log[0].entry.value == 0 - assert raft_node_1._log[1].term == 4 + assert raft_node_1._log[1].term == 1 assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_1._log[1].entry.value == 7 - assert raft_node_1._log[2].term == 5 + assert raft_node_1._log[2].term == 1 assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_1._log[2].entry.value == 8 - assert raft_node_1._log[3].term == 6 + assert raft_node_1._log[3].term == 1 assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_1._log[3].entry.value == 9 assert raft_node_1._commit_index == 3 @@ -754,13 +769,13 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_2._log[0].term == 0 assert raft_node_2._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_2._log[0].entry.value == 0 - assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].term == 1 assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_2._log[1].entry.value == 7 - assert raft_node_2._log[2].term == 5 + assert raft_node_2._log[2].term == 1 assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_2._log[2].entry.value == 8 - assert raft_node_2._log[3].term == 6 + assert raft_node_2._log[3].term == 1 assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_2._log[3].entry.value == 9 assert raft_node_2._commit_index == 3 @@ -769,76 +784,70 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_3._log[0].term == 0 assert raft_node_3._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_3._log[0].entry.value == 0 - assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].term == 1 assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_3._log[1].entry.value == 7 - assert raft_node_3._log[2].term == 5 + assert raft_node_3._log[2].term == 1 assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_3._log[2].entry.value == 8 - assert raft_node_3._log[3].term == 6 + assert raft_node_3._log[3].term == 1 assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_3._log[3].entry.value == 9 assert raft_node_3._commit_index == 3 + assert len(raft_node_4._log) == 1 + 3 + assert raft_node_4._log[0].term == 0 + assert raft_node_4._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty + assert raft_node_4._log[0].entry.value == 0 + assert raft_node_4._log[1].term == 1 + assert raft_node_4._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_4._log[1].entry.value == 7 + assert raft_node_4._log[2].term == 1 + assert raft_node_4._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_4._log[2].entry.value == 8 + assert raft_node_4._log[3].term == 1 + assert raft_node_4._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_4._log[3].entry.value == 9 + assert raft_node_4._commit_index == 3 + await asyncio.sleep(TERM_TIMEOUT + 1) - _logger.info("================== TEST 1: Change LEADER and check logs ==================") - # New LEADER => raft_node_2 + _logger.info("================== TEST 2: Leadership Change to Node 42 and Add New Entry ==================") - raft_node_1.election_timeout = ELECTION_TIMEOUT + 1 - raft_node_2.election_timeout = ELECTION_TIMEOUT - raft_node_3.election_timeout = ELECTION_TIMEOUT + 1 - - raft_node_1._change_state(RaftState.FOLLOWER) - raft_node_2._change_state(RaftState.FOLLOWER) - raft_node_3._change_state(RaftState.FOLLOWER) + # New LEADER => raft_node_2 - # Need to reset the votes of each node so that node 41 doesn't win again. - raft_node_1._voted_for = None - raft_node_2._voted_for = None - raft_node_3._voted_for = None + await asyncio.sleep(TERM_TIMEOUT) - await asyncio.sleep(2 * ELECTION_TIMEOUT + 1) + raft_node_1.close() # Simulation of the disappearance of a leader from a cluster - assert raft_node_1._state == RaftState.FOLLOWER - assert raft_node_1._term == 20, "received heartbeat from LEADER" - assert raft_node_1._voted_for == 42 + await asyncio.sleep( + ELECTION_TIMEOUT + 9 * TERM_TIMEOUT + ) # Nine terms are needed for complete replication of logs from the leader to the followers. assert raft_node_2._state == RaftState.LEADER - assert raft_node_2._term == 20 + assert raft_node_2._term == 2 assert raft_node_2._voted_for == 42 + assert raft_node_4._state == RaftState.FOLLOWER + assert raft_node_4._term == 2, "received heartbeat from LEADER" + assert raft_node_4._voted_for == 42 + assert raft_node_3._state == RaftState.FOLLOWER - assert raft_node_3._term == 20, "received heartbeat from LEADER" + assert raft_node_3._term == 2, "received heartbeat from LEADER" assert raft_node_3._voted_for == 42 # check that all logs are saved from previous LEADER - assert len(raft_node_1._log) == 1 + 3 - assert raft_node_1._log[0].term == 0 - assert raft_node_1._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty - assert raft_node_1._log[0].entry.value == 0 - assert raft_node_1._log[1].term == 4 - assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" - assert raft_node_1._log[1].entry.value == 7 - assert raft_node_1._log[2].term == 5 - assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" - assert raft_node_1._log[2].entry.value == 8 - assert raft_node_1._log[3].term == 6 - assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" - assert raft_node_1._log[3].entry.value == 9 - assert raft_node_1._commit_index == 3 - assert len(raft_node_2._log) == 1 + 3 assert raft_node_2._log[0].term == 0 assert raft_node_2._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_2._log[0].entry.value == 0 - assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].term == 1 assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_2._log[1].entry.value == 7 - assert raft_node_2._log[2].term == 5 + assert raft_node_2._log[2].term == 1 assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_2._log[2].entry.value == 8 - assert raft_node_2._log[3].term == 6 + assert raft_node_2._log[3].term == 1 assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_2._log[3].entry.value == 9 assert raft_node_2._commit_index == 3 @@ -847,21 +856,34 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_3._log[0].term == 0 assert raft_node_3._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_3._log[0].entry.value == 0 - assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].term == 1 assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_3._log[1].entry.value == 7 - assert raft_node_3._log[2].term == 5 + assert raft_node_3._log[2].term == 1 assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_3._log[2].entry.value == 8 - assert raft_node_3._log[3].term == 6 + assert raft_node_3._log[3].term == 1 assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_3._log[3].entry.value == 9 assert raft_node_3._commit_index == 3 - _logger.info("================== TEST 2: Append new log entry fron another LEADER ==================") + assert len(raft_node_4._log) == 1 + 3 + assert raft_node_4._log[0].term == 0 + assert raft_node_4._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty + assert raft_node_4._log[0].entry.value == 0 + assert raft_node_4._log[1].term == 1 + assert raft_node_4._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_4._log[1].entry.value == 7 + assert raft_node_4._log[2].term == 1 + assert raft_node_4._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_4._log[2].entry.value == 8 + assert raft_node_4._log[3].term == 1 + assert raft_node_4._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_4._log[3].entry.value == 9 + assert raft_node_4._commit_index == 3 new_entry = sirius_cyber_corp.LogEntry_1( - term=7, + term=raft_node_2._term, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_4"), value=13, @@ -889,69 +911,76 @@ async def _unittest_raft_leader_changes() -> None: assert len(raft_node_2._log) == 1 + 4 assert raft_node_2._log[0].term == 0 assert raft_node_2._log[0].entry.value == 0 - assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].term == 1 assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_2._log[1].entry.value == 7 - assert raft_node_2._log[2].term == 5 + assert raft_node_2._log[2].term == 1 assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_2._log[2].entry.value == 8 - assert raft_node_2._log[3].term == 6 + assert raft_node_2._log[3].term == 1 assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_2._log[3].entry.value == 9 - assert raft_node_2._log[4].term == 7 + assert raft_node_2._log[4].term == 2 assert raft_node_2._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" assert raft_node_2._log[4].entry.value == 13 assert raft_node_2._commit_index == 4 - assert len(raft_node_1._log) == 1 + 4 - assert raft_node_1._log[0].term == 0 - assert raft_node_1._log[0].entry.value == 0 - assert raft_node_1._log[1].term == 4 - assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" - assert raft_node_1._log[1].entry.value == 7 - assert raft_node_1._log[2].term == 5 - assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" - assert raft_node_1._log[2].entry.value == 8 - assert raft_node_1._log[3].term == 6 - assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" - assert raft_node_1._log[3].entry.value == 9 - assert raft_node_1._log[4].term == 7 - assert raft_node_1._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" - assert raft_node_1._log[4].entry.value == 13 - assert raft_node_1._commit_index == 4 - assert len(raft_node_3._log) == 1 + 4 assert raft_node_3._log[0].term == 0 assert raft_node_3._log[0].entry.value == 0 - assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].term == 1 assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_3._log[1].entry.value == 7 - assert raft_node_3._log[2].term == 5 + assert raft_node_3._log[2].term == 1 assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_3._log[2].entry.value == 8 - assert raft_node_3._log[3].term == 6 + assert raft_node_3._log[3].term == 1 assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_3._log[3].entry.value == 9 - assert raft_node_3._log[4].term == 7 + assert raft_node_3._log[4].term == 2 assert raft_node_3._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" assert raft_node_3._log[4].entry.value == 13 assert raft_node_3._commit_index == 4 + assert len(raft_node_4._log) == 1 + 4 + assert raft_node_4._log[0].term == 0 + assert raft_node_4._log[0].entry.value == 0 + assert raft_node_4._log[1].term == 1 + assert raft_node_4._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_4._log[1].entry.value == 7 + assert raft_node_4._log[2].term == 1 + assert raft_node_4._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_4._log[2].entry.value == 8 + assert raft_node_4._log[3].term == 1 + assert raft_node_4._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_4._log[3].entry.value == 9 + assert raft_node_4._log[4].term == 2 + assert raft_node_4._log[4].entry.name.value.tobytes().decode("utf-8") == "top_4" + assert raft_node_4._log[4].entry.value == 13 + assert raft_node_4._commit_index == 4 + + raft_node_2.close() + raft_node_3.close() + raft_node_4.close() + await asyncio.sleep(1) + _logger.info( - "================== TEST 3: Replace log entry 3 with new entries from another LEADER ==================" + "================== TEST 3: Replace Log Entry 3 with a New Entry from LEADER Node 42 ==================" ) + await asyncio.sleep(TERM_TIMEOUT) + new_entry = sirius_cyber_corp.LogEntry_1( - term=7, + term=2, entry=sirius_cyber_corp.Entry_1( name=uavcan.primitive.String_1(value="top_3"), value=10, ), ) request = sirius_cyber_corp.AppendEntries_1.Request( - term=raft_node_1._term, + term=raft_node_2._term, prev_log_index=2, # index of top_2 - prev_log_term=raft_node_1._log[2].term, + prev_log_term=raft_node_2._log[2].term, log_entry=new_entry, ) metadata = pycyphal.presentation.ServiceRequestMetadata( @@ -969,49 +998,48 @@ async def _unittest_raft_leader_changes() -> None: assert raft_node_2._log[0].term == 0 assert raft_node_2._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_2._log[0].entry.value == 0 - assert raft_node_2._log[1].term == 4 + assert raft_node_2._log[1].term == 1 assert raft_node_2._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_2._log[1].entry.value == 7 - assert raft_node_2._log[2].term == 5 + assert raft_node_2._log[2].term == 1 assert raft_node_2._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_2._log[2].entry.value == 8 - assert raft_node_2._log[3].term == 7 + assert raft_node_2._log[3].term == 2 assert raft_node_2._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" assert raft_node_2._log[3].entry.value == 10 assert raft_node_2._commit_index == 3 # check if the new entry is replicated in the follower nodes - assert len(raft_node_1._log) == 1 + 3 - assert raft_node_1._log[0].term == 0 - assert raft_node_1._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty - assert raft_node_1._log[0].entry.value == 0 - assert raft_node_1._log[1].term == 4 - assert raft_node_1._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" - assert raft_node_1._log[1].entry.value == 7 - assert raft_node_1._log[2].term == 5 - assert raft_node_1._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" - assert raft_node_1._log[2].entry.value == 8 - assert raft_node_1._log[3].term == 7 - assert raft_node_1._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" - assert raft_node_1._log[3].entry.value == 10 - assert raft_node_1._commit_index == 3 + assert len(raft_node_4._log) == 1 + 4 + assert raft_node_4._log[0].term == 0 + assert raft_node_4._log[0].entry.value == 0 + assert raft_node_4._log[1].term == 1 + assert raft_node_4._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" + assert raft_node_4._log[1].entry.value == 7 + assert raft_node_4._log[2].term == 1 + assert raft_node_4._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" + assert raft_node_4._log[2].entry.value == 8 + assert raft_node_4._log[3].term == 1 + assert raft_node_4._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" + assert raft_node_4._log[3].entry.value == 9 + assert raft_node_4._commit_index == 4 - assert len(raft_node_3._log) == 1 + 3 + assert len(raft_node_3._log) == 1 + 4 assert raft_node_3._log[0].term == 0 assert raft_node_3._log[0].entry.name.value.tobytes().decode("utf-8") == "" # index zero entry is empty assert raft_node_3._log[0].entry.value == 0 - assert raft_node_3._log[1].term == 4 + assert raft_node_3._log[1].term == 1 assert raft_node_3._log[1].entry.name.value.tobytes().decode("utf-8") == "top_1" assert raft_node_3._log[1].entry.value == 7 - assert raft_node_3._log[2].term == 5 + assert raft_node_3._log[2].term == 1 assert raft_node_3._log[2].entry.name.value.tobytes().decode("utf-8") == "top_2" assert raft_node_3._log[2].entry.value == 8 - assert raft_node_3._log[3].term == 7 + assert raft_node_3._log[3].term == 1 assert raft_node_3._log[3].entry.name.value.tobytes().decode("utf-8") == "top_3" - assert raft_node_3._log[3].entry.value == 10 - assert raft_node_3._commit_index == 3 + assert raft_node_3._log[3].entry.value == 9 + assert raft_node_3._commit_index == 4 - raft_node_1.close() raft_node_2.close() raft_node_3.close() + raft_node_4.close() await asyncio.sleep(1) diff --git a/tests/raft_node.py b/tests/raft_node.py index 370e1b1..1e4a711 100644 --- a/tests/raft_node.py +++ b/tests/raft_node.py @@ -13,6 +13,7 @@ # Add parent directory to Python path # Get the absolute path of the parent directory parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # Add the parent directory to the Python path sys.path.append(parent_dir) # This can be removed if setting PYTHONPATH (export PYTHONPATH=cyraft) @@ -121,6 +122,7 @@ async def _unittest_raft_node_term_timeout() -> None: await asyncio.sleep(TERM_TIMEOUT) # + 0.1 to make sure the timer has been reset assert raft_node._term == 0 # 0 because we have manually set the node to leader (instead of waiting for election) + assert raft_node._term == 0 # 0 because we have manually set the node to leader (instead of waiting for election) await asyncio.sleep(TERM_TIMEOUT) assert raft_node._term == 0 await asyncio.sleep(TERM_TIMEOUT) @@ -785,3 +787,4 @@ async def _unittest_raft_node_append_entries_rpc() -> None: assert raft_node._log[3].entry.value == 12 assert raft_node._log[4].term == 10 assert raft_node._log[4].entry.value == 13 + raft_node.close() From 4d535dd4b706c1d665997e1a8410e7c4ccd97a93 Mon Sep 17 00:00:00 2001 From: Nikita Date: Mon, 1 Jul 2024 07:32:18 +0000 Subject: [PATCH 45/48] Fix small issues with tests --- cyraft/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cyraft/node.py b/cyraft/node.py index e84e73e..1772875 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -488,7 +488,7 @@ def _append_entries_processing( # Update current_term (if follower) (leaders will update their own term on timeout) if self._state == RaftState.FOLLOWER: - self._change_state(RaftState.FOLLOWER) # this will reset the election timeout as well + self._reset_election_timeout() self._term = request.log_entry[0].term async def _send_heartbeat(self, remote_node_index: int) -> None: From 62340fba6ad77b3dee937ad7259552b3551e2eba Mon Sep 17 00:00:00 2001 From: Nikita Date: Wed, 3 Jul 2024 14:15:52 +0000 Subject: [PATCH 46/48] implemented the Black Code Style --- .github/workflows/cyraft-test.yml | 3 +++ cyraft/node.py | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 0d0dc02..4c967cc 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -38,8 +38,10 @@ jobs: run: | # Upgrade pip to the latest version, to ensure we have the latest package manager python -m pip install --upgrade pip + # Install flake8, which is a Python linter that checks for syntax errors and coding standards pip install flake8 + # If a requirements.txt file exists in the repository, install the dependencies specified in it if [ -f requirements.txt ]; then pip install -r requirements.txt; fi @@ -87,6 +89,7 @@ jobs: - name: Lint with black run: | # Run black to format the code + black --line-length 120 . black . --check --diff --color --verbose && flake8 --select=E901,E999,F821,F822,F823 . # Test with pytest, to run unit tests and verify the application's functionality diff --git a/cyraft/node.py b/cyraft/node.py index 1772875..4393aee 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -357,7 +357,6 @@ async def _serve_append_entries( self._change_state(RaftState.FOLLOWER) # this will reset the election timeout as well self._term = request.term # update term - return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=True) # Reply false if term < currentTerm (§5.1) @@ -386,7 +385,6 @@ async def _serve_append_entries( self._node.id, ) return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) - except IndexError as e: except IndexError as e: _logger.info( c["append_entries"] @@ -702,7 +700,6 @@ def _change_state(self, new_state: RaftState) -> None: elif self._state == RaftState.LEADER: assert self._prev_state == RaftState.CANDIDATE, "Invalid state change 3" - # Cancel the election timeout (if it exists), and schedule a new term timeout. if hasattr(self, "_election_timer"): self._election_timer.cancel() @@ -783,7 +780,6 @@ async def _on_term_timeout(self) -> None: self._cluster[remote_node_index], ) - _logger.info( c["raft_logic"] + "Node ID: %d -- Value of next index = %d and value of remote next index = %s for node %s" From cc622cc21df8651ca3e60a5043e28dd692beee37 Mon Sep 17 00:00:00 2001 From: Nikita Date: Mon, 29 Jul 2024 10:36:37 +0000 Subject: [PATCH 47/48] add line with black --- .github/workflows/cyraft-test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cyraft-test.yml b/.github/workflows/cyraft-test.yml index 4c967cc..5c5cce8 100644 --- a/.github/workflows/cyraft-test.yml +++ b/.github/workflows/cyraft-test.yml @@ -41,7 +41,6 @@ jobs: # Install flake8, which is a Python linter that checks for syntax errors and coding standards pip install flake8 - # If a requirements.txt file exists in the repository, install the dependencies specified in it if [ -f requirements.txt ]; then pip install -r requirements.txt; fi @@ -89,7 +88,7 @@ jobs: - name: Lint with black run: | # Run black to format the code - black --line-length 120 . + black --line-length 120 . black . --check --diff --color --verbose && flake8 --select=E901,E999,F821,F822,F823 . # Test with pytest, to run unit tests and verify the application's functionality From bf86b628b2b6e93fb50e18b7212bbf4fa599fae5 Mon Sep 17 00:00:00 2001 From: Nikita Date: Mon, 29 Jul 2024 11:00:44 +0000 Subject: [PATCH 48/48] Added black formater to CI --- cyraft/node.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cyraft/node.py b/cyraft/node.py index 4393aee..8050d02 100644 --- a/cyraft/node.py +++ b/cyraft/node.py @@ -364,13 +364,10 @@ async def _serve_append_entries( _logger.info( c["append_entries"] + "Node ID: %d -- Append entries request denied (term (%d) < currentTerm (%d))" - + "Node ID: %d -- Append entries request denied (term (%d) < currentTerm (%d))" + c["end_color"], self._node.id, request.term, self._term, - request.term, - self._term, ) return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) @@ -387,10 +384,6 @@ async def _serve_append_entries( return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False) except IndexError as e: _logger.info( - c["append_entries"] - + "Node ID: %d -- Append entries request denied (log mismatch 2). IndexError: %s. " - + "prev_log_index: %d, log_length: %d" - + c["end_color"], c["append_entries"] + "Node ID: %d -- Append entries request denied (log mismatch 2). IndexError: %s. " + "prev_log_index: %d, log_length: %d" @@ -399,9 +392,6 @@ async def _serve_append_entries( str(e), request.prev_log_index, len(self._log), - str(e), - request.prev_log_index, - len(self._log), ) return sirius_cyber_corp.AppendEntries_1.Response(term=self._term, success=False)