From 231a3918c571a53d5b7d937bad91b6c20022ba8d Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Thu, 16 Jan 2025 19:08:22 +0800
Subject: [PATCH 01/30] test: add api endpoints - disperse_data - get_range

---
 src/node/api_clients/rest.py                |  6 ++++++
 tests/data_integrity/test_data_integrity.py | 10 +++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/node/api_clients/rest.py b/src/node/api_clients/rest.py
index a899d83..72a1ec2 100644
--- a/src/node/api_clients/rest.py
+++ b/src/node/api_clients/rest.py
@@ -23,3 +23,9 @@ def rest_call_text(self, method, endpoint, payload=None):
     def info(self):
         status_response = self.rest_call("get", "cryptarchia/info")
         return status_response.json()
+
+    def disperse_data(self, dispersal_request):
+        return self.rest_call("post", "disperse-data", json.dumps(dispersal_request))
+
+    def get_range(self, app_id, data_range):
+        return self.rest_call("post", "da/get-range", json.dumps({app_id, data_range}))
diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index 635559e..39b8056 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -1,6 +1,14 @@
 class TestDataIntegrity:
     main_nodes = []
 
-    def test_cluster_start(self):
+    def test_da_identify_retrieve_missing_columns(self):
         for node in self.main_nodes:
             print(node)
+
+    def test_da_sampling_determines_data_presence(self):
+        for node in self.main_nodes:
+            print(node)
+
+        # Disperse data
+        # Get data from range
+        # Compare

From a5a3f179afeaf3a4eaa80bcb194dddbedfdd788f Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Thu, 16 Jan 2025 19:52:36 +0800
Subject: [PATCH 02/30] test: setup main nodes fixture

---
 src/steps/common.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/steps/common.py b/src/steps/common.py
index 91fdea5..caf8fa9 100644
--- a/src/steps/common.py
+++ b/src/steps/common.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from src.env_vars import NODE_1, NODE_2
+from src.env_vars import NODE_1, NODE_2, CFGSYNC, NOMOS, NOMOS_EXECUTOR
 from src.libs.custom_logger import get_custom_logger
 from src.node.nomos_node import NomosNode
 
@@ -13,8 +13,17 @@ class StepsCommon:
     @pytest.fixture(scope="function")
     def setup_main_nodes(self, request):
         logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
-        self.node1 = NomosNode(NODE_1, f"node1_{request.cls.test_id}")
+        self.node1 = NomosNode(CFGSYNC, "cfgsync")
+        self.node2 = NomosNode(NOMOS, "nomos_node_0")
+        self.node3 = NomosNode(NOMOS_EXECUTOR, "nomos_node_1")
         self.node1.start()
-        self.node2 = NomosNode(NODE_2, f"node2_{request.cls.test_id}")
         self.node2.start()
-        self.main_nodes.extend([self.node1, self.node2])
+        self.node3.start()
+        self.main_nodes.extend([self.node1, self.node2, self.node3])
+
+        try:
+            self.node2.ensure_ready()
+            self.node3.ensure_ready()
+        except Exception as ex:
+            logger.error(f"REST service did not become ready in time: {ex}")
+            raise

From 2fc0cd197459461b1a0a1697c96736752701b7ae Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Fri, 17 Jan 2025 14:23:52 +0800
Subject: [PATCH 03/30] test: use setup_main_nodes fixture - add disperse_data

---
 src/node/api_clients/rest.py                |  6 +++---
 src/node/nomos_node.py                      |  3 +++
 src/steps/da.py                             | 19 ++++++++++++++++++
 src/test_data.py                            | 10 ++++++++++
 tests/data_integrity/test_data_integrity.py | 13 ++++++++----
 tests/e2e/test_2node_alive.py               | 22 ++++++---------------
 6 files changed, 50 insertions(+), 23 deletions(-)
 create mode 100644 src/steps/da.py

diff --git a/src/node/api_clients/rest.py b/src/node/api_clients/rest.py
index 72a1ec2..0b697ab 100644
--- a/src/node/api_clients/rest.py
+++ b/src/node/api_clients/rest.py
@@ -24,8 +24,8 @@ def info(self):
         status_response = self.rest_call("get", "cryptarchia/info")
         return status_response.json()
 
-    def disperse_data(self, dispersal_request):
-        return self.rest_call("post", "disperse-data", json.dumps(dispersal_request))
+    def send_dispersal_request(self, data):
+        return self.rest_call("post", "disperse-data", json.dumps(data))
 
     def get_range(self, app_id, data_range):
-        return self.rest_call("post", "da/get-range", json.dumps({app_id, data_range}))
+        return self.rest_call("post", "da/get-range", json.dumps({"app_id": app_id, "data_range": data_range}))
diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py
index cd3f079..bf74887 100644
--- a/src/node/nomos_node.py
+++ b/src/node/nomos_node.py
@@ -145,3 +145,6 @@ def check_nomos_log_errors(self, whitelist=None):
 
         matches = self._docker_manager.search_log_for_keywords(self._log_path, keywords, False)
         assert not matches, f"Found errors {matches}"
+
+    def send_dispersal_request(self, data):
+        return self._api.send_dispersal_request(data)
diff --git a/src/steps/da.py b/src/steps/da.py
new file mode 100644
index 0000000..a96ee21
--- /dev/null
+++ b/src/steps/da.py
@@ -0,0 +1,19 @@
+import allure
+
+from src.steps.common import StepsCommon
+
+
+def prepare_dispersal_data(data):
+    dispersal_data = {"data": data, "metadata": {"app_id": 10, "index": 0}}
+    return dispersal_data
+
+
+class StepsDataAvailability(StepsCommon):
+
+    @allure.step
+    def disperse_data(self, data):
+        dispersal_data = prepare_dispersal_data(data)
+        try:
+            self.node3.send_dispersal_request(dispersal_data)
+        except Exception as ex:
+            assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
diff --git a/src/test_data.py b/src/test_data.py
index e4d7665..4fd7cc3 100644
--- a/src/test_data.py
+++ b/src/test_data.py
@@ -24,3 +24,13 @@
     "race condition",
     "double free",
 ]
+
+DATA_TO_DISPERSE = {
+    "Hello World!",
+    "1234567890",
+    '{"key": "value"}',
+    "这是一些中文",
+    "🚀🌟✨",
+    "Lorem ipsum dolor sit amet",
+    "<html><body>Hello</body></html>",
+}
diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index 39b8056..6be91a1 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -1,4 +1,11 @@
-class TestDataIntegrity:
+import pytest
+
+from src.steps.da import StepsDataAvailability
+from src.test_data import DATA_TO_DISPERSE
+
+
+@pytest.mark.usefixtures("setup_main_nodes")
+class TestDataIntegrity(StepsDataAvailability):
     main_nodes = []
 
     def test_da_identify_retrieve_missing_columns(self):
@@ -6,9 +13,7 @@ def test_da_identify_retrieve_missing_columns(self):
             print(node)
 
     def test_da_sampling_determines_data_presence(self):
-        for node in self.main_nodes:
-            print(node)
+        self.disperse_data(DATA_TO_DISPERSE[0])
 
-        # Disperse data
         # Get data from range
         # Compare
diff --git a/tests/e2e/test_2node_alive.py b/tests/e2e/test_2node_alive.py
index 9553332..0a7b6f8 100644
--- a/tests/e2e/test_2node_alive.py
+++ b/tests/e2e/test_2node_alive.py
@@ -1,24 +1,14 @@
+import pytest
+
 from src.env_vars import CFGSYNC, NOMOS, NOMOS_EXECUTOR
 from src.libs.custom_logger import get_custom_logger
 from src.node.nomos_node import NomosNode
+from src.steps.common import StepsCommon
 
 logger = get_custom_logger(__name__)
 
 
-class Test2NodeClAlive:
+class Test2NodeClAlive(StepsCommon):
+    @pytest.mark.usefixtures("setup_main_nodes")
     def test_cluster_start(self):
-
-        self.node1 = NomosNode(CFGSYNC, "cfgsync")
-        self.node2 = NomosNode(NOMOS, "nomos_node_0")
-        self.node3 = NomosNode(NOMOS_EXECUTOR, "nomos_node_1")
-
-        self.node1.start()
-        self.node2.start()
-        self.node3.start()
-
-        try:
-            self.node2.ensure_ready()
-            self.node3.ensure_ready()
-        except Exception as ex:
-            logger.error(f"REST service did not become ready in time: {ex}")
-            raise
+        logger.debug("Two nodes cluster started successfully!")

From 0d32bcd61cb9b8f4cdf597185ede1eef379ad3ba Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Fri, 17 Jan 2025 14:50:10 +0800
Subject: [PATCH 04/30] test: add main_nodes as cluster setup

---
 src/steps/common.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/steps/common.py b/src/steps/common.py
index caf8fa9..2e52594 100644
--- a/src/steps/common.py
+++ b/src/steps/common.py
@@ -10,6 +10,11 @@
 
 
 class StepsCommon:
+    @pytest.fixture(scope="function", autouse=True)
+    def cluster_setup(self):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.main_nodes = []
+
     @pytest.fixture(scope="function")
     def setup_main_nodes(self, request):
         logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")

From b4244562254921553e6f267fae90ed827515256d Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Fri, 17 Jan 2025 14:56:41 +0800
Subject: [PATCH 05/30] fix: test data set to list

---
 src/test_data.py              | 4 ++--
 tests/e2e/test_2node_alive.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test_data.py b/src/test_data.py
index 4fd7cc3..31cdf31 100644
--- a/src/test_data.py
+++ b/src/test_data.py
@@ -25,7 +25,7 @@
     "double free",
 ]
 
-DATA_TO_DISPERSE = {
+DATA_TO_DISPERSE = [
     "Hello World!",
     "1234567890",
     '{"key": "value"}',
@@ -33,4 +33,4 @@
     "🚀🌟✨",
     "Lorem ipsum dolor sit amet",
     "<html><body>Hello</body></html>",
-}
+]
diff --git a/tests/e2e/test_2node_alive.py b/tests/e2e/test_2node_alive.py
index 0a7b6f8..81273b0 100644
--- a/tests/e2e/test_2node_alive.py
+++ b/tests/e2e/test_2node_alive.py
@@ -11,4 +11,4 @@
 class Test2NodeClAlive(StepsCommon):
     @pytest.mark.usefixtures("setup_main_nodes")
     def test_cluster_start(self):
-        logger.debug("Two nodes cluster started successfully!")
+        logger.debug("Two node cluster started successfully!")

From 0eac80fc579324bb13d9243b845df801ca565153 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Fri, 17 Jan 2025 15:04:02 +0800
Subject: [PATCH 06/30] fix: convert string into list of bytes

---
 src/steps/da.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/steps/da.py b/src/steps/da.py
index a96ee21..33a2ab7 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -4,7 +4,8 @@
 
 
 def prepare_dispersal_data(data):
-    dispersal_data = {"data": data, "metadata": {"app_id": 10, "index": 0}}
+    data_bytes = data.encode("utf-8")
+    dispersal_data = {"data": list(data_bytes), "metadata": {"app_id": 10, "index": 0}}
     return dispersal_data
 
 

From b83019c7f403ded3cf7d9e12259a9e31bcc3ab6d Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Fri, 17 Jan 2025 15:14:46 +0800
Subject: [PATCH 07/30] fix: app_id should be list of 32 integers

---
 src/steps/da.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/steps/da.py b/src/steps/da.py
index 33a2ab7..98e587f 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -5,7 +5,7 @@
 
 def prepare_dispersal_data(data):
     data_bytes = data.encode("utf-8")
-    dispersal_data = {"data": list(data_bytes), "metadata": {"app_id": 10, "index": 0}}
+    dispersal_data = {"data": list(data_bytes), "metadata": {"app_id": [1] + [0] * 31, "index": 0}}
     return dispersal_data
 
 

From 1025e36a1d12cd06bb82f3200d405e88c5daa800 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Fri, 17 Jan 2025 15:16:51 +0800
Subject: [PATCH 08/30] fix: index should be list of 8 integers

---
 src/steps/da.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/steps/da.py b/src/steps/da.py
index 98e587f..f3e6123 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -5,7 +5,7 @@
 
 def prepare_dispersal_data(data):
     data_bytes = data.encode("utf-8")
-    dispersal_data = {"data": list(data_bytes), "metadata": {"app_id": [1] + [0] * 31, "index": 0}}
+    dispersal_data = {"data": list(data_bytes), "metadata": {"app_id": [1] + [0] * 31, "index": [0] * 8}}
     return dispersal_data
 
 

From 53bb7906af01c5a0cc79cf9ec6cac759dd84472b Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Fri, 17 Jan 2025 19:27:43 +0800
Subject: [PATCH 09/30] test: get_data_range

---
 src/node/api_clients/rest.py                |  4 ++--
 src/node/nomos_node.py                      |  3 +++
 src/steps/da.py                             | 21 +++++++++++++++++----
 tests/data_integrity/test_data_integrity.py |  7 +++----
 4 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/src/node/api_clients/rest.py b/src/node/api_clients/rest.py
index 0b697ab..604a6ed 100644
--- a/src/node/api_clients/rest.py
+++ b/src/node/api_clients/rest.py
@@ -27,5 +27,5 @@ def info(self):
     def send_dispersal_request(self, data):
         return self.rest_call("post", "disperse-data", json.dumps(data))
 
-    def get_range(self, app_id, data_range):
-        return self.rest_call("post", "da/get-range", json.dumps({"app_id": app_id, "data_range": data_range}))
+    def send_get_range(self, query):
+        return self.rest_call("post", "da/get-range", json.dumps(query))
diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py
index bf74887..6a838bc 100644
--- a/src/node/nomos_node.py
+++ b/src/node/nomos_node.py
@@ -148,3 +148,6 @@ def check_nomos_log_errors(self, whitelist=None):
 
     def send_dispersal_request(self, data):
         return self._api.send_dispersal_request(data)
+
+    def send_get_data_range_request(self, data):
+        return self._api.send_get_range(data)
diff --git a/src/steps/da.py b/src/steps/da.py
index f3e6123..fe9ce18 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -3,18 +3,31 @@
 from src.steps.common import StepsCommon
 
 
-def prepare_dispersal_data(data):
+def prepare_dispersal_request(data, app_id, index):
     data_bytes = data.encode("utf-8")
     dispersal_data = {"data": list(data_bytes), "metadata": {"app_id": [1] + [0] * 31, "index": [0] * 8}}
     return dispersal_data
 
 
+def prepare_get_range_request(app_id, start_index, end_index):
+    query_data = {"app_id": app_id, "range": {"start": start_index, "end": end_index}}
+    return query_data
+
+
 class StepsDataAvailability(StepsCommon):
 
     @allure.step
-    def disperse_data(self, data):
-        dispersal_data = prepare_dispersal_data(data)
+    def disperse_data(self, data, app_id, index):
+        request = prepare_dispersal_request(data)
+        try:
+            self.node3.send_dispersal_request(request)
+        except Exception as ex:
+            assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
+
+    @allure.step
+    def get_data_range(self, app_id, start, end):
+        query = prepare_get_range_request(app_id, start, end)
         try:
-            self.node3.send_dispersal_request(dispersal_data)
+            self.node2.send_get_data_range_request(query)
         except Exception as ex:
             assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index 6be91a1..4c2c5f4 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -13,7 +13,6 @@ def test_da_identify_retrieve_missing_columns(self):
             print(node)
 
     def test_da_sampling_determines_data_presence(self):
-        self.disperse_data(DATA_TO_DISPERSE[0])
-
-        # Get data from range
-        # Compare
+        self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)
+        received_data = self.get_data_range([0] * 31 + [1], [0] * 8, [0] * 7 + [5])
+        assert DATA_TO_DISPERSE[0] == received_data

From a56dbfc34bf0dd63204d0f198f6fafc55287ed40 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Fri, 17 Jan 2025 19:49:53 +0800
Subject: [PATCH 10/30] fix: remove hardcoded values from disperse_dat

---
 src/steps/da.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/steps/da.py b/src/steps/da.py
index fe9ce18..f0922a9 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -5,7 +5,7 @@
 
 def prepare_dispersal_request(data, app_id, index):
     data_bytes = data.encode("utf-8")
-    dispersal_data = {"data": list(data_bytes), "metadata": {"app_id": [1] + [0] * 31, "index": [0] * 8}}
+    dispersal_data = {"data": list(data_bytes), "metadata": {"app_id": app_id, "index": index}}
     return dispersal_data
 
 
@@ -18,7 +18,7 @@ class StepsDataAvailability(StepsCommon):
 
     @allure.step
     def disperse_data(self, data, app_id, index):
-        request = prepare_dispersal_request(data)
+        request = prepare_dispersal_request(data, app_id, index)
         try:
             self.node3.send_dispersal_request(request)
         except Exception as ex:

From b028574a8c1b88717610400bb27e66a9daf87954 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Mon, 20 Jan 2025 13:48:12 +0800
Subject: [PATCH 11/30] fix: add data extraction and decoding

---
 src/steps/da.py                             | 10 +++++++++-
 tests/data_integrity/test_data_integrity.py |  2 +-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/steps/da.py b/src/steps/da.py
index f0922a9..c749e5f 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -26,8 +26,16 @@ def disperse_data(self, data, app_id, index):
 
     @allure.step
     def get_data_range(self, app_id, start, end):
+        response = []
         query = prepare_get_range_request(app_id, start, end)
         try:
-            self.node2.send_get_data_range_request(query)
+            response = self.node2.send_get_data_range_request(query)
         except Exception as ex:
             assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
+
+        # Extract data for each index in received order
+        extracted_data = []
+        for item in response:
+            extracted_data.append(item[1])
+
+        return extracted_data
diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index 4c2c5f4..a645e34 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -15,4 +15,4 @@ def test_da_identify_retrieve_missing_columns(self):
     def test_da_sampling_determines_data_presence(self):
         self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)
         received_data = self.get_data_range([0] * 31 + [1], [0] * 8, [0] * 7 + [5])
-        assert DATA_TO_DISPERSE[0] == received_data
+        assert DATA_TO_DISPERSE[0] == received_data[0].decode("utf-8")

From 578efed1505ac70ba67e0acd40a1aa503689c5e5 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Mon, 20 Jan 2025 14:02:18 +0800
Subject: [PATCH 12/30] fix: decode data bytes

---
 src/steps/da.py                             | 16 ++++++++--------
 tests/data_integrity/test_data_integrity.py |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/steps/da.py b/src/steps/da.py
index c749e5f..f999eb0 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -1,3 +1,5 @@
+import json
+
 import allure
 
 from src.steps.common import StepsCommon
@@ -26,16 +28,14 @@ def disperse_data(self, data, app_id, index):
 
     @allure.step
     def get_data_range(self, app_id, start, end):
-        response = []
+        response_bytes = []
         query = prepare_get_range_request(app_id, start, end)
         try:
-            response = self.node2.send_get_data_range_request(query)
+            response_bytes = self.node2.send_get_data_range_request(query)
         except Exception as ex:
             assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
 
-        # Extract data for each index in received order
-        extracted_data = []
-        for item in response:
-            extracted_data.append(item[1])
-
-        return extracted_data
+        # Extract data ss string for each index in the received order
+        response = response_bytes.decode("utf-8")
+        parsed_data = json.loads(response)
+        return [item[1] for item in parsed_data]
diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index a645e34..a993ad6 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -15,4 +15,4 @@ def test_da_identify_retrieve_missing_columns(self):
     def test_da_sampling_determines_data_presence(self):
         self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)
         received_data = self.get_data_range([0] * 31 + [1], [0] * 8, [0] * 7 + [5])
-        assert DATA_TO_DISPERSE[0] == received_data[0].decode("utf-8")
+        assert DATA_TO_DISPERSE[0] == received_data[0]

From 536864e1685a4e8bc52635f4a35baf417d6da0a9 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Mon, 20 Jan 2025 14:14:59 +0800
Subject: [PATCH 13/30] fix: access response.content

---
 src/steps/da.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/steps/da.py b/src/steps/da.py
index f999eb0..50e58a6 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -35,7 +35,8 @@ def get_data_range(self, app_id, start, end):
         except Exception as ex:
             assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
 
-        # Extract data ss string for each index in the received order
-        response = response_bytes.decode("utf-8")
-        parsed_data = json.loads(response)
-        return [item[1] for item in parsed_data]
+        # Extract data as a string for each index in the received order
+        parsed_data = []
+        for item in response_bytes.content:
+            parsed_data.append(item.decode("utf-8"))
+        return parsed_data

From c5f82e6176190c47d247c982d98150901daebfbc Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Mon, 20 Jan 2025 14:23:59 +0800
Subject: [PATCH 14/30] test: debug received data

---
 src/steps/da.py                             | 2 +-
 tests/data_integrity/test_data_integrity.py | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/steps/da.py b/src/steps/da.py
index 50e58a6..72ff684 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -38,5 +38,5 @@ def get_data_range(self, app_id, start, end):
         # Extract data as a string for each index in the received order
         parsed_data = []
         for item in response_bytes.content:
-            parsed_data.append(item.decode("utf-8"))
+            parsed_data.append(item)
         return parsed_data
diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index a993ad6..7b765d5 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -1,8 +1,11 @@
 import pytest
 
+from src.libs.custom_logger import get_custom_logger
 from src.steps.da import StepsDataAvailability
 from src.test_data import DATA_TO_DISPERSE
 
+logger = get_custom_logger(__name__)
+
 
 @pytest.mark.usefixtures("setup_main_nodes")
 class TestDataIntegrity(StepsDataAvailability):
@@ -15,4 +18,5 @@ def test_da_identify_retrieve_missing_columns(self):
     def test_da_sampling_determines_data_presence(self):
         self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)
         received_data = self.get_data_range([0] * 31 + [1], [0] * 8, [0] * 7 + [5])
+        logger.debug(f"received data {received_data}")
         assert DATA_TO_DISPERSE[0] == received_data[0]

From 01b6025afb5e9b4294c4f3b97588cfe9532c3c07 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Mon, 20 Jan 2025 14:32:19 +0800
Subject: [PATCH 15/30] test: use json response

---
 src/node/api_clients/rest.py | 3 ++-
 src/steps/da.py              | 8 +++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/node/api_clients/rest.py b/src/node/api_clients/rest.py
index 604a6ed..6993e30 100644
--- a/src/node/api_clients/rest.py
+++ b/src/node/api_clients/rest.py
@@ -28,4 +28,5 @@ def send_dispersal_request(self, data):
         return self.rest_call("post", "disperse-data", json.dumps(data))
 
     def send_get_range(self, query):
-        return self.rest_call("post", "da/get-range", json.dumps(query))
+        response = self.rest_call("post", "da/get-range", json.dumps(query))
+        return response.json()
diff --git a/src/steps/da.py b/src/steps/da.py
index 72ff684..d549fc0 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -28,15 +28,13 @@ def disperse_data(self, data, app_id, index):
 
     @allure.step
     def get_data_range(self, app_id, start, end):
-        response_bytes = []
+        response = []
         query = prepare_get_range_request(app_id, start, end)
         try:
-            response_bytes = self.node2.send_get_data_range_request(query)
+            response = self.node2.send_get_data_range_request(query)
         except Exception as ex:
             assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
 
         # Extract data as a string for each index in the received order
-        parsed_data = []
-        for item in response_bytes.content:
-            parsed_data.append(item)
+        parsed_data = json.loads(response)
         return parsed_data

From be897bf95e4727cd2ec2eccc798d07f36b7082d8 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Mon, 20 Jan 2025 14:34:48 +0800
Subject: [PATCH 16/30] test: show response

---
 src/steps/da.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/steps/da.py b/src/steps/da.py
index d549fc0..81564a9 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -36,5 +36,5 @@ def get_data_range(self, app_id, start, end):
             assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
 
         # Extract data as a string for each index in the received order
-        parsed_data = json.loads(response)
-        return parsed_data
+        # parsed_data = json.loads(response)
+        return response

From 77ce25cb0b9b68eab1b4389f2b1b5cedffc178ae Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Mon, 20 Jan 2025 14:37:43 +0800
Subject: [PATCH 17/30] test: let the test decode data

---
 src/steps/da.py                             | 2 --
 tests/data_integrity/test_data_integrity.py | 3 +--
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/steps/da.py b/src/steps/da.py
index 81564a9..d73aa20 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -35,6 +35,4 @@ def get_data_range(self, app_id, start, end):
         except Exception as ex:
             assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
 
-        # Extract data as a string for each index in the received order
-        # parsed_data = json.loads(response)
         return response
diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index 7b765d5..86671fa 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -18,5 +18,4 @@ def test_da_identify_retrieve_missing_columns(self):
     def test_da_sampling_determines_data_presence(self):
         self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)
         received_data = self.get_data_range([0] * 31 + [1], [0] * 8, [0] * 7 + [5])
-        logger.debug(f"received data {received_data}")
-        assert DATA_TO_DISPERSE[0] == received_data[0]
+        assert DATA_TO_DISPERSE[0] == received_data[0][1].decode("utf-8")

From 7cdcc0d084ce727a4d300bd90dd1684e8cc3dc8e Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Mon, 20 Jan 2025 14:50:26 +0800
Subject: [PATCH 18/30] test: type as bytes before decoding

---
 tests/data_integrity/test_data_integrity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index 86671fa..beac499 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -18,4 +18,4 @@ def test_da_identify_retrieve_missing_columns(self):
     def test_da_sampling_determines_data_presence(self):
         self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)
         received_data = self.get_data_range([0] * 31 + [1], [0] * 8, [0] * 7 + [5])
-        assert DATA_TO_DISPERSE[0] == received_data[0][1].decode("utf-8")
+        assert DATA_TO_DISPERSE[0] == bytes(received_data[0][1]).decode("utf-8")

From d021e52efd681515cff5970945d3ce77b424a9f1 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Tue, 21 Jan 2025 14:55:26 +0800
Subject: [PATCH 19/30] fix: main nodes setup to 2 node cl setup

---
 src/steps/common.py                         | 2 +-
 tests/data_integrity/test_data_integrity.py | 5 ++---
 tests/e2e/test_2node_alive.py               | 2 +-
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/steps/common.py b/src/steps/common.py
index 2e52594..9b5e204 100644
--- a/src/steps/common.py
+++ b/src/steps/common.py
@@ -16,7 +16,7 @@ def cluster_setup(self):
         self.main_nodes = []
 
     @pytest.fixture(scope="function")
-    def setup_main_nodes(self, request):
+    def setup_2_node_cluster(self, request):
         logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
         self.node1 = NomosNode(CFGSYNC, "cfgsync")
         self.node2 = NomosNode(NOMOS, "nomos_node_0")
diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index beac499..b1922a7 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -7,14 +7,13 @@
 logger = get_custom_logger(__name__)
 
 
-@pytest.mark.usefixtures("setup_main_nodes")
 class TestDataIntegrity(StepsDataAvailability):
     main_nodes = []
 
     def test_da_identify_retrieve_missing_columns(self):
-        for node in self.main_nodes:
-            print(node)
+        self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)
 
+    @pytest.mark.usefixtures("setup_2_node_cluster")
     def test_da_sampling_determines_data_presence(self):
         self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)
         received_data = self.get_data_range([0] * 31 + [1], [0] * 8, [0] * 7 + [5])
diff --git a/tests/e2e/test_2node_alive.py b/tests/e2e/test_2node_alive.py
index 81273b0..fbf87d9 100644
--- a/tests/e2e/test_2node_alive.py
+++ b/tests/e2e/test_2node_alive.py
@@ -9,6 +9,6 @@
 
 
 class Test2NodeClAlive(StepsCommon):
-    @pytest.mark.usefixtures("setup_main_nodes")
+    @pytest.mark.usefixtures("setup_2_node_cluster")
     def test_cluster_start(self):
         logger.debug("Two node cluster started successfully!")

From c6a8b44353adec4d45bd00e231e689c9db730d10 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Tue, 21 Jan 2025 15:30:24 +0800
Subject: [PATCH 20/30] fix: add 5 node cluster config

---
 cluster_config/cfgsync-2node.yaml | 31 +++++++++++++++++++++++++++++++
 cluster_config/cfgsync-5node.yaml | 31 +++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 cluster_config/cfgsync-2node.yaml
 create mode 100644 cluster_config/cfgsync-5node.yaml

diff --git a/cluster_config/cfgsync-2node.yaml b/cluster_config/cfgsync-2node.yaml
new file mode 100644
index 0000000..10840a5
--- /dev/null
+++ b/cluster_config/cfgsync-2node.yaml
@@ -0,0 +1,31 @@
+port: 4400
+n_hosts: 2
+timeout: 30
+
+# ConsensusConfig related parameters
+security_param: 10
+active_slot_coeff: 0.9
+
+# DaConfig related parameters
+subnetwork_size: 2
+dispersal_factor: 2
+num_samples: 1
+num_subnets: 2
+old_blobs_check_interval_secs: 5
+blobs_validity_duration_secs: 60
+global_params_path: "/kzgrs_test_params"
+
+# Tracing
+tracing_settings:
+  logger: Stdout
+  tracing: !Otlp
+    endpoint: http://tempo:4317/
+    sample_ratio: 0.5
+    service_name: node
+  filter: !EnvFilter
+    filters:
+      nomos: debug
+  metrics: !Otlp
+    endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
+    host_identifier: node
+  level: INFO
\ No newline at end of file
diff --git a/cluster_config/cfgsync-5node.yaml b/cluster_config/cfgsync-5node.yaml
new file mode 100644
index 0000000..dde1f71
--- /dev/null
+++ b/cluster_config/cfgsync-5node.yaml
@@ -0,0 +1,31 @@
+port: 4400
+n_hosts: 5
+timeout: 30
+
+# ConsensusConfig related parameters
+security_param: 10
+active_slot_coeff: 0.9
+
+# DaConfig related parameters
+subnetwork_size: 2
+dispersal_factor: 2
+num_samples: 1
+num_subnets: 2
+old_blobs_check_interval_secs: 5
+blobs_validity_duration_secs: 60
+global_params_path: "/kzgrs_test_params"
+
+# Tracing
+tracing_settings:
+  logger: Stdout
+  tracing: !Otlp
+    endpoint: http://tempo:4317/
+    sample_ratio: 0.5
+    service_name: node
+  filter: !EnvFilter
+    filters:
+      nomos: debug
+  metrics: !Otlp
+    endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
+    host_identifier: node
+  level: INFO
\ No newline at end of file

From 696a6b2b6630a4a2efd720180facbd035b553381 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Tue, 21 Jan 2025 15:39:45 +0800
Subject: [PATCH 21/30] test: default config for 5 nodes temporarily

---
 cluster_config/cfgsync.yaml                 |  2 +-
 src/steps/common.py                         | 27 +++++++++++++++++++++
 tests/data_integrity/test_data_integrity.py |  1 +
 3 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/cluster_config/cfgsync.yaml b/cluster_config/cfgsync.yaml
index 10840a5..dde1f71 100644
--- a/cluster_config/cfgsync.yaml
+++ b/cluster_config/cfgsync.yaml
@@ -1,5 +1,5 @@
 port: 4400
-n_hosts: 2
+n_hosts: 5
 timeout: 30
 
 # ConsensusConfig related parameters
diff --git a/src/steps/common.py b/src/steps/common.py
index 9b5e204..78a905b 100644
--- a/src/steps/common.py
+++ b/src/steps/common.py
@@ -32,3 +32,30 @@ def setup_2_node_cluster(self, request):
         except Exception as ex:
             logger.error(f"REST service did not become ready in time: {ex}")
             raise
+
+    @pytest.fixture(scope="function")
+    def setup_5_node_cluster(self, request):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.node1 = NomosNode(CFGSYNC, "cfgsync")
+        self.node2 = NomosNode(NOMOS, "nomos_node_0")
+        self.node3 = NomosNode(NOMOS, "nomos_node_1")
+        self.node4 = NomosNode(NOMOS, "nomos_node_2")
+        self.node5 = NomosNode(NOMOS, "nomos_node_3")
+        self.node6 = NomosNode(NOMOS_EXECUTOR, "nomos_node_4")
+        self.node1.start()
+        self.node2.start()
+        self.node3.start()
+        self.node4.start()
+        self.node5.start()
+        self.node6.start()
+        self.main_nodes.extend([self.node1, self.node2, self.node3, self.node4, self.node5, self.node6])
+
+        try:
+            self.node2.ensure_ready()
+            self.node3.ensure_ready()
+            self.node4.ensure_ready()
+            self.node5.ensure_ready()
+            self.node6.ensure_ready()
+        except Exception as ex:
+            logger.error(f"REST service did not become ready in time: {ex}")
+            raise
diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index b1922a7..f209110 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -10,6 +10,7 @@
 class TestDataIntegrity(StepsDataAvailability):
     main_nodes = []
 
+    @pytest.mark.usefixtures("setup_5_node_cluster")
     def test_da_identify_retrieve_missing_columns(self):
         self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)
 

From ca1686a9fb882397e66335f3692c1e0ee487ac0c Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Tue, 21 Jan 2025 16:33:47 +0800
Subject: [PATCH 22/30] test: more robust volume initialization

---
 src/node/nomos_node.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py
index 6a838bc..a328efa 100644
--- a/src/node/nomos_node.py
+++ b/src/node/nomos_node.py
@@ -36,9 +36,12 @@ def __init__(self, node_type, container_name=""):
         self._container = None
 
         cwd = os.getcwd()
+        updated_volumes = []
         for i, volume in enumerate(self._volumes):
-            self._volumes[i] = cwd + "/" + volume
+            updated_volumes.append(cwd + "/" + volume)
+        self._volumes = updated_volumes
 
+        logger.debug(f"NomosNode instance initialized with volumes {self._volumes}")
         logger.debug(f"NomosNode instance initialized with log path {self._log_path}")
 
     @retry(stop=stop_after_delay(60), wait=wait_fixed(0.1), reraise=True)

From 56a4f6ba5c37a418fb41aef9bdd95f5ea5c75112 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Tue, 21 Jan 2025 16:57:23 +0800
Subject: [PATCH 23/30] fix: find executor node - use oneliner for volumes

---
 src/node/nomos_node.py | 10 +++++-----
 src/steps/da.py        | 11 ++++++++++-
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py
index a328efa..d44cdef 100644
--- a/src/node/nomos_node.py
+++ b/src/node/nomos_node.py
@@ -29,6 +29,7 @@ def __init__(self, node_type, container_name=""):
         self._internal_ports = nomos_nodes[node_type]["ports"]
         self._volumes = nomos_nodes[node_type]["volumes"]
         self._entrypoint = nomos_nodes[node_type]["entrypoint"]
+        self._node_type = node_type
 
         self._log_path = os.path.join(DOCKER_LOG_DIR, f"{container_name}__{self._image_name.replace('/', '_')}.log")
         self._docker_manager = DockerManager(self._image_name)
@@ -36,12 +37,8 @@ def __init__(self, node_type, container_name=""):
         self._container = None
 
         cwd = os.getcwd()
-        updated_volumes = []
-        for i, volume in enumerate(self._volumes):
-            updated_volumes.append(cwd + "/" + volume)
-        self._volumes = updated_volumes
+        self._volumes = [cwd + "/" + volume for volume in self._volumes]
 
-        logger.debug(f"NomosNode instance initialized with volumes {self._volumes}")
         logger.debug(f"NomosNode instance initialized with log path {self._log_path}")
 
     @retry(stop=stop_after_delay(60), wait=wait_fixed(0.1), reraise=True)
@@ -139,6 +136,9 @@ def is_nomos(self):
     def info(self):
         return self._api.info()
 
+    def node_type(self):
+        return self._node_type
+
     def check_nomos_log_errors(self, whitelist=None):
         keywords = LOG_ERROR_KEYWORDS
 
diff --git a/src/steps/da.py b/src/steps/da.py
index d73aa20..d780686 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -2,6 +2,7 @@
 
 import allure
 
+from src.env_vars import NOMOS_EXECUTOR
 from src.steps.common import StepsCommon
 
 
@@ -18,11 +19,19 @@ def prepare_get_range_request(app_id, start_index, end_index):
 
 class StepsDataAvailability(StepsCommon):
 
+    def find_executor_node(self):
+        executor = {}
+        for node in self.main_nodes:
+            if node.node_type() == NOMOS_EXECUTOR:
+                executor = node
+        return executor
+
     @allure.step
     def disperse_data(self, data, app_id, index):
         request = prepare_dispersal_request(data, app_id, index)
+        executor = self.find_executor_node()
         try:
-            self.node3.send_dispersal_request(request)
+            executor.send_dispersal_request(request)
         except Exception as ex:
             assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
 

From 4bc7c74d911943a9026205d6b735a0f61e60a400 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Wed, 22 Jan 2025 14:47:33 +0800
Subject: [PATCH 24/30] test: retrieve data from half of nodes -

---
 src/steps/da.py                             | 4 ++--
 tests/data_integrity/test_data_integrity.py | 9 +++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/steps/da.py b/src/steps/da.py
index d780686..9b2e91e 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -36,11 +36,11 @@ def disperse_data(self, data, app_id, index):
             assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
 
     @allure.step
-    def get_data_range(self, app_id, start, end):
+    def get_data_range(self, node, app_id, start, end):
         response = []
         query = prepare_get_range_request(app_id, start, end)
         try:
-            response = self.node2.send_get_data_range_request(query)
+            response = node.send_get_data_range_request(query)
         except Exception as ex:
             assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
 
diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index f209110..738f035 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -13,6 +13,15 @@ class TestDataIntegrity(StepsDataAvailability):
     @pytest.mark.usefixtures("setup_5_node_cluster")
     def test_da_identify_retrieve_missing_columns(self):
         self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)
+        received_data = []
+        # Get data only from half of nodes
+        for node in self.main_nodes[2:4]:
+            received_data.append(self.get_data_range(node, [0] * 31 + [1], [0] * 8, [0] * 7 + [3]))
+
+        # Use received blob data to reconstruct the original data
+        # nomos-cli reconstruct command required
+        reconstructed_data = []
+        assert DATA_TO_DISPERSE[0] == bytes(reconstructed_data).decode("utf-8")
 
     @pytest.mark.usefixtures("setup_2_node_cluster")
     def test_da_sampling_determines_data_presence(self):

From 82abae7a2d8297530209a2d06d3d3f6f56b50a5d Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Wed, 22 Jan 2025 14:52:08 +0800
Subject: [PATCH 25/30] fix: retrieve data from second node -
 test_da_sampling_determines_data_presence

---
 tests/data_integrity/test_data_integrity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index 738f035..e849bcf 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -26,5 +26,5 @@ def test_da_identify_retrieve_missing_columns(self):
     @pytest.mark.usefixtures("setup_2_node_cluster")
     def test_da_sampling_determines_data_presence(self):
         self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)
-        received_data = self.get_data_range([0] * 31 + [1], [0] * 8, [0] * 7 + [5])
+        received_data = self.get_data_range(self.node2, [0] * 31 + [1], [0] * 8, [0] * 7 + [5])
         assert DATA_TO_DISPERSE[0] == bytes(received_data[0][1]).decode("utf-8")

From 1b0731f2ad3f71db605d2fb562e39cc518e5205e Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Wed, 22 Jan 2025 15:19:18 +0800
Subject: [PATCH 26/30] fix: copy appropriate config for desired cluster

---
 src/steps/common.py | 14 +++++++++++++-
 src/steps/da.py     |  2 --
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/steps/common.py b/src/steps/common.py
index 78a905b..fd59c10 100644
--- a/src/steps/common.py
+++ b/src/steps/common.py
@@ -1,14 +1,24 @@
 import inspect
+import os
+import shutil
 
 import pytest
 
-from src.env_vars import NODE_1, NODE_2, CFGSYNC, NOMOS, NOMOS_EXECUTOR
+from src.env_vars import CFGSYNC, NOMOS, NOMOS_EXECUTOR
 from src.libs.custom_logger import get_custom_logger
 from src.node.nomos_node import NomosNode
 
 logger = get_custom_logger(__name__)
 
 
+def prepare_cluster_config(node_count):
+    cwd = os.getcwd()
+    config_dir = "cluster_config"
+    src = f"{cwd}/{config_dir}/cfgsync-{node_count}node.yaml"
+    dst = f"{cwd}/{config_dir}/cfgsync.yaml"
+    shutil.copyfile(src, dst)
+
+
 class StepsCommon:
     @pytest.fixture(scope="function", autouse=True)
     def cluster_setup(self):
@@ -18,6 +28,7 @@ def cluster_setup(self):
     @pytest.fixture(scope="function")
     def setup_2_node_cluster(self, request):
         logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        prepare_cluster_config(2)
         self.node1 = NomosNode(CFGSYNC, "cfgsync")
         self.node2 = NomosNode(NOMOS, "nomos_node_0")
         self.node3 = NomosNode(NOMOS_EXECUTOR, "nomos_node_1")
@@ -36,6 +47,7 @@ def setup_2_node_cluster(self, request):
     @pytest.fixture(scope="function")
     def setup_5_node_cluster(self, request):
         logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        prepare_cluster_config(5)
         self.node1 = NomosNode(CFGSYNC, "cfgsync")
         self.node2 = NomosNode(NOMOS, "nomos_node_0")
         self.node3 = NomosNode(NOMOS, "nomos_node_1")
diff --git a/src/steps/da.py b/src/steps/da.py
index 9b2e91e..bbf1379 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -1,5 +1,3 @@
-import json
-
 import allure
 
 from src.env_vars import NOMOS_EXECUTOR

From 5f68514156f4442ebd10e6ff7b8cf77477caa843 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Wed, 22 Jan 2025 15:40:38 +0800
Subject: [PATCH 27/30] fix: optimize setup fixtures

---
 src/steps/common.py | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/src/steps/common.py b/src/steps/common.py
index fd59c10..90e734f 100644
--- a/src/steps/common.py
+++ b/src/steps/common.py
@@ -19,6 +19,16 @@ def prepare_cluster_config(node_count):
     shutil.copyfile(src, dst)
 
 
+def start_nodes(nodes):
+    for node in nodes:
+        node.start()
+
+
+def ensure_nodes_ready(nodes):
+    for node in nodes:
+        node.ensure_ready()
+
+
 class StepsCommon:
     @pytest.fixture(scope="function", autouse=True)
     def cluster_setup(self):
@@ -32,14 +42,11 @@ def setup_2_node_cluster(self, request):
         self.node1 = NomosNode(CFGSYNC, "cfgsync")
         self.node2 = NomosNode(NOMOS, "nomos_node_0")
         self.node3 = NomosNode(NOMOS_EXECUTOR, "nomos_node_1")
-        self.node1.start()
-        self.node2.start()
-        self.node3.start()
         self.main_nodes.extend([self.node1, self.node2, self.node3])
+        start_nodes(self.main_nodes)
 
         try:
-            self.node2.ensure_ready()
-            self.node3.ensure_ready()
+            ensure_nodes_ready(self.main_nodes[2:])
         except Exception as ex:
             logger.error(f"REST service did not become ready in time: {ex}")
             raise
@@ -54,20 +61,11 @@ def setup_5_node_cluster(self, request):
         self.node4 = NomosNode(NOMOS, "nomos_node_2")
         self.node5 = NomosNode(NOMOS, "nomos_node_3")
         self.node6 = NomosNode(NOMOS_EXECUTOR, "nomos_node_4")
-        self.node1.start()
-        self.node2.start()
-        self.node3.start()
-        self.node4.start()
-        self.node5.start()
-        self.node6.start()
         self.main_nodes.extend([self.node1, self.node2, self.node3, self.node4, self.node5, self.node6])
+        start_nodes(self.main_nodes)
 
         try:
-            self.node2.ensure_ready()
-            self.node3.ensure_ready()
-            self.node4.ensure_ready()
-            self.node5.ensure_ready()
-            self.node6.ensure_ready()
+            ensure_nodes_ready(self.main_nodes[2:])
         except Exception as ex:
             logger.error(f"REST service did not become ready in time: {ex}")
             raise

From 0c91fe7e130527353f2cbd2ff1c4d443d0d6c4be Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Thu, 23 Jan 2025 11:40:36 +0800
Subject: [PATCH 28/30] fix: add padding for dispersal data

---
 src/steps/da.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/steps/da.py b/src/steps/da.py
index bbf1379..c534e75 100644
--- a/src/steps/da.py
+++ b/src/steps/da.py
@@ -4,9 +4,29 @@
 from src.steps.common import StepsCommon
 
 
+def add_padding(orig_bytes):
+    block_size = 31
+    """
+    Pads a list of bytes (integers in [0..255]) using a PKCS#7-like scheme:
+    - The value of each padded byte is the number of bytes padded.
+    - If the original data is already a multiple of the block size,
+      an additional full block of bytes (each the block size) is added.
+    """
+    original_len = len(orig_bytes)
+    padding_needed = block_size - (original_len % block_size)
+    # If the data is already a multiple of block_size, add a full block of padding
+    if padding_needed == 0:
+        padding_needed = block_size
+
+    # Each padded byte will be equal to padding_needed
+    padded_bytes = orig_bytes + [padding_needed] * padding_needed
+    return padded_bytes
+
+
 def prepare_dispersal_request(data, app_id, index):
     data_bytes = data.encode("utf-8")
-    dispersal_data = {"data": list(data_bytes), "metadata": {"app_id": app_id, "index": index}}
+    padded_bytes = add_padding(list(data_bytes))
+    dispersal_data = {"data": padded_bytes, "metadata": {"app_id": app_id, "index": index}}
     return dispersal_data
 
 

From 2738c9e9f37e1bf6e616cf45b4668de31c4785b1 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Mon, 27 Jan 2025 15:16:53 +0800
Subject: [PATCH 29/30] fix: remove unused logger

---
 tests/data_integrity/test_data_integrity.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index e849bcf..80eba22 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -1,11 +1,8 @@
 import pytest
 
-from src.libs.custom_logger import get_custom_logger
 from src.steps.da import StepsDataAvailability
 from src.test_data import DATA_TO_DISPERSE
 
-logger = get_custom_logger(__name__)
-
 
 class TestDataIntegrity(StepsDataAvailability):
     main_nodes = []

From b4d32df1db99c62255e30cbb8d27db968f08a854 Mon Sep 17 00:00:00 2001
From: Roman <zajic@zajic.net>
Date: Mon, 27 Jan 2025 18:59:08 +0800
Subject: [PATCH 30/30] fix: skip tests pending on Nomos node changes

---
 tests/data_integrity/test_data_integrity.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/data_integrity/test_data_integrity.py b/tests/data_integrity/test_data_integrity.py
index 80eba22..6afc39a 100644
--- a/tests/data_integrity/test_data_integrity.py
+++ b/tests/data_integrity/test_data_integrity.py
@@ -7,6 +7,7 @@
 class TestDataIntegrity(StepsDataAvailability):
     main_nodes = []
 
+    @pytest.mark.skip(reason="Waiting for PR https://github.com/logos-co/nomos-node/pull/994")
     @pytest.mark.usefixtures("setup_5_node_cluster")
     def test_da_identify_retrieve_missing_columns(self):
         self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)
@@ -20,6 +21,7 @@ def test_da_identify_retrieve_missing_columns(self):
         reconstructed_data = []
         assert DATA_TO_DISPERSE[0] == bytes(reconstructed_data).decode("utf-8")
 
+    @pytest.mark.skip(reason="Waiting for Nomos testnet images could evolve blockchain")
     @pytest.mark.usefixtures("setup_2_node_cluster")
     def test_da_sampling_determines_data_presence(self):
         self.disperse_data(DATA_TO_DISPERSE[0], [0] * 31 + [1], [0] * 8)