From 675cebf240fc84485db25f075759591a9a6a68f9 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Tue, 2 Jul 2024 15:31:28 +0000
Subject: [PATCH 01/42] Handle json parsing errors in rpc check

---
 core/schains/rpc.py               |  25 +++++--
 tests/schains/monitor/rpc_test.py | 105 +++++++++++++++++-------------
 2 files changed, 78 insertions(+), 52 deletions(-)

diff --git a/core/schains/rpc.py b/core/schains/rpc.py
index 10519e14b..a9279e265 100644
--- a/core/schains/rpc.py
+++ b/core/schains/rpc.py
@@ -17,6 +17,8 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import json
+import logging
 import time
 
 from tools.configs import ALLOWED_TIMESTAMP_DIFF
@@ -24,6 +26,9 @@
 from tools.helper import post_request
 
 
+logger = logging.getLogger(__name__)
+
+
 def make_rpc_call(http_endpoint, method, params=None, timeout=None) -> bool:
     params = params or []
     return post_request(
@@ -47,10 +52,16 @@ def check_endpoint_alive(http_endpoint, timeout=None):
 
 def check_endpoint_blocks(http_endpoint):
     res = make_rpc_call(http_endpoint, 'eth_getBlockByNumber', ['latest', False])
-    if res and res.json():
-        res_data = res.json()
-        latest_schain_timestamp_hex = res_data['result']['timestamp']
-        latest_schain_timestamp = int(latest_schain_timestamp_hex, 16)
-        admin_timestamp = int(time.time())
-        return abs(latest_schain_timestamp - admin_timestamp) < ALLOWED_TIMESTAMP_DIFF
-    return False
+    healthy = False
+    if res:
+        try:
+            res_data = res.json()
+            latest_schain_timestamp_hex = res_data['result']['timestamp']
+            latest_schain_timestamp = int(latest_schain_timestamp_hex, 16)
+            admin_timestamp = int(time.time())
+            healthy = abs(latest_schain_timestamp - admin_timestamp) < ALLOWED_TIMESTAMP_DIFF
+        except (json.JSONDecodeError, KeyError, ValueError) as e:
+            logger.warning('Failed to parse response, error: %s', e)
+    else:
+        logger.warning('Empty response from skaled')
+    return healthy
diff --git a/tests/schains/monitor/rpc_test.py b/tests/schains/monitor/rpc_test.py
index 702707b20..5445ef887 100644
--- a/tests/schains/monitor/rpc_test.py
+++ b/tests/schains/monitor/rpc_test.py
@@ -1,11 +1,20 @@
+import datetime
+import json
+import mock
 from time import sleep
 
+import freezegun
+import requests
+
 from core.schains.monitor.rpc import handle_failed_schain_rpc
 from core.schains.runner import get_container_info
+from core.schains.rpc import check_endpoint_blocks
 from tools.configs.containers import SCHAIN_CONTAINER
-
 from web.models.schain import SChainRecord
 
+CURRENT_TIMESTAMP = 1594903080
+CURRENT_DATETIME = datetime.datetime.utcfromtimestamp(CURRENT_TIMESTAMP)
+
 
 def test_handle_failed_schain_rpc_no_container(schain_db, dutils, skaled_status):
     schain_record = SChainRecord.get_by_name(schain_db)
@@ -15,21 +24,17 @@ def test_handle_failed_schain_rpc_no_container(schain_db, dutils, skaled_status)
         schain={'name': schain_db},
         schain_record=schain_record,
         skaled_status=skaled_status,
-        dutils=dutils
+        dutils=dutils,
     )
     assert not dutils.is_container_exists(container_name)
 
 
 def test_handle_failed_schain_rpc_exit_time_reached(
-    schain_db,
-    dutils,
-    cleanup_schain_containers,
-    skaled_status_exit_time_reached
+    schain_db, dutils, cleanup_schain_containers, skaled_status_exit_time_reached
 ):
     schain_record = SChainRecord.get_by_name(schain_db)
 
-    image_name, container_name, _, _ = get_container_info(
-        SCHAIN_CONTAINER, schain_db)
+    image_name, container_name, _, _ = get_container_info(SCHAIN_CONTAINER, schain_db)
 
     dutils.run_container(image_name=image_name, name=container_name, entrypoint='bash -c "exit 0"')
     sleep(7)
@@ -42,7 +47,7 @@ def test_handle_failed_schain_rpc_exit_time_reached(
         schain={'name': schain_db},
         schain_record=schain_record,
         skaled_status=skaled_status_exit_time_reached,
-        dutils=dutils
+        dutils=dutils,
     )
     assert dutils.is_container_exists(container_name)
 
@@ -51,20 +56,14 @@ def test_handle_failed_schain_rpc_exit_time_reached(
 
 
 def test_monitor_schain_downloading_snapshot(
-    schain_db,
-    dutils,
-    cleanup_schain_containers,
-    skaled_status_downloading_snapshot
+    schain_db, dutils, cleanup_schain_containers, skaled_status_downloading_snapshot
 ):
     schain_record = SChainRecord.get_by_name(schain_db)
 
-    image_name, container_name, _, _ = get_container_info(
-        SCHAIN_CONTAINER, schain_db)
+    image_name, container_name, _, _ = get_container_info(SCHAIN_CONTAINER, schain_db)
 
     dutils.run_container(
-        image_name=image_name,
-        name=container_name,
-        entrypoint='bash -c "sleep 100"'
+        image_name=image_name, name=container_name, entrypoint='bash -c "sleep 100"'
     )
     sleep(7)
     schain_record.set_failed_rpc_count(100)
@@ -76,25 +75,19 @@ def test_monitor_schain_downloading_snapshot(
         schain={'name': schain_db},
         schain_record=schain_record,
         skaled_status=skaled_status_downloading_snapshot,
-        dutils=dutils
+        dutils=dutils,
     )
     container_info = dutils.get_info(container_name)
     assert container_info['stats']['State']['FinishedAt'] == finished_at
 
 
 def test_handle_failed_schain_rpc_stuck_max_retries(
-    schain_db,
-    dutils,
-    skaled_status,
-    cleanup_schain_containers
+    schain_db, dutils, skaled_status, cleanup_schain_containers
 ):
     schain_record = SChainRecord.get_by_name(schain_db)
-    image_name, container_name, _, _ = get_container_info(
-        SCHAIN_CONTAINER, schain_db)
+    image_name, container_name, _, _ = get_container_info(SCHAIN_CONTAINER, schain_db)
     dutils.run_container(
-        image_name=image_name,
-        name=container_name,
-        entrypoint='bash -c "sleep 100"'
+        image_name=image_name, name=container_name, entrypoint='bash -c "sleep 100"'
     )
 
     schain_record.set_failed_rpc_count(100)
@@ -107,7 +100,7 @@ def test_handle_failed_schain_rpc_stuck_max_retries(
         schain={'name': schain_db},
         schain_record=schain_record,
         skaled_status=skaled_status,
-        dutils=dutils
+        dutils=dutils,
     )
     container_info = dutils.get_info(container_name)
     assert container_info['stats']['State']['FinishedAt'] == finished_at
@@ -115,12 +108,9 @@ def test_handle_failed_schain_rpc_stuck_max_retries(
 
 def test_monitor_container_exited(schain_db, dutils, cleanup_schain_containers, skaled_status):
     schain_record = SChainRecord.get_by_name(schain_db)
-    image_name, container_name, _, _ = get_container_info(
-        SCHAIN_CONTAINER, schain_db)
+    image_name, container_name, _, _ = get_container_info(SCHAIN_CONTAINER, schain_db)
     dutils.run_container(
-        image_name=image_name,
-        name=container_name,
-        entrypoint='bash -c "exit 100;"'
+        image_name=image_name, name=container_name, entrypoint='bash -c "exit 100;"'
     )
 
     schain_record.set_failed_rpc_count(100)
@@ -134,7 +124,7 @@ def test_monitor_container_exited(schain_db, dutils, cleanup_schain_containers,
         schain={'name': schain_db},
         schain_record=schain_record,
         skaled_status=skaled_status,
-        dutils=dutils
+        dutils=dutils,
     )
     assert schain_record.restart_count == 0
     container_info = dutils.get_info(container_name)
@@ -142,18 +132,12 @@ def test_monitor_container_exited(schain_db, dutils, cleanup_schain_containers,
 
 
 def test_handle_failed_schain_rpc_stuck(
-        schain_db,
-        dutils,
-        cleanup_schain_containers,
-        skaled_status
+    schain_db, dutils, cleanup_schain_containers, skaled_status
 ):
     schain_record = SChainRecord.get_by_name(schain_db)
-    image_name, container_name, _, _ = get_container_info(
-        SCHAIN_CONTAINER, schain_db)
+    image_name, container_name, _, _ = get_container_info(SCHAIN_CONTAINER, schain_db)
     dutils.run_container(
-        image_name=image_name,
-        name=container_name,
-        entrypoint='bash -c "sleep 100"'
+        image_name=image_name, name=container_name, entrypoint='bash -c "sleep 100"'
     )
 
     schain_record.set_failed_rpc_count(100)
@@ -167,8 +151,39 @@ def test_handle_failed_schain_rpc_stuck(
         schain={'name': schain_db},
         schain_record=schain_record,
         skaled_status=skaled_status,
-        dutils=dutils
+        dutils=dutils,
     )
     assert schain_record.restart_count == 1
     container_info = dutils.get_info(container_name)
     assert container_info['stats']['State']['FinishedAt'] != finished_at
+
+
+@mock.patch('tools.helper.requests.post')
+@freezegun.freeze_time(CURRENT_DATETIME)
+def test_check_endpoint_blocks(post_request_mock):
+    endpoint = 'http://127.0.0.1:10003'
+
+    post_request_mock.side_effect = requests.exceptions.RequestException('Test error')
+    assert check_endpoint_blocks(endpoint) is False
+    post_request_mock.side_effect = None
+
+    response_dummy = mock.Mock()
+    post_request_mock.return_value = response_dummy
+
+    response_dummy.json = mock.Mock(return_value={})
+    assert check_endpoint_blocks(endpoint) is False
+
+    response_dummy.json = mock.Mock(
+        side_effect=json.JSONDecodeError('Test error', doc='doc', pos=1)
+    )
+    assert check_endpoint_blocks(endpoint) is False
+
+    response_dummy.json = mock.Mock(return_value={'result': {'timestamp': '0xhhhhh'}})
+    assert check_endpoint_blocks(endpoint) is False
+
+    response_dummy.json = mock.Mock(return_value={'result': {'timestamp': '0x1'}})
+    assert check_endpoint_blocks(endpoint) is False
+
+    hex_offset_ts = hex(CURRENT_TIMESTAMP + 1)
+    response_dummy.json = mock.Mock(return_value={'result': {'timestamp': hex_offset_ts}})
+    assert check_endpoint_blocks(endpoint) is True

From d8fa24f19d942fc1ad06a11e9375b3cf5319935c Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Wed, 3 Jul 2024 12:55:42 +0000
Subject: [PATCH 02/42] Increase amount of ETH sent to node wallets

---
 tests/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/utils.py b/tests/utils.py
index 29f37e745..c94f9238e 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -45,7 +45,7 @@
     DIR_PATH, os.pardir, 'helper-scripts', 'contracts_data', 'ima.json')
 
 
-ETH_AMOUNT_PER_NODE = 1
+ETH_AMOUNT_PER_NODE = 2
 CONFIG_STREAM = "1.0.0-testnet"
 
 

From a29738be0a4489ded7bfec5b99f3d075e0119e80 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 11:55:32 +0000
Subject: [PATCH 03/42] Add default gas price for tests

---
 scripts/helper.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/helper.sh b/scripts/helper.sh
index afebbca74..b8f15641a 100644
--- a/scripts/helper.sh
+++ b/scripts/helper.sh
@@ -23,6 +23,7 @@ export_test_env () {
     export SCHAIN_STOP_TIMEOUT=1
     export ABI_FILEPATH=${ABI_FILEPATH="$PWD/helper-scripts/contracts_data/manager.json"}
     export IMA_ABI_FILEPATH=${IMA_ABI_FILEPATH}
+    export DEFAULT_GAS_PRICE_WEI=1000000000
 
     cp $PWD/helper-scripts/contracts_data/ima.json $SKALE_DIR_HOST/contracts_info
 }

From 4b7e37cc604b966da969cd03cf50eaeef1464ac4 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Fri, 5 Jul 2024 15:23:09 +0000
Subject: [PATCH 04/42] Fix archive node firewall rules after node rotation

---
 core/schains/firewall/rule_controller.py |  2 +-
 tests/firewall/rule_controller_test.py   | 45 ++++++++++++++++++++++++
 2 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/core/schains/firewall/rule_controller.py b/core/schains/firewall/rule_controller.py
index 2fda28168..51e8920a8 100644
--- a/core/schains/firewall/rule_controller.py
+++ b/core/schains/firewall/rule_controller.py
@@ -87,7 +87,7 @@ def get_missing(self) -> Dict['str', Any]:
         return missing
 
     def is_configured(self) -> bool:
-        return all((self.base_port, self.own_ip, self.node_ips))
+        return all((self.base_port, self.node_ips))
 
     def configure(
         self,
diff --git a/tests/firewall/rule_controller_test.py b/tests/firewall/rule_controller_test.py
index e0aa26a15..d4f04d598 100644
--- a/tests/firewall/rule_controller_test.py
+++ b/tests/firewall/rule_controller_test.py
@@ -150,6 +150,50 @@ def test_schain_rule_controller_configure():
     own_ip = '1.1.1.1'
     node_ips = ['1.1.1.1', '2.2.2.2', '3.3.3.3', '4.4.4.4']
     base_port = 10000
+
+    src.configure(base_port=base_port)
+    with pytest.raises(NotInitializedError):
+        src.public_ports()
+
+    src.configure(base_port=base_port, node_ips=node_ips)
+    assert list(src.public_ports) == [10003, 10008, 10002, 10007, 10009]
+
+    expected_rules = {
+        SChainRule(port=10000, first_ip='1.1.1.1', last_ip=None),
+        SChainRule(port=10000, first_ip='2.2.2.2', last_ip=None),
+        SChainRule(port=10000, first_ip='3.3.3.3', last_ip=None),
+        SChainRule(port=10000, first_ip='4.4.4.4', last_ip=None),
+        SChainRule(port=10001, first_ip='1.1.1.1', last_ip=None),
+        SChainRule(port=10001, first_ip='2.2.2.2', last_ip=None),
+        SChainRule(port=10001, first_ip='3.3.3.3', last_ip=None),
+        SChainRule(port=10001, first_ip='4.4.4.4', last_ip=None),
+        SChainRule(port=10002, first_ip=None, last_ip=None),
+        SChainRule(port=10003, first_ip=None, last_ip=None),
+        SChainRule(port=10004, first_ip='1.1.1.1', last_ip=None),
+        SChainRule(port=10004, first_ip='2.2.2.2', last_ip=None),
+        SChainRule(port=10004, first_ip='3.3.3.3', last_ip=None),
+        SChainRule(port=10004, first_ip='4.4.4.4', last_ip=None),
+        SChainRule(port=10005, first_ip='1.1.1.1', last_ip=None),
+        SChainRule(port=10005, first_ip='2.2.2.2', last_ip=None),
+        SChainRule(port=10005, first_ip='3.3.3.3', last_ip=None),
+        SChainRule(port=10005, first_ip='4.4.4.4', last_ip=None),
+        SChainRule(port=10007, first_ip=None, last_ip=None),
+        SChainRule(port=10008, first_ip=None, last_ip=None),
+        SChainRule(port=10009, first_ip=None, last_ip=None),
+        SChainRule(port=10010, first_ip='1.1.1.1', last_ip=None),
+        SChainRule(port=10010, first_ip='2.2.2.2', last_ip=None),
+        SChainRule(port=10010, first_ip='3.3.3.3', last_ip=None),
+        SChainRule(port=10010, first_ip='4.4.4.4', last_ip=None)
+    }
+    src.configure(base_port=base_port, node_ips=node_ips)
+
+    assert not src.is_rules_synced()
+    assert list(src.expected_rules()) == list(sorted(expected_rules))
+    src.sync()
+    assert src.is_rules_synced()
+    assert list(src.expected_rules()) == list(sorted(expected_rules))
+    assert list(src.actual_rules()) == list(sorted(expected_rules))
+
     expected_rules = {
         SChainRule(port=10000, first_ip='2.2.2.2', last_ip=None),
         SChainRule(port=10000, first_ip='3.3.3.3', last_ip=None),
@@ -173,6 +217,7 @@ def test_schain_rule_controller_configure():
         SChainRule(port=10010, first_ip='4.4.4.4', last_ip=None)
     }
     src.configure(base_port=base_port, own_ip=own_ip, node_ips=node_ips)
+
     assert not src.is_rules_synced()
     assert list(src.expected_rules()) == list(sorted(expected_rules))
     src.sync()

From 097d1c24b82c3a9a7bb7070ad6c65cc06b73d5c5 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Sat, 6 Jul 2024 17:00:45 +0000
Subject: [PATCH 05/42] Replace hyphen with underscore in metric names

---
 core/schains/checks.py                        |   4 +-
 core/schains/dkg/client.py                    | 228 +++++++++---------
 core/schains/monitor/action.py                |   9 +-
 core/schains/monitor/main.py                  |  99 +++-----
 core/schains/monitor/post_rotation_monitor.py |  38 ---
 tools/helper.py                               |   4 +
 6 files changed, 169 insertions(+), 213 deletions(-)
 delete mode 100644 core/schains/monitor/post_rotation_monitor.py

diff --git a/core/schains/checks.py b/core/schains/checks.py
index 73f12313b..288c3ed4b 100644
--- a/core/schains/checks.py
+++ b/core/schains/checks.py
@@ -59,7 +59,7 @@
 
 from tools.configs.containers import IMA_CONTAINER, SCHAIN_CONTAINER
 from tools.docker_utils import DockerUtils
-from tools.helper import write_json
+from tools.helper import no_hyphens, write_json
 from tools.resources import get_statsd_client
 from tools.str_formatters import arguments_list_string
 
@@ -524,5 +524,5 @@ def log_checks_dict(schain_name, checks_dict):
 
 def send_to_statsd(statsd_client: statsd.StatsClient, schain_name: str, checks_dict: dict) -> None:
     for check, result in checks_dict.items():
-        mname = f'admin.checks.{schain_name}.{check}'
+        mname = f'admin.checks.{check}.{no_hyphens(schain_name)}'
         statsd_client.gauge(mname, int(result))
diff --git a/core/schains/dkg/client.py b/core/schains/dkg/client.py
index 28c001592..584bad0b6 100644
--- a/core/schains/dkg/client.py
+++ b/core/schains/dkg/client.py
@@ -29,6 +29,7 @@
 
 from core.schains.dkg.broadcast_filter import Filter
 from core.schains.dkg.structures import ComplaintReason, DKGStep
+from tools.helper import no_hyphens
 from tools.configs import NODE_DATA_PATH, SGX_CERTIFICATES_FOLDER
 from tools.resources import get_statsd_client
 from tools.sgx_utils import sgx_unreachable_retry
@@ -78,7 +79,7 @@ def convert_g2_point_to_hex(data):
     data_hexed = ''
     for coord in data:
         temp = hex(int(coord))[2:]
-        while (len(temp) < 64):
+        while len(temp) < 64:
             temp = '0' + temp
         data_hexed += temp
     return data_hexed
@@ -88,7 +89,7 @@ def convert_hex_to_g2_array(data):
     g2_array = []
     while len(data) > 0:
         cur = data[:256]
-        g2_array.append([str(x) for x in [int(cur[64 * i:64 * i + 64], 16) for i in range(4)]])
+        g2_array.append([str(x) for x in [int(cur[64 * i : 64 * i + 64], 16) for i in range(4)]])
         data = data[256:]
     return g2_array
 
@@ -96,14 +97,14 @@ def convert_hex_to_g2_array(data):
 def convert_str_to_key_share(sent_secret_key_contribution, n):
     return_value = []
     for i in range(n):
-        public_key = sent_secret_key_contribution[i * 192 + 64: (i + 1) * 192]
-        key_share = bytes.fromhex(sent_secret_key_contribution[i * 192: i * 192 + 64])
+        public_key = sent_secret_key_contribution[i * 192 + 64 : (i + 1) * 192]
+        key_share = bytes.fromhex(sent_secret_key_contribution[i * 192 : i * 192 + 64])
         return_value.append(KeyShare(public_key, key_share).tuple)
     return return_value
 
 
 def convert_key_share_to_str(data, n):
-    return "".join(to_verify(s) for s in [data[i * 192:(i + 1) * 192] for i in range(n)])
+    return ''.join(to_verify(s) for s in [data[i * 192 : (i + 1) * 192] for i in range(n)])
 
 
 def to_verify(share):
@@ -112,24 +113,24 @@ def to_verify(share):
 
 def generate_poly_name(group_index_str, node_id, dkg_id):
     return (
-            "POLY:SCHAIN_ID:"
-            f"{group_index_str}"
-            ":NODE_ID:"
-            f"{str(node_id)}"
-            ":DKG_ID:"
-            f"{str(dkg_id)}"
-        )
+        'POLY:SCHAIN_ID:'
+        f'{group_index_str}'
+        ':NODE_ID:'
+        f'{str(node_id)}'
+        ':DKG_ID:'
+        f'{str(dkg_id)}'
+    )
 
 
 def generate_bls_key_name(group_index_str, node_id, dkg_id):
     return (
-            "BLS_KEY:SCHAIN_ID:"
-            f"{group_index_str}"
-            ":NODE_ID:"
-            f"{str(node_id)}"
-            ":DKG_ID:"
-            f"{str(dkg_id)}"
-        )
+        'BLS_KEY:SCHAIN_ID:'
+        f'{group_index_str}'
+        ':NODE_ID:'
+        f'{str(node_id)}'
+        ':DKG_ID:'
+        f'{str(dkg_id)}'
+    )
 
 
 class DKGClient:
@@ -146,10 +147,11 @@ def __init__(
         node_ids_contract,
         eth_key_name,
         rotation_id,
-        step: DKGStep = DKGStep.NONE
+        step: DKGStep = DKGStep.NONE,
     ):
-        self.sgx = SgxClient(os.environ['SGX_SERVER_URL'], n=n, t=t,
-                             path_to_cert=SGX_CERTIFICATES_FOLDER)
+        self.sgx = SgxClient(
+            os.environ['SGX_SERVER_URL'], n=n, t=t, path_to_cert=SGX_CERTIFICATES_FOLDER
+        )
         self.schain_name = schain_name
         self.group_index = skale.schains.name_to_group_id(schain_name)
         self.node_id_contract = node_id_contract
@@ -169,9 +171,9 @@ def __init__(
         self.node_ids_contract = node_ids_contract
         self.dkg_contract_functions = self.skale.dkg.contract.functions
         self.dkg_timeout = self.skale.constants_holder.get_dkg_timeout()
-        self.complaint_error_event_hash = self.skale.web3.to_hex(self.skale.web3.keccak(
-            text="ComplaintError(string)"
-        ))
+        self.complaint_error_event_hash = self.skale.web3.to_hex(
+            self.skale.web3.keccak(text='ComplaintError(string)')
+        )
         self.statsd_client = get_statsd_client()
         self._last_completed_step = step  # last step
         logger.info(f'sChain: {self.schain_name}. DKG timeout is {self.dkg_timeout}')
@@ -181,9 +183,11 @@ def last_completed_step(self) -> DKGStep:
         return self._last_completed_step
 
     @last_completed_step.setter
-    def last_completed_step(self, value: DKGStep):
-        self.statsd_client.gauge(f'admin.dkg.last_completed_step.{self.schain_name}', value)
-        self._last_completed_step = value
+    def last_completed_step(self, step: DKGStep):
+        self.statsd_client.gauge(
+            f'admin.dkg.last_completed_step.{no_hyphens(self.schain_name)}', step.value
+        )
+        self._last_completed_step = step
 
     def is_channel_opened(self):
         return self.skale.dkg.is_channel_opened(self.group_index)
@@ -193,7 +197,7 @@ def check_complaint_logs(self, logs):
 
     def store_broadcasted_data(self, data, from_node):
         self.incoming_secret_key_contribution[from_node] = data[1][
-            192 * self.node_id_dkg: 192 * (self.node_id_dkg + 1)
+            192 * self.node_id_dkg : 192 * (self.node_id_dkg + 1)
         ]
         if from_node == self.node_id_dkg:
             self.incoming_verification_vector[from_node] = convert_hex_to_g2_array(data[0])
@@ -214,11 +218,11 @@ def verification_vector(self):
 
     @sgx_unreachable_retry
     def secret_key_contribution(self):
-        self.sent_secret_key_contribution = self.sgx.get_secret_key_contribution_v2(self.poly_name,
-                                                                                    self.public_keys
-                                                                                    )
+        self.sent_secret_key_contribution = self.sgx.get_secret_key_contribution_v2(
+            self.poly_name, self.public_keys
+        )
         self.incoming_secret_key_contribution[self.node_id_dkg] = self.sent_secret_key_contribution[
-            self.node_id_dkg * 192: (self.node_id_dkg + 1) * 192
+            self.node_id_dkg * 192 : (self.node_id_dkg + 1) * 192
         ]
         return convert_str_to_key_share(self.sent_secret_key_contribution, self.n)
 
@@ -233,12 +237,14 @@ def broadcast(self):
             )
 
         is_broadcast_possible = self.skale.dkg.contract.functions.isBroadcastPossible(
-            self.group_index, self.node_id_contract).call({'from': self.skale.wallet.address})
+            self.group_index, self.node_id_contract
+        ).call({'from': self.skale.wallet.address})
 
         channel_opened = self.is_channel_opened()
         if not is_broadcast_possible or not channel_opened:
-            logger.info(f'sChain: {self.schain_name}. '
-                        f'{self.node_id_dkg} node could not sent broadcast')
+            logger.info(
+                f'sChain: {self.schain_name}. ' f'{self.node_id_dkg} node could not sent broadcast'
+            )
             return
 
         verification_vector = self.verification_vector()
@@ -249,7 +255,7 @@ def broadcast(self):
             self.node_id_contract,
             verification_vector,
             secret_key_contribution,
-            self.rotation_id
+            self.rotation_id,
         )
         self.last_completed_step = DKGStep.BROADCAST
         logger.info('Everything is sent from %d node', self.node_id_dkg)
@@ -262,28 +268,31 @@ def receive_from_node(self, from_node, broadcasted_data):
         try:
             if not self.verification(from_node):
                 raise DkgVerificationError(
-                    f"sChain: {self.schain_name}. "
-                    f"Fatal error : user {str(from_node + 1)} "
+                    f'sChain: {self.schain_name}. '
+                    f'Fatal error : user {str(from_node + 1)} '
                     f"hasn't passed verification by user {str(self.node_id_dkg + 1)}"
                 )
-            logger.info(f'sChain: {self.schain_name}. '
-                        f'All data from {from_node} was received and verified')
+            logger.info(
+                f'sChain: {self.schain_name}. '
+                f'All data from {from_node} was received and verified'
+            )
         except SgxUnreachableError as e:
             raise SgxUnreachableError(
-                    f"sChain: {self.schain_name}. "
-                    f"Fatal error : user {str(from_node + 1)} "
-                    f"hasn't passed verification by user {str(self.node_id_dkg + 1)}"
-                    f"with SgxUnreachableError: ", e
-                )
+                f'sChain: {self.schain_name}. '
+                f'Fatal error : user {str(from_node + 1)} '
+                f"hasn't passed verification by user {str(self.node_id_dkg + 1)}"
+                f'with SgxUnreachableError: ',
+                e,
+            )
 
     @sgx_unreachable_retry
     def verification(self, from_node):
-        return self.sgx.verify_secret_share_v2(self.incoming_verification_vector[from_node],
-                                               self.eth_key_name,
-                                               to_verify(
-                                                self.incoming_secret_key_contribution[from_node]
-                                               ),
-                                               self.node_id_dkg)
+        return self.sgx.verify_secret_share_v2(
+            self.incoming_verification_vector[from_node],
+            self.eth_key_name,
+            to_verify(self.incoming_secret_key_contribution[from_node]),
+            self.node_id_dkg,
+        )
 
     @sgx_unreachable_retry
     def is_bls_key_generated(self):
@@ -298,17 +307,20 @@ def is_bls_key_generated(self):
 
     @sgx_unreachable_retry
     def generate_bls_key(self):
-        received_secret_key_contribution = "".join(to_verify(
-                                                    self.incoming_secret_key_contribution[j]
-                                                    )
-                                                   for j in range(self.sgx.n))
-        logger.info(f'sChain: {self.schain_name}. '
-                    f'DKGClient is going to create BLS private key with name {self.bls_name}')
-        bls_private_key = self.sgx.create_bls_private_key_v2(self.poly_name, self.bls_name,
-                                                             self.eth_key_name,
-                                                             received_secret_key_contribution)
-        logger.info(f'sChain: {self.schain_name}. '
-                    'DKGClient is going to fetch BLS public key with name {self.bls_name}')
+        received_secret_key_contribution = ''.join(
+            to_verify(self.incoming_secret_key_contribution[j]) for j in range(self.sgx.n)
+        )
+        logger.info(
+            f'sChain: {self.schain_name}. '
+            f'DKGClient is going to create BLS private key with name {self.bls_name}'
+        )
+        bls_private_key = self.sgx.create_bls_private_key_v2(
+            self.poly_name, self.bls_name, self.eth_key_name, received_secret_key_contribution
+        )
+        logger.info(
+            f'sChain: {self.schain_name}. '
+            'DKGClient is going to fetch BLS public key with name {self.bls_name}'
+        )
         self.public_key = self.sgx.get_bls_public_key(self.bls_name)
         return bls_private_key
 
@@ -326,72 +338,70 @@ def get_bls_public_keys(self):
     def alright(self):
         logger.info(f'sChain {self.schain_name} sending alright transaction')
         is_alright_possible = self.skale.dkg.is_alright_possible(
-            self.group_index, self.node_id_contract, self.skale.wallet.address)
+            self.group_index, self.node_id_contract, self.skale.wallet.address
+        )
 
         if not is_alright_possible or not self.is_channel_opened():
-            logger.info(f'sChain: {self.schain_name}. '
-                        f'{self.node_id_dkg} node could not sent an alright note')
+            logger.info(
+                f'sChain: {self.schain_name}. '
+                f'{self.node_id_dkg} node could not sent an alright note'
+            )
             return
         self.skale.dkg.alright(
-            self.group_index,
-            self.node_id_contract,
-            gas_limit=ALRIGHT_GAS_LIMIT,
-            multiplier=2
+            self.group_index, self.node_id_contract, gas_limit=ALRIGHT_GAS_LIMIT, multiplier=2
         )
         self.last_completed_step = DKGStep.ALRIGHT
         logger.info(f'sChain: {self.schain_name}. {self.node_id_dkg} node sent an alright note')
 
     def send_complaint(self, to_node: int, reason: ComplaintReason):
-        logger.info(f'sChain: {self.schain_name}. '
-                    f'{self.node_id_dkg} node is trying to sent a {reason} on {to_node} node')
+        logger.info(
+            f'sChain: {self.schain_name}. '
+            f'{self.node_id_dkg} node is trying to sent a {reason} on {to_node} node'
+        )
 
         is_complaint_possible = self.skale.dkg.is_complaint_possible(
-            self.group_index, self.node_id_contract, self.node_ids_dkg[to_node],
-            self.skale.wallet.address
+            self.group_index,
+            self.node_id_contract,
+            self.node_ids_dkg[to_node],
+            self.skale.wallet.address,
         )
         is_channel_opened = self.is_channel_opened()
         logger.info(
-            'Complaint possible %s, channel opened %s',
-            is_complaint_possible,
-            is_channel_opened
+            'Complaint possible %s, channel opened %s', is_complaint_possible, is_channel_opened
         )
 
         if not is_complaint_possible or not is_channel_opened:
-            logger.info(
-                '%d node could not sent a complaint on %d node',
-                self.node_id_dkg,
-                to_node
-            )
+            logger.info('%d node could not sent a complaint on %d node', self.node_id_dkg, to_node)
             return False
 
         reason_to_step = {
             ComplaintReason.NO_BROADCAST: DKGStep.COMPLAINT_NO_BROADCAST,
             ComplaintReason.BAD_DATA: DKGStep.COMPLAINT_BAD_DATA,
             ComplaintReason.NO_ALRIGHT: DKGStep.COMPLAINT_NO_ALRIGHT,
-            ComplaintReason.NO_RESPONSE: DKGStep.COMPLAINT_NO_RESPONSE
+            ComplaintReason.NO_RESPONSE: DKGStep.COMPLAINT_NO_RESPONSE,
         }
 
         try:
             if reason == ComplaintReason.BAD_DATA:
                 tx_res = self.skale.dkg.complaint_bad_data(
-                    self.group_index,
-                    self.node_id_contract,
-                    self.node_ids_dkg[to_node]
+                    self.group_index, self.node_id_contract, self.node_ids_dkg[to_node]
                 )
             else:
                 tx_res = self.skale.dkg.complaint(
-                    self.group_index,
-                    self.node_id_contract,
-                    self.node_ids_dkg[to_node]
+                    self.group_index, self.node_id_contract, self.node_ids_dkg[to_node]
                 )
             if self.check_complaint_logs(tx_res.receipt['logs'][0]):
-                logger.info(f'sChain: {self.schain_name}. '
-                            f'{self.node_id_dkg} node sent a complaint on {to_node} node')
+                logger.info(
+                    f'sChain: {self.schain_name}. '
+                    f'{self.node_id_dkg} node sent a complaint on {to_node} node'
+                )
                 self.last_completed_step = reason_to_step[reason]
                 return True
             else:
-                logger.info(f'sChain: {self.schain_name}. Complaint from {self.node_id_dkg} on '
-                            f'{to_node} node was rejected')
+                logger.info(
+                    f'sChain: {self.schain_name}. Complaint from {self.node_id_dkg} on '
+                    f'{to_node} node was rejected'
+                )
                 return False
         except TransactionFailedError as e:
             logger.error(f'DKG complaint failed: sChain {self.schain_name}')
@@ -400,8 +410,7 @@ def send_complaint(self, to_node: int, reason: ComplaintReason):
     @sgx_unreachable_retry
     def get_complaint_response(self, to_node_index):
         response = self.sgx.complaint_response(
-            self.poly_name,
-            self.node_ids_contract[to_node_index]
+            self.poly_name, self.node_ids_contract[to_node_index]
         )
         share, dh_key = response.share, response.dh_key
         verification_vector_mult = response.verification_vector_mult
@@ -413,11 +422,13 @@ def get_complaint_response(self, to_node_index):
 
     def response(self, to_node_index):
         is_pre_response_possible = self.skale.dkg.is_pre_response_possible(
-            self.group_index, self.node_id_contract, self.skale.wallet.address)
+            self.group_index, self.node_id_contract, self.skale.wallet.address
+        )
 
         if not is_pre_response_possible or not self.is_channel_opened():
-            logger.info(f'sChain: {self.schain_name}. '
-                        f'{self.node_id_dkg} node could not sent a response')
+            logger.info(
+                f'sChain: {self.schain_name}. ' f'{self.node_id_dkg} node could not sent a response'
+            )
             return
 
         share, dh_key, verification_vector_mult = self.get_complaint_response(to_node_index)
@@ -428,24 +439,22 @@ def response(self, to_node_index):
                 self.node_id_contract,
                 convert_g2_points_to_array(self.incoming_verification_vector[self.node_id_dkg]),
                 convert_g2_points_to_array(verification_vector_mult),
-                convert_str_to_key_share(self.sent_secret_key_contribution, self.n)
+                convert_str_to_key_share(self.sent_secret_key_contribution, self.n),
             )
             self.last_completed_step = DKGStep.PRE_RESPONSE
 
             is_response_possible = self.skale.dkg.is_response_possible(
-                self.group_index, self.node_id_contract, self.skale.wallet.address)
+                self.group_index, self.node_id_contract, self.skale.wallet.address
+            )
 
             if not is_response_possible or not self.is_channel_opened():
-                logger.info(f'sChain: {self.schain_name}. '
-                            f'{self.node_id_dkg} node could not sent a response')
+                logger.info(
+                    f'sChain: {self.schain_name}. '
+                    f'{self.node_id_dkg} node could not sent a response'
+                )
                 return
 
-            self.skale.dkg.response(
-                self.group_index,
-                self.node_id_contract,
-                int(dh_key, 16),
-                share
-            )
+            self.skale.dkg.response(self.group_index, self.node_id_contract, int(dh_key, 16), share)
             self.last_completed_step = DKGStep.RESPONSE
             logger.info(f'sChain: {self.schain_name}. {self.node_id_dkg} node sent a response')
         except TransactionFailedError as e:
@@ -461,8 +470,7 @@ def fetch_all_broadcasted_data(self):
             broadcasted_data = [event.verificationVector, event.secretKeyContribution]
             self.store_broadcasted_data(broadcasted_data, from_node)
             logger.info(
-                f'sChain: {self.schain_name}. Received by {self.node_id_dkg} from '
-                f'{from_node}'
+                f'sChain: {self.schain_name}. Received by {self.node_id_dkg} from ' f'{from_node}'
             )
 
     def is_all_data_received(self, from_node):
diff --git a/core/schains/monitor/action.py b/core/schains/monitor/action.py
index dcc7696b4..6e543d2e9 100644
--- a/core/schains/monitor/action.py
+++ b/core/schains/monitor/action.py
@@ -78,6 +78,7 @@
 from tools.configs import SYNC_NODE
 from tools.configs.containers import IMA_CONTAINER, SCHAIN_CONTAINER
 from tools.docker_utils import DockerUtils
+from tools.helper import no_hyphens
 from tools.node_options import NodeOptions
 from tools.notifications.messages import notify_repair_mode
 from tools.resources import get_statsd_client
@@ -178,7 +179,7 @@ def config_dir(self) -> bool:
     @BaseActionManager.monitor_block
     def dkg(self) -> bool:
         initial_status = self.checks.dkg.status
-        with self.statsd_client.timer(f'admin.dkg.{self.name}'):
+        with self.statsd_client.timer(f'admin.dkg.{no_hyphens(self.name)}'):
             if not initial_status:
                 logger.info('Initing dkg client')
                 dkg_client = get_dkg_client(
@@ -212,7 +213,7 @@ def dkg(self) -> bool:
 
     @BaseActionManager.monitor_block
     def upstream_config(self) -> bool:
-        with self.statsd_client.timer(f'admin.upstream_config.{self.name}'):
+        with self.statsd_client.timer(f'admin.upstream_config.{no_hyphens(self.name)}'):
             logger.info(
                 'Creating new upstream_config rotation_id: %s, stream: %s',
                 self.rotation_data.get('rotation_id'), self.stream_version
@@ -348,7 +349,7 @@ def firewall_rules(self, upstream: bool = False) -> bool:
 
             ranges = self.econfig.ranges
             logger.info('Adding ranges %s', ranges)
-            with self.statsd_client.timer(f'admin.firewall.{self.name}'):
+            with self.statsd_client.timer(f'admin.firewall.{no_hyphens(self.name)}'):
                 self.rc.configure(
                     base_port=base_port,
                     own_ip=own_ip,
@@ -356,7 +357,7 @@ def firewall_rules(self, upstream: bool = False) -> bool:
                     sync_ip_ranges=ranges
                 )
                 self.statsd_client.gauge(
-                    f'admin.expected_rules.{self.name}',
+                    f'admin.expected_rules.{no_hyphens(self.name)}',
                     len(self.rc.expected_rules())
                 )
                 self.rc.sync()
diff --git a/core/schains/monitor/main.py b/core/schains/monitor/main.py
index 27405872c..9dd3a46c2 100644
--- a/core/schains/monitor/main.py
+++ b/core/schains/monitor/main.py
@@ -31,20 +31,11 @@
 
 from core.node import get_skale_node_version
 from core.node_config import NodeConfig
-from core.schains.checks import (
-    ConfigChecks,
-    get_api_checks_status,
-    TG_ALLOWED_CHECKS,
-    SkaledChecks
-)
+from core.schains.checks import ConfigChecks, get_api_checks_status, TG_ALLOWED_CHECKS, SkaledChecks
 from core.schains.config.file_manager import ConfigFileManager
 from core.schains.firewall import get_default_rule_controller
 from core.schains.firewall.utils import get_sync_agent_ranges
-from core.schains.monitor import (
-    get_skaled_monitor,
-    RegularConfigMonitor,
-    SyncConfigMonitor
-)
+from core.schains.monitor import get_skaled_monitor, RegularConfigMonitor, SyncConfigMonitor
 from core.schains.monitor.action import ConfigActionManager, SkaledActionManager
 from core.schains.external_config import ExternalConfig, ExternalState
 from core.schains.task import keep_tasks_running, Task
@@ -55,7 +46,7 @@
 from tools.docker_utils import DockerUtils
 from tools.configs import SYNC_NODE
 from tools.notifications.messages import notify_checks
-from tools.helper import is_node_part_of_chain
+from tools.helper import is_node_part_of_chain, no_hyphens
 from tools.resources import get_statsd_client
 from web.models.schain import SChainRecord
 
@@ -70,11 +61,7 @@
 
 
 def run_config_pipeline(
-    skale: Skale,
-    skale_ima: SkaleIma,
-    schain: Dict,
-    node_config: NodeConfig,
-    stream_version: str
+    skale: Skale, skale_ima: SkaleIma, schain: Dict, node_config: NodeConfig, stream_version: str
 ) -> None:
     name = schain['name']
     schain_record = SChainRecord.get_by_name(name)
@@ -84,9 +71,7 @@ def run_config_pipeline(
     current_nodes = get_current_nodes(skale, name)
 
     estate = ExternalState(
-        ima_linked=ima_linked,
-        chain_id=skale_ima.web3.eth.chain_id,
-        ranges=allowed_ranges
+        ima_linked=ima_linked, chain_id=skale_ima.web3.eth.chain_id, ranges=allowed_ranges
     )
     econfig = ExternalConfig(name)
     config_checks = ConfigChecks(
@@ -97,7 +82,7 @@ def run_config_pipeline(
         rotation_id=rotation_data['rotation_id'],
         current_nodes=current_nodes,
         econfig=econfig,
-        estate=estate
+        estate=estate,
     )
 
     config_am = ConfigActionManager(
@@ -109,7 +94,7 @@ def run_config_pipeline(
         checks=config_checks,
         current_nodes=current_nodes,
         estate=estate,
-        econfig=econfig
+        econfig=econfig,
     )
 
     status = config_checks.get_all(log=False, expose=True)
@@ -123,17 +108,16 @@ def run_config_pipeline(
         mon = RegularConfigMonitor(config_am, config_checks)
     statsd_client = get_statsd_client()
 
-    statsd_client.incr(f'admin.config.pipeline.{name}.{mon.__class__.__name__}')
-    statsd_client.gauge(f'admin.schain.rotation_id.{name}', rotation_data['rotation_id'])
-    with statsd_client.timer(f'admin.config.pipeline.{name}.duration'):
+    statsd_client.incr(f'admin.config.pipeline.{mon.__class__.__name__}.{no_hyphens(name)}')
+    statsd_client.gauge(
+        f'admin.schain.rotation_id.{no_hyphens(name)}', rotation_data['rotation_id']
+    )
+    with statsd_client.timer(f'admin.config.pipeline.duration.{no_hyphens(name)}'):
         mon.run()
 
 
 def run_skaled_pipeline(
-    skale: Skale,
-    schain: Dict,
-    node_config: NodeConfig,
-    dutils: DockerUtils
+    skale: Skale, schain: Dict, node_config: NodeConfig, dutils: DockerUtils
 ) -> None:
     name = schain['name']
     schain_record = SChainRecord.get_by_name(name)
@@ -146,7 +130,7 @@ def run_skaled_pipeline(
         schain_record=schain_record,
         rule_controller=rc,
         dutils=dutils,
-        sync_node=SYNC_NODE
+        sync_node=SYNC_NODE,
     )
 
     skaled_status = get_skaled_status(name)
@@ -157,12 +141,11 @@ def run_skaled_pipeline(
         checks=skaled_checks,
         node_config=node_config,
         econfig=ExternalConfig(name),
-        dutils=dutils
+        dutils=dutils,
     )
     status = skaled_checks.get_all(log=False, expose=True)
     automatic_repair = get_automatic_repair_option()
-    api_status = get_api_checks_status(
-        status=status, allowed=TG_ALLOWED_CHECKS)
+    api_status = get_api_checks_status(status=status, allowed=TG_ALLOWED_CHECKS)
     notify_checks(name, node_config.all(), api_status)
 
     logger.info('Skaled status: %s', status)
@@ -174,22 +157,20 @@ def run_skaled_pipeline(
         status=status,
         schain_record=schain_record,
         skaled_status=skaled_status,
-        automatic_repair=automatic_repair
+        automatic_repair=automatic_repair,
     )
 
     statsd_client = get_statsd_client()
-    statsd_client.incr(f'schain.skaled.pipeline.{name}.{mon.__name__}')
-    with statsd_client.timer(f'admin.skaled.pipeline.{name}.duration'):
+    statsd_client.incr(f'schain.skaled.pipeline.{mon.__name__}.{no_hyphens(name)}')
+    with statsd_client.timer(f'admin.skaled.pipeline.duration.{no_hyphens(name)}'):
         mon(skaled_am, skaled_checks).run()
 
 
 def post_monitor_sleep():
     schain_monitor_sleep = random.randint(
-        MIN_SCHAIN_MONITOR_SLEEP_INTERVAL,
-        MAX_SCHAIN_MONITOR_SLEEP_INTERVAL
+        MIN_SCHAIN_MONITOR_SLEEP_INTERVAL, MAX_SCHAIN_MONITOR_SLEEP_INTERVAL
     )
-    logger.info('Monitor iteration completed, sleeping for %d',
-                schain_monitor_sleep)
+    logger.info('Monitor iteration completed, sleeping for %d', schain_monitor_sleep)
     time.sleep(schain_monitor_sleep)
 
 
@@ -202,7 +183,7 @@ def create_and_execute_tasks(
     schain_record,
     executor,
     futures,
-    dutils
+    dutils,
 ):
     reload(web3_request)
     name = schain['name']
@@ -216,13 +197,15 @@ def create_and_execute_tasks(
 
     logger.info(
         'sync_config_run %s, config_version %s, stream_version %s',
-        schain_record.sync_config_run, schain_record.config_version, stream_version
+        schain_record.sync_config_run,
+        schain_record.config_version,
+        stream_version,
     )
 
     statsd_client = get_statsd_client()
     monitor_last_seen_ts = schain_record.monitor_last_seen.timestamp()
-    statsd_client.incr(f'admin.schain.monitor.{name}')
-    statsd_client.gauge(f'admin.schain.monitor_last_seen.{name}', monitor_last_seen_ts)
+    statsd_client.incr(f'admin.schain.monitor.{no_hyphens(name)}')
+    statsd_client.gauge(f'admin.schain.monitor_last_seen.{no_hyphens(name)}', monitor_last_seen_ts)
 
     tasks = []
     if not leaving_chain:
@@ -236,12 +219,14 @@ def create_and_execute_tasks(
                     skale_ima=skale_ima,
                     schain=schain,
                     node_config=node_config,
-                    stream_version=stream_version
+                    stream_version=stream_version,
                 ),
-                sleep=CONFIG_PIPELINE_SLEEP
-            ))
-    if schain_record.config_version != stream_version or \
-       (schain_record.sync_config_run and schain_record.first_run):
+                sleep=CONFIG_PIPELINE_SLEEP,
+            )
+        )
+    if schain_record.config_version != stream_version or (
+        schain_record.sync_config_run and schain_record.first_run
+    ):
         ConfigFileManager(name).remove_skaled_config()
     else:
         logger.info('Adding skaled task to the pool')
@@ -253,10 +238,11 @@ def create_and_execute_tasks(
                     skale=skale,
                     schain=schain,
                     node_config=node_config,
-                    dutils=dutils
+                    dutils=dutils,
                 ),
-                sleep=SKALED_PIPELINE_SLEEP
-            ))
+                sleep=SKALED_PIPELINE_SLEEP,
+            )
+        )
 
     if len(tasks) == 0:
         logger.warning('No tasks to run')
@@ -264,12 +250,7 @@ def create_and_execute_tasks(
 
 
 def run_monitor_for_schain(
-    skale,
-    skale_ima,
-    node_config: NodeConfig,
-    schain,
-    dutils=None,
-    once=False
+    skale, skale_ima, node_config: NodeConfig, schain, dutils=None, once=False
 ):
     stream_version = get_skale_node_version()
     tasks_number = 2
@@ -287,7 +268,7 @@ def run_monitor_for_schain(
                     schain_record,
                     executor,
                     futures,
-                    dutils
+                    dutils,
                 )
                 if once:
                     return True
diff --git a/core/schains/monitor/post_rotation_monitor.py b/core/schains/monitor/post_rotation_monitor.py
deleted file mode 100644
index 8200ab5a9..000000000
--- a/core/schains/monitor/post_rotation_monitor.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#   -*- coding: utf-8 -*-
-#
-#   This file is part of SKALE Admin
-#
-#   Copyright (C) 2021 SKALE Labs
-#
-#   This program is free software: you can redistribute it and/or modify
-#   it under the terms of the GNU Affero General Public License as published by
-#   the Free Software Foundation, either version 3 of the License, or
-#   (at your option) any later version.
-#
-#   This program is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#   GNU Affero General Public License for more details.
-#
-#   You should have received a copy of the GNU Affero General Public License
-#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-import logging
-
-from core.schains.monitor.base_monitor import BaseMonitor
-
-
-logger = logging.getLogger(__name__)
-
-
-class PostRotationMonitor(BaseMonitor):
-    """
-    PostRotationMonitor be executed for the sChain on the staying node when rotation is complete.
-    This type of monitor reloads skaled container.
-    """
-    @BaseMonitor.monitor_runner
-    def run(self):
-        logger.info(f'{self.p} was stopped after rotation. Going to restart')
-        self.config(overwrite=True)
-        self.firewall_rules()
-        self.recreated_schain_containers()
diff --git a/tools/helper.py b/tools/helper.py
index 8cfc28c3e..71b788f4f 100644
--- a/tools/helper.py
+++ b/tools/helper.py
@@ -184,3 +184,7 @@ def is_zero_address(address: str) -> bool:
 def is_address_contract(web3, address) -> bool:
     """Returns true if contract is deployed at the requested address"""
     return web3.eth.get_code(address) != b''
+
+
+def no_hyphens(name: str) -> str:
+    return name.replace('-', '_')

From cc447738946cd26c585b9acf8e634b5bea31aac2 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 15:32:29 +0000
Subject: [PATCH 06/42] Extract firewall tests into separate GA action

---
 .github/workflows/test.yml   | 15 ++++++++++++++-
 scripts/run_core_tests.sh    |  4 +---
 scripts/run_firewall_test.sh | 11 +++++++++--
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ae31339aa..6c5c8be6c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -40,14 +40,27 @@ jobs:
         run: | 
           sudo lsblk -f
           sudo free -h
+
       - name: Run core tests
         run: |
           bash ./scripts/run_core_tests.sh
-      - name: Show stats after tests
+
+      - name: Show stats after core tests
+        if: always()
+        run: |
+          sudo lsblk -f
+          sudo free -h
+
+      - name: Run firewall tests
+        run: |
+          bash ./scripts/run_firewall_tests.sh
+
+      - name: Show stats after firewall tests
         if: always()
         run: | 
           sudo lsblk -f
           sudo free -h
+
       - name: Run codecov
         run: |
           codecov -t $CODECOV_TOKEN
diff --git a/scripts/run_core_tests.sh b/scripts/run_core_tests.sh
index 9987b4af2..ae867c129 100755
--- a/scripts/run_core_tests.sh
+++ b/scripts/run_core_tests.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -e
+set -ea
 
 export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $DIR/helper.sh
@@ -15,5 +15,3 @@ bash scripts/run_redis.sh
 
 py.test --cov-config=.coveragerc --cov=. tests/ --ignore=tests/firewall $@
 tests_cleanup
-scripts/run_firewall_test.sh
-tests_cleanup
diff --git a/scripts/run_firewall_test.sh b/scripts/run_firewall_test.sh
index 037007baf..6858b80e1 100755
--- a/scripts/run_firewall_test.sh
+++ b/scripts/run_firewall_test.sh
@@ -1,8 +1,13 @@
 #!/usr/bin/env bash
-set -e
+set -ea
+
+export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $DIR/helper.sh
+source $DIR/../helper-scripts/helper.sh
+
+export_test_env
 
 docker rm -f test-firewall || true
-DIR=$PWD
 docker build -t admin:base .
 docker build -f tests.Dockerfile -t test-firewall .
 docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
@@ -16,3 +21,5 @@ docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
     -e SKALE_DIR_HOST=/skale_dir_host \
     --cap-add=NET_ADMIN --cap-add=NET_RAW \
     --name test-firewall test-firewall pytest --cov core.schains.firewall tests/firewall/ $@
+
+tests_cleanup

From 1da25bc419f465e15e45f46af5d0295c8a233303 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 16:34:04 +0000
Subject: [PATCH 07/42] Fix github action test.yml

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6c5c8be6c..418609d1b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -53,7 +53,7 @@ jobs:
 
       - name: Run firewall tests
         run: |
-          bash ./scripts/run_firewall_tests.sh
+          bash ./scripts/run_firewall_test.sh
 
       - name: Show stats after firewall tests
         if: always()

From c0ea3065c5ae0589832d936a83f68e2018a1c1e2 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 18:01:31 +0000
Subject: [PATCH 08/42] Add images cleanup after core tests

---
 .github/workflows/test.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 418609d1b..9da27ba69 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -8,6 +8,7 @@ env:
   SGX_WALLET_TAG: "1.83.0-beta.5"
   CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   PYTHON_VERSION: 3.11
+
 jobs:
   test_core:
     runs-on: ubuntu-latest
@@ -20,21 +21,27 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
+
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+
       - name: Install python dependencies
         run: bash ./scripts/install_python_dependencies.sh
+
       - name: Lint with flake8
         run: flake8 .
+
       - name: Launch anvil node
         run: |
           docker run -d --network host --name anvil ghcr.io/foundry-rs/foundry anvil && sleep 5 && docker logs anvil --tail 1000
+
       - name: Deploy manager & ima contracts
         run: |
           bash ./helper-scripts/deploy_test_ima.sh
           docker rmi -f skalenetwork/skale-manager:${{ env.MANAGER_TAG }}
+
       - name: Show stats before tests
         if: always()
         run: | 
@@ -44,6 +51,7 @@ jobs:
       - name: Run core tests
         run: |
           bash ./scripts/run_core_tests.sh
+          docker rmi -f $(docker ps -aq)
 
       - name: Show stats after core tests
         if: always()

From c05b2e63ee437297cff7095126925a6f57266244 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Fri, 5 Jul 2024 09:00:51 +0000
Subject: [PATCH 09/42] Fix image cleanup

---
 .github/workflows/test.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9da27ba69..6f7720c2b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -40,6 +40,9 @@ jobs:
       - name: Deploy manager & ima contracts
         run: |
           bash ./helper-scripts/deploy_test_ima.sh
+
+      - name: Cleanup skale-manager image
+        run: |
           docker rmi -f skalenetwork/skale-manager:${{ env.MANAGER_TAG }}
 
       - name: Show stats before tests
@@ -51,7 +54,11 @@ jobs:
       - name: Run core tests
         run: |
           bash ./scripts/run_core_tests.sh
-          docker rmi -f $(docker ps -aq)
+
+      - name: Cleanup docker artifacts
+        run: |
+          docker rm -f $(docker ps -aq)
+          docker rmi -f $(docker images -q)
 
       - name: Show stats after core tests
         if: always()

From b3bcbd13f01fa1282932b146642a153e1385d2c6 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Fri, 5 Jul 2024 12:47:16 +0000
Subject: [PATCH 10/42] Fix run_firewall_test script

---
 scripts/run_firewall_test.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/scripts/run_firewall_test.sh b/scripts/run_firewall_test.sh
index 6858b80e1..c3960525b 100755
--- a/scripts/run_firewall_test.sh
+++ b/scripts/run_firewall_test.sh
@@ -2,16 +2,12 @@
 set -ea
 
 export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source $DIR/helper.sh
-source $DIR/../helper-scripts/helper.sh
-
-export_test_env
 
 docker rm -f test-firewall || true
 docker build -t admin:base .
 docker build -f tests.Dockerfile -t test-firewall .
-docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
-    -v "$DIR/tests/skale-data":"/skale_vol" \
+docker run -v "$DIR/../tests/skale-data/node_data":"/skale_node_data" \
+    -v "$DIR/../tests/skale-data":"/skale_vol" \
     -e SGX_SERVER_URL="https://127.0.0.1:1026" \
     -e ENDPOINT="http://127.0.0.1:8545" \
     -e DB_USER="test" \
@@ -21,5 +17,3 @@ docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
     -e SKALE_DIR_HOST=/skale_dir_host \
     --cap-add=NET_ADMIN --cap-add=NET_RAW \
     --name test-firewall test-firewall pytest --cov core.schains.firewall tests/firewall/ $@
-
-tests_cleanup

From 7c7045a0d92910b38b526ee5865c3bab08482679 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 11:55:32 +0000
Subject: [PATCH 11/42] Add default gas price for tests

---
 scripts/helper.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/helper.sh b/scripts/helper.sh
index afebbca74..b8f15641a 100644
--- a/scripts/helper.sh
+++ b/scripts/helper.sh
@@ -23,6 +23,7 @@ export_test_env () {
     export SCHAIN_STOP_TIMEOUT=1
     export ABI_FILEPATH=${ABI_FILEPATH="$PWD/helper-scripts/contracts_data/manager.json"}
     export IMA_ABI_FILEPATH=${IMA_ABI_FILEPATH}
+    export DEFAULT_GAS_PRICE_WEI=1000000000
 
     cp $PWD/helper-scripts/contracts_data/ima.json $SKALE_DIR_HOST/contracts_info
 }

From 2a16403657f7101c1c4aa32585fdc239c04d08e6 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 15:32:29 +0000
Subject: [PATCH 12/42] Extract firewall tests into separate GA action

---
 .github/workflows/test.yml   | 15 ++++++++++++++-
 scripts/run_core_tests.sh    |  4 +---
 scripts/run_firewall_test.sh | 11 +++++++++--
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ae31339aa..6c5c8be6c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -40,14 +40,27 @@ jobs:
         run: | 
           sudo lsblk -f
           sudo free -h
+
       - name: Run core tests
         run: |
           bash ./scripts/run_core_tests.sh
-      - name: Show stats after tests
+
+      - name: Show stats after core tests
+        if: always()
+        run: |
+          sudo lsblk -f
+          sudo free -h
+
+      - name: Run firewall tests
+        run: |
+          bash ./scripts/run_firewall_tests.sh
+
+      - name: Show stats after firewall tests
         if: always()
         run: | 
           sudo lsblk -f
           sudo free -h
+
       - name: Run codecov
         run: |
           codecov -t $CODECOV_TOKEN
diff --git a/scripts/run_core_tests.sh b/scripts/run_core_tests.sh
index 9987b4af2..ae867c129 100755
--- a/scripts/run_core_tests.sh
+++ b/scripts/run_core_tests.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -e
+set -ea
 
 export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $DIR/helper.sh
@@ -15,5 +15,3 @@ bash scripts/run_redis.sh
 
 py.test --cov-config=.coveragerc --cov=. tests/ --ignore=tests/firewall $@
 tests_cleanup
-scripts/run_firewall_test.sh
-tests_cleanup
diff --git a/scripts/run_firewall_test.sh b/scripts/run_firewall_test.sh
index 037007baf..6858b80e1 100755
--- a/scripts/run_firewall_test.sh
+++ b/scripts/run_firewall_test.sh
@@ -1,8 +1,13 @@
 #!/usr/bin/env bash
-set -e
+set -ea
+
+export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $DIR/helper.sh
+source $DIR/../helper-scripts/helper.sh
+
+export_test_env
 
 docker rm -f test-firewall || true
-DIR=$PWD
 docker build -t admin:base .
 docker build -f tests.Dockerfile -t test-firewall .
 docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
@@ -16,3 +21,5 @@ docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
     -e SKALE_DIR_HOST=/skale_dir_host \
     --cap-add=NET_ADMIN --cap-add=NET_RAW \
     --name test-firewall test-firewall pytest --cov core.schains.firewall tests/firewall/ $@
+
+tests_cleanup

From cf5a041f4008b7663d1e8f3775e8fa0e4677bf19 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 16:34:04 +0000
Subject: [PATCH 13/42] Fix github action test.yml

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6c5c8be6c..418609d1b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -53,7 +53,7 @@ jobs:
 
       - name: Run firewall tests
         run: |
-          bash ./scripts/run_firewall_tests.sh
+          bash ./scripts/run_firewall_test.sh
 
       - name: Show stats after firewall tests
         if: always()

From dcca563b9386261fbed6b7160eb94e4d0c338a91 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 18:01:31 +0000
Subject: [PATCH 14/42] Add images cleanup after core tests

---
 .github/workflows/test.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 418609d1b..9da27ba69 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -8,6 +8,7 @@ env:
   SGX_WALLET_TAG: "1.83.0-beta.5"
   CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   PYTHON_VERSION: 3.11
+
 jobs:
   test_core:
     runs-on: ubuntu-latest
@@ -20,21 +21,27 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
+
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+
       - name: Install python dependencies
         run: bash ./scripts/install_python_dependencies.sh
+
       - name: Lint with flake8
         run: flake8 .
+
       - name: Launch anvil node
         run: |
           docker run -d --network host --name anvil ghcr.io/foundry-rs/foundry anvil && sleep 5 && docker logs anvil --tail 1000
+
       - name: Deploy manager & ima contracts
         run: |
           bash ./helper-scripts/deploy_test_ima.sh
           docker rmi -f skalenetwork/skale-manager:${{ env.MANAGER_TAG }}
+
       - name: Show stats before tests
         if: always()
         run: | 
@@ -44,6 +51,7 @@ jobs:
       - name: Run core tests
         run: |
           bash ./scripts/run_core_tests.sh
+          docker rmi -f $(docker ps -aq)
 
       - name: Show stats after core tests
         if: always()

From 7786d96e455ca4d51715082a755593a64d694007 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Fri, 5 Jul 2024 09:00:51 +0000
Subject: [PATCH 15/42] Fix image cleanup

---
 .github/workflows/test.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9da27ba69..6f7720c2b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -40,6 +40,9 @@ jobs:
       - name: Deploy manager & ima contracts
         run: |
           bash ./helper-scripts/deploy_test_ima.sh
+
+      - name: Cleanup skale-manager image
+        run: |
           docker rmi -f skalenetwork/skale-manager:${{ env.MANAGER_TAG }}
 
       - name: Show stats before tests
@@ -51,7 +54,11 @@ jobs:
       - name: Run core tests
         run: |
           bash ./scripts/run_core_tests.sh
-          docker rmi -f $(docker ps -aq)
+
+      - name: Cleanup docker artifacts
+        run: |
+          docker rm -f $(docker ps -aq)
+          docker rmi -f $(docker images -q)
 
       - name: Show stats after core tests
         if: always()

From febc753e0a7c90521003545437d1de8b9f754a42 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Fri, 5 Jul 2024 12:47:16 +0000
Subject: [PATCH 16/42] Fix run_firewall_test script

---
 scripts/run_firewall_test.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/scripts/run_firewall_test.sh b/scripts/run_firewall_test.sh
index 6858b80e1..c3960525b 100755
--- a/scripts/run_firewall_test.sh
+++ b/scripts/run_firewall_test.sh
@@ -2,16 +2,12 @@
 set -ea
 
 export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source $DIR/helper.sh
-source $DIR/../helper-scripts/helper.sh
-
-export_test_env
 
 docker rm -f test-firewall || true
 docker build -t admin:base .
 docker build -f tests.Dockerfile -t test-firewall .
-docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
-    -v "$DIR/tests/skale-data":"/skale_vol" \
+docker run -v "$DIR/../tests/skale-data/node_data":"/skale_node_data" \
+    -v "$DIR/../tests/skale-data":"/skale_vol" \
     -e SGX_SERVER_URL="https://127.0.0.1:1026" \
     -e ENDPOINT="http://127.0.0.1:8545" \
     -e DB_USER="test" \
@@ -21,5 +17,3 @@ docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
     -e SKALE_DIR_HOST=/skale_dir_host \
     --cap-add=NET_ADMIN --cap-add=NET_RAW \
     --name test-firewall test-firewall pytest --cov core.schains.firewall tests/firewall/ $@
-
-tests_cleanup

From 2e387c8864b3cf6bfb791f418eec0beef73b22d4 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Tue, 9 Jul 2024 09:25:56 +0000
Subject: [PATCH 17/42] Add additional logs (schain record fields)

---
 core/schains/monitor/main.py | 1 +
 web/models/schain.py         | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/core/schains/monitor/main.py b/core/schains/monitor/main.py
index 27405872c..ba57ba2a1 100644
--- a/core/schains/monitor/main.py
+++ b/core/schains/monitor/main.py
@@ -137,6 +137,7 @@ def run_skaled_pipeline(
 ) -> None:
     name = schain['name']
     schain_record = SChainRecord.get_by_name(name)
+    logger.info('Record: %s', SChainRecord.to_dict(schain_record))
 
     dutils = dutils or DockerUtils()
 
diff --git a/web/models/schain.py b/web/models/schain.py
index c685d864a..a7f67eb79 100644
--- a/web/models/schain.py
+++ b/web/models/schain.py
@@ -98,7 +98,13 @@ def to_dict(cls, record):
             'monitor_last_seen': record.monitor_last_seen.timestamp(),
             'monitor_id': record.monitor_id,
             'config_version': record.config_version,
-            'ssl_change_date': record.ssl_change_date.timestamp()
+            'ssl_change_date': record.ssl_change_date.timestamp(),
+            'repair_mode': record.repair_mode,
+            'backup_run': record.backup_run,
+            'sync_config_run': record.sync_config_run,
+            'snapshot_from': record.snapshot_from,
+            'restart_count': record.restart_count,
+            'failed_rpc_count': record.failed_rpc_count
         }
 
     def upload(self, *args, **kwargs) -> None:

From 467cd0ea888a2656f55ddc76a41e9d3dacc47f6d Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 11:55:32 +0000
Subject: [PATCH 18/42] Add default gas price for tests

---
 scripts/helper.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/helper.sh b/scripts/helper.sh
index afebbca74..b8f15641a 100644
--- a/scripts/helper.sh
+++ b/scripts/helper.sh
@@ -23,6 +23,7 @@ export_test_env () {
     export SCHAIN_STOP_TIMEOUT=1
     export ABI_FILEPATH=${ABI_FILEPATH="$PWD/helper-scripts/contracts_data/manager.json"}
     export IMA_ABI_FILEPATH=${IMA_ABI_FILEPATH}
+    export DEFAULT_GAS_PRICE_WEI=1000000000
 
     cp $PWD/helper-scripts/contracts_data/ima.json $SKALE_DIR_HOST/contracts_info
 }

From 46a91b722873a832b478f27686e7ba67f1f1690c Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 15:32:29 +0000
Subject: [PATCH 19/42] Extract firewall tests into separate GA action

---
 .github/workflows/test.yml   | 15 ++++++++++++++-
 scripts/run_core_tests.sh    |  4 +---
 scripts/run_firewall_test.sh | 11 +++++++++--
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ae31339aa..6c5c8be6c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -40,14 +40,27 @@ jobs:
         run: | 
           sudo lsblk -f
           sudo free -h
+
       - name: Run core tests
         run: |
           bash ./scripts/run_core_tests.sh
-      - name: Show stats after tests
+
+      - name: Show stats after core tests
+        if: always()
+        run: |
+          sudo lsblk -f
+          sudo free -h
+
+      - name: Run firewall tests
+        run: |
+          bash ./scripts/run_firewall_tests.sh
+
+      - name: Show stats after firewall tests
         if: always()
         run: | 
           sudo lsblk -f
           sudo free -h
+
       - name: Run codecov
         run: |
           codecov -t $CODECOV_TOKEN
diff --git a/scripts/run_core_tests.sh b/scripts/run_core_tests.sh
index 9987b4af2..ae867c129 100755
--- a/scripts/run_core_tests.sh
+++ b/scripts/run_core_tests.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -e
+set -ea
 
 export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $DIR/helper.sh
@@ -15,5 +15,3 @@ bash scripts/run_redis.sh
 
 py.test --cov-config=.coveragerc --cov=. tests/ --ignore=tests/firewall $@
 tests_cleanup
-scripts/run_firewall_test.sh
-tests_cleanup
diff --git a/scripts/run_firewall_test.sh b/scripts/run_firewall_test.sh
index 037007baf..6858b80e1 100755
--- a/scripts/run_firewall_test.sh
+++ b/scripts/run_firewall_test.sh
@@ -1,8 +1,13 @@
 #!/usr/bin/env bash
-set -e
+set -ea
+
+export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $DIR/helper.sh
+source $DIR/../helper-scripts/helper.sh
+
+export_test_env
 
 docker rm -f test-firewall || true
-DIR=$PWD
 docker build -t admin:base .
 docker build -f tests.Dockerfile -t test-firewall .
 docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
@@ -16,3 +21,5 @@ docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
     -e SKALE_DIR_HOST=/skale_dir_host \
     --cap-add=NET_ADMIN --cap-add=NET_RAW \
     --name test-firewall test-firewall pytest --cov core.schains.firewall tests/firewall/ $@
+
+tests_cleanup

From 439ae1b76d4fc09012f49877b368098e2d9bba99 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 16:34:04 +0000
Subject: [PATCH 20/42] Fix github action test.yml

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6c5c8be6c..418609d1b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -53,7 +53,7 @@ jobs:
 
       - name: Run firewall tests
         run: |
-          bash ./scripts/run_firewall_tests.sh
+          bash ./scripts/run_firewall_test.sh
 
       - name: Show stats after firewall tests
         if: always()

From 0aaee6f1b75fef27e380fcb2614b3851126ad2af Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 18:01:31 +0000
Subject: [PATCH 21/42] Add images cleanup after core tests

---
 .github/workflows/test.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 418609d1b..9da27ba69 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -8,6 +8,7 @@ env:
   SGX_WALLET_TAG: "1.83.0-beta.5"
   CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   PYTHON_VERSION: 3.11
+
 jobs:
   test_core:
     runs-on: ubuntu-latest
@@ -20,21 +21,27 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
+
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+
       - name: Install python dependencies
         run: bash ./scripts/install_python_dependencies.sh
+
       - name: Lint with flake8
         run: flake8 .
+
       - name: Launch anvil node
         run: |
           docker run -d --network host --name anvil ghcr.io/foundry-rs/foundry anvil && sleep 5 && docker logs anvil --tail 1000
+
       - name: Deploy manager & ima contracts
         run: |
           bash ./helper-scripts/deploy_test_ima.sh
           docker rmi -f skalenetwork/skale-manager:${{ env.MANAGER_TAG }}
+
       - name: Show stats before tests
         if: always()
         run: | 
@@ -44,6 +51,7 @@ jobs:
       - name: Run core tests
         run: |
           bash ./scripts/run_core_tests.sh
+          docker rmi -f $(docker ps -aq)
 
       - name: Show stats after core tests
         if: always()

From 36b38a9311f2d108f63891d436fac5213c7d88d8 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Fri, 5 Jul 2024 09:00:51 +0000
Subject: [PATCH 22/42] Fix image cleanup

---
 .github/workflows/test.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9da27ba69..6f7720c2b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -40,6 +40,9 @@ jobs:
       - name: Deploy manager & ima contracts
         run: |
           bash ./helper-scripts/deploy_test_ima.sh
+
+      - name: Cleanup skale-manager image
+        run: |
           docker rmi -f skalenetwork/skale-manager:${{ env.MANAGER_TAG }}
 
       - name: Show stats before tests
@@ -51,7 +54,11 @@ jobs:
       - name: Run core tests
         run: |
           bash ./scripts/run_core_tests.sh
-          docker rmi -f $(docker ps -aq)
+
+      - name: Cleanup docker artifacts
+        run: |
+          docker rm -f $(docker ps -aq)
+          docker rmi -f $(docker images -q)
 
       - name: Show stats after core tests
         if: always()

From b53b60a0c634702b0ba28296c550636b4de150b1 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Fri, 5 Jul 2024 12:47:16 +0000
Subject: [PATCH 23/42] Fix run_firewall_test script

---
 scripts/run_firewall_test.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/scripts/run_firewall_test.sh b/scripts/run_firewall_test.sh
index 6858b80e1..c3960525b 100755
--- a/scripts/run_firewall_test.sh
+++ b/scripts/run_firewall_test.sh
@@ -2,16 +2,12 @@
 set -ea
 
 export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source $DIR/helper.sh
-source $DIR/../helper-scripts/helper.sh
-
-export_test_env
 
 docker rm -f test-firewall || true
 docker build -t admin:base .
 docker build -f tests.Dockerfile -t test-firewall .
-docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
-    -v "$DIR/tests/skale-data":"/skale_vol" \
+docker run -v "$DIR/../tests/skale-data/node_data":"/skale_node_data" \
+    -v "$DIR/../tests/skale-data":"/skale_vol" \
     -e SGX_SERVER_URL="https://127.0.0.1:1026" \
     -e ENDPOINT="http://127.0.0.1:8545" \
     -e DB_USER="test" \
@@ -21,5 +17,3 @@ docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
     -e SKALE_DIR_HOST=/skale_dir_host \
     --cap-add=NET_ADMIN --cap-add=NET_RAW \
     --name test-firewall test-firewall pytest --cov core.schains.firewall tests/firewall/ $@
-
-tests_cleanup

From 7c712fbb8556eb0b7a30d6b0e47b3b4da9c0c4f4 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 11:55:32 +0000
Subject: [PATCH 24/42] Add default gas price for tests

---
 scripts/helper.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/helper.sh b/scripts/helper.sh
index afebbca74..b8f15641a 100644
--- a/scripts/helper.sh
+++ b/scripts/helper.sh
@@ -23,6 +23,7 @@ export_test_env () {
     export SCHAIN_STOP_TIMEOUT=1
     export ABI_FILEPATH=${ABI_FILEPATH="$PWD/helper-scripts/contracts_data/manager.json"}
     export IMA_ABI_FILEPATH=${IMA_ABI_FILEPATH}
+    export DEFAULT_GAS_PRICE_WEI=1000000000
 
     cp $PWD/helper-scripts/contracts_data/ima.json $SKALE_DIR_HOST/contracts_info
 }

From 1791df32f88504d29e9eaedd771739d5e57ac202 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 15:32:29 +0000
Subject: [PATCH 25/42] Extract firewall tests into separate GA action

---
 .github/workflows/test.yml   | 15 ++++++++++++++-
 scripts/run_core_tests.sh    |  4 +---
 scripts/run_firewall_test.sh | 11 +++++++++--
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ae31339aa..6c5c8be6c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -40,14 +40,27 @@ jobs:
         run: | 
           sudo lsblk -f
           sudo free -h
+
       - name: Run core tests
         run: |
           bash ./scripts/run_core_tests.sh
-      - name: Show stats after tests
+
+      - name: Show stats after core tests
+        if: always()
+        run: |
+          sudo lsblk -f
+          sudo free -h
+
+      - name: Run firewall tests
+        run: |
+          bash ./scripts/run_firewall_tests.sh
+
+      - name: Show stats after firewall tests
         if: always()
         run: | 
           sudo lsblk -f
           sudo free -h
+
       - name: Run codecov
         run: |
           codecov -t $CODECOV_TOKEN
diff --git a/scripts/run_core_tests.sh b/scripts/run_core_tests.sh
index 9987b4af2..ae867c129 100755
--- a/scripts/run_core_tests.sh
+++ b/scripts/run_core_tests.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -e
+set -ea
 
 export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source $DIR/helper.sh
@@ -15,5 +15,3 @@ bash scripts/run_redis.sh
 
 py.test --cov-config=.coveragerc --cov=. tests/ --ignore=tests/firewall $@
 tests_cleanup
-scripts/run_firewall_test.sh
-tests_cleanup
diff --git a/scripts/run_firewall_test.sh b/scripts/run_firewall_test.sh
index 037007baf..6858b80e1 100755
--- a/scripts/run_firewall_test.sh
+++ b/scripts/run_firewall_test.sh
@@ -1,8 +1,13 @@
 #!/usr/bin/env bash
-set -e
+set -ea
+
+export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $DIR/helper.sh
+source $DIR/../helper-scripts/helper.sh
+
+export_test_env
 
 docker rm -f test-firewall || true
-DIR=$PWD
 docker build -t admin:base .
 docker build -f tests.Dockerfile -t test-firewall .
 docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
@@ -16,3 +21,5 @@ docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
     -e SKALE_DIR_HOST=/skale_dir_host \
     --cap-add=NET_ADMIN --cap-add=NET_RAW \
     --name test-firewall test-firewall pytest --cov core.schains.firewall tests/firewall/ $@
+
+tests_cleanup

From feca96ca6bb50245369c1454ecc7b4b1a0083951 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 16:34:04 +0000
Subject: [PATCH 26/42] Fix github action test.yml

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6c5c8be6c..418609d1b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -53,7 +53,7 @@ jobs:
 
       - name: Run firewall tests
         run: |
-          bash ./scripts/run_firewall_tests.sh
+          bash ./scripts/run_firewall_test.sh
 
       - name: Show stats after firewall tests
         if: always()

From 5b2dd256447cd3b9bb4bfbf77b34f30ed83f086f Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 4 Jul 2024 18:01:31 +0000
Subject: [PATCH 27/42] Add images cleanup after core tests

---
 .github/workflows/test.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 418609d1b..9da27ba69 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -8,6 +8,7 @@ env:
   SGX_WALLET_TAG: "1.83.0-beta.5"
   CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   PYTHON_VERSION: 3.11
+
 jobs:
   test_core:
     runs-on: ubuntu-latest
@@ -20,21 +21,27 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
+
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+
       - name: Install python dependencies
         run: bash ./scripts/install_python_dependencies.sh
+
       - name: Lint with flake8
         run: flake8 .
+
       - name: Launch anvil node
         run: |
           docker run -d --network host --name anvil ghcr.io/foundry-rs/foundry anvil && sleep 5 && docker logs anvil --tail 1000
+
       - name: Deploy manager & ima contracts
         run: |
           bash ./helper-scripts/deploy_test_ima.sh
           docker rmi -f skalenetwork/skale-manager:${{ env.MANAGER_TAG }}
+
       - name: Show stats before tests
         if: always()
         run: | 
@@ -44,6 +51,7 @@ jobs:
       - name: Run core tests
         run: |
           bash ./scripts/run_core_tests.sh
+          docker rmi -f $(docker ps -aq)
 
       - name: Show stats after core tests
         if: always()

From 82661930d5072d997224fb50e23e6085944b899e Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Fri, 5 Jul 2024 09:00:51 +0000
Subject: [PATCH 28/42] Fix image cleanup

---
 .github/workflows/test.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9da27ba69..6f7720c2b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -40,6 +40,9 @@ jobs:
       - name: Deploy manager & ima contracts
         run: |
           bash ./helper-scripts/deploy_test_ima.sh
+
+      - name: Cleanup skale-manager image
+        run: |
           docker rmi -f skalenetwork/skale-manager:${{ env.MANAGER_TAG }}
 
       - name: Show stats before tests
@@ -51,7 +54,11 @@ jobs:
       - name: Run core tests
         run: |
           bash ./scripts/run_core_tests.sh
-          docker rmi -f $(docker ps -aq)
+
+      - name: Cleanup docker artifacts
+        run: |
+          docker rm -f $(docker ps -aq)
+          docker rmi -f $(docker images -q)
 
       - name: Show stats after core tests
         if: always()

From 3fe25134a80fd7ef35727a8a26ecb098949eae63 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Fri, 5 Jul 2024 12:47:16 +0000
Subject: [PATCH 29/42] Fix run_firewall_test script

---
 scripts/run_firewall_test.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/scripts/run_firewall_test.sh b/scripts/run_firewall_test.sh
index 6858b80e1..c3960525b 100755
--- a/scripts/run_firewall_test.sh
+++ b/scripts/run_firewall_test.sh
@@ -2,16 +2,12 @@
 set -ea
 
 export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source $DIR/helper.sh
-source $DIR/../helper-scripts/helper.sh
-
-export_test_env
 
 docker rm -f test-firewall || true
 docker build -t admin:base .
 docker build -f tests.Dockerfile -t test-firewall .
-docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
-    -v "$DIR/tests/skale-data":"/skale_vol" \
+docker run -v "$DIR/../tests/skale-data/node_data":"/skale_node_data" \
+    -v "$DIR/../tests/skale-data":"/skale_vol" \
     -e SGX_SERVER_URL="https://127.0.0.1:1026" \
     -e ENDPOINT="http://127.0.0.1:8545" \
     -e DB_USER="test" \
@@ -21,5 +17,3 @@ docker run -v "$DIR/tests/skale-data/node_data":"/skale_node_data" \
     -e SKALE_DIR_HOST=/skale_dir_host \
     --cap-add=NET_ADMIN --cap-add=NET_RAW \
     --name test-firewall test-firewall pytest --cov core.schains.firewall tests/firewall/ $@
-
-tests_cleanup

From e547f09ac520cc2b4660e2ef8b02b7f740b572b2 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Tue, 9 Jul 2024 11:05:39 +0000
Subject: [PATCH 30/42] Revert ETH_AMOUNT_PER_NODE to 1

---
 tests/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/utils.py b/tests/utils.py
index c94f9238e..29f37e745 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -45,7 +45,7 @@
     DIR_PATH, os.pardir, 'helper-scripts', 'contracts_data', 'ima.json')
 
 
-ETH_AMOUNT_PER_NODE = 2
+ETH_AMOUNT_PER_NODE = 1
 CONFIG_STREAM = "1.0.0-testnet"
 
 

From 82c09887d82a2d357ad3ca234a8a04eb3a0280f3 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Wed, 10 Jul 2024 14:25:35 +0000
Subject: [PATCH 31/42] Check last successful DKG in sync config monitor

---
 core/schains/checks.py                       | 151 ++++++++-----------
 core/schains/monitor/config_monitor.py       |  10 +-
 core/schains/monitor/main.py                 |   4 +
 tests/schains/monitor/config_monitor_test.py |  85 ++++++++++-
 4 files changed, 155 insertions(+), 95 deletions(-)

diff --git a/core/schains/checks.py b/core/schains/checks.py
index 73f12313b..8cde844fd 100644
--- a/core/schains/checks.py
+++ b/core/schains/checks.py
@@ -32,11 +32,11 @@
     get_base_port_from_config,
     get_node_ips_from_config,
     get_own_ip_from_config,
-    get_local_schain_http_endpoint_from_config
+    get_local_schain_http_endpoint_from_config,
 )
 from core.schains.config.main import (
     get_skaled_config_rotations_ids,
-    get_upstream_config_rotation_ids
+    get_upstream_config_rotation_ids,
 )
 from core.schains.dkg.utils import get_secret_key_share_filepath
 from core.schains.firewall.types import IRuleController
@@ -45,14 +45,14 @@
 from core.schains.rpc import (
     check_endpoint_alive,
     check_endpoint_blocks,
-    get_endpoint_alive_check_timeout
+    get_endpoint_alive_check_timeout,
 )
 from core.schains.external_config import ExternalConfig, ExternalState
 from core.schains.runner import (
     get_container_name,
     get_ima_container_time_frame,
     get_image_name,
-    is_new_image_pulled
+    is_new_image_pulled,
 )
 from core.schains.skaled_exit_codes import SkaledExitCodes
 from core.schains.volume import is_volume_exists
@@ -79,7 +79,7 @@
     'rpc',
     'blocks',
     'process',
-    'ima_container'
+    'ima_container',
 ]
 
 TG_ALLOWED_CHECKS = [
@@ -90,7 +90,7 @@
     'rpc',
     'blocks',
     'process',
-    'ima_container'
+    'ima_container',
 ]
 
 
@@ -111,11 +111,13 @@ class IChecks(ABC):
     def get_name(self) -> str:
         pass
 
-    def get_all(self,
-                log: bool = True,
-                save: bool = False,
-                expose: bool = False,
-                needed: Optional[List[str]] = None) -> Dict:
+    def get_all(
+        self,
+        log: bool = True,
+        save: bool = False,
+        expose: bool = False,
+        needed: Optional[List[str]] = None,
+    ) -> Dict:
         if needed:
             names = needed
         else:
@@ -140,25 +142,27 @@ def is_healthy(self) -> bool:
 
     @classmethod
     def get_check_names(cls):
-        return list(filter(
-            lambda c: not c.startswith('_') and isinstance(
-                getattr(cls, c), property),
-            dir(cls)
-        ))
+        return list(
+            filter(
+                lambda c: not c.startswith('_') and isinstance(getattr(cls, c), property), dir(cls)
+            )
+        )
 
 
 class ConfigChecks(IChecks):
-    def __init__(self,
-                 schain_name: str,
-                 node_id: int,
-                 schain_record: SChainRecord,
-                 rotation_id: int,
-                 stream_version: str,
-                 current_nodes: list[ExtendedManagerNodeInfo],
-                 estate: ExternalState,
-                 sync_node: bool = False,
-                 econfig: Optional[ExternalConfig] = None
-                 ) -> None:
+    def __init__(
+        self,
+        schain_name: str,
+        node_id: int,
+        schain_record: SChainRecord,
+        rotation_id: int,
+        stream_version: str,
+        current_nodes: list[ExtendedManagerNodeInfo],
+        estate: ExternalState,
+        last_dkg_successful: bool,
+        sync_node: bool = False,
+        econfig: Optional[ExternalConfig] = None,
+    ) -> None:
         self.name = schain_name
         self.node_id = node_id
         self.schain_record = schain_record
@@ -166,11 +170,10 @@ def __init__(self,
         self.stream_version = stream_version
         self.current_nodes = current_nodes
         self.estate = estate
+        self._last_dkg_successful = last_dkg_successful
         self.sync_node = sync_node
         self.econfig = econfig or ExternalConfig(schain_name)
-        self.cfm: ConfigFileManager = ConfigFileManager(
-            schain_name=schain_name
-        )
+        self.cfm: ConfigFileManager = ConfigFileManager(schain_name=schain_name)
         self.statsd_client = get_statsd_client()
 
     def get_name(self) -> str:
@@ -182,13 +185,14 @@ def config_dir(self) -> CheckRes:
         dir_path = self.cfm.dirname
         return CheckRes(os.path.isdir(dir_path))
 
+    def last_dkg_successful(self) -> CheckRes:
+        """Checks that last dkg was successfuly completed"""
+        return CheckRes(self._last_dkg_successful)
+
     @property
     def dkg(self) -> CheckRes:
         """Checks that DKG procedure is completed"""
-        secret_key_share_filepath = get_secret_key_share_filepath(
-            self.name,
-            self.rotation_id
-        )
+        secret_key_share_filepath = get_secret_key_share_filepath(self.name, self.rotation_id)
         return CheckRes(os.path.isfile(secret_key_share_filepath))
 
     @property
@@ -227,17 +231,14 @@ def upstream_config(self) -> CheckRes:
             exists,
             node_ips_updated,
             stream_updated,
-            triggered
+            triggered,
         )
         return CheckRes(exists and node_ips_updated and stream_updated and not triggered)
 
     @property
     def external_state(self) -> CheckRes:
         actual_state = self.econfig.get()
-        logger.debug(
-            'Checking external config. Current %s. Saved %s',
-            self.estate, actual_state
-        )
+        logger.debug('Checking external config. Current %s. Saved %s', self.estate, actual_state)
         return CheckRes(self.econfig.synced(self.estate))
 
 
@@ -250,7 +251,7 @@ def __init__(
         *,
         econfig: Optional[ExternalConfig] = None,
         dutils: Optional[DockerUtils] = None,
-        sync_node: bool = False
+        sync_node: bool = False,
     ):
         self.name = schain_name
         self.schain_record = schain_record
@@ -259,9 +260,7 @@ def __init__(
         self.econfig = econfig or ExternalConfig(name=schain_name)
         self.sync_node = sync_node
         self.rc = rule_controller
-        self.cfm: ConfigFileManager = ConfigFileManager(
-            schain_name=schain_name
-        )
+        self.cfm: ConfigFileManager = ConfigFileManager(schain_name=schain_name)
         self.statsd_client = get_statsd_client()
 
     def get_name(self) -> str:
@@ -278,9 +277,7 @@ def rotation_id_updated(self) -> CheckRes:
         upstream_rotations = get_upstream_config_rotation_ids(self.cfm)
         config_rotations = get_skaled_config_rotations_ids(self.cfm)
         logger.debug(
-            'Comparing rotation_ids. Upstream: %s. Config: %s',
-            upstream_rotations,
-            config_rotations
+            'Comparing rotation_ids. Upstream: %s. Config: %s', upstream_rotations, config_rotations
         )
         return CheckRes(upstream_rotations == config_rotations)
 
@@ -292,19 +289,14 @@ def config_updated(self) -> CheckRes:
 
     @property
     def config(self) -> CheckRes:
-        """ Checks that sChain config file exists """
+        """Checks that sChain config file exists"""
         return CheckRes(self.cfm.skaled_config_exists())
 
     @property
     def volume(self) -> CheckRes:
         """Checks that sChain volume exists"""
 
-        return CheckRes(
-            is_volume_exists(
-                self.name,
-                sync_node=self.sync_node,
-                dutils=self.dutils)
-            )
+        return CheckRes(is_volume_exists(self.name, sync_node=self.sync_node, dutils=self.dutils))
 
     @property
     def firewall_rules(self) -> CheckRes:
@@ -316,10 +308,7 @@ def firewall_rules(self) -> CheckRes:
             own_ip = get_own_ip_from_config(conf)
             ranges = self.econfig.ranges
             self.rc.configure(
-                base_port=base_port,
-                own_ip=own_ip,
-                node_ips=node_ips,
-                sync_ip_ranges=ranges
+                base_port=base_port, own_ip=own_ip, node_ips=node_ips, sync_ip_ranges=ranges
             )
             logger.debug(f'Rule controller {self.rc.expected_rules()}')
             return CheckRes(self.rc.is_rules_synced())
@@ -364,19 +353,19 @@ def ima_container(self) -> CheckRes:
             updated_time_frame = time_frame == container_time_frame
             logger.debug(
                 'IMA image %s, container image %s, time frame %d, container_time_frame %d',
-                expected_image, image, time_frame, container_time_frame
+                expected_image,
+                image,
+                time_frame,
+                container_time_frame,
             )
 
         data = {
             'container_running': container_running,
             'updated_image': updated_image,
             'new_image_pulled': new_image_pulled,
-            'updated_time_frame': updated_time_frame
+            'updated_time_frame': updated_time_frame,
         }
-        logger.debug(
-            '%s, IMA check - %s',
-            self.name, data
-        )
+        logger.debug('%s, IMA check - %s', self.name, data)
         result: bool = all(data.values())
         return CheckRes(result, data=data)
 
@@ -387,9 +376,7 @@ def rpc(self) -> CheckRes:
         if self.config:
             config = self.cfm.skaled_config
             http_endpoint = get_local_schain_http_endpoint_from_config(config)
-            timeout = get_endpoint_alive_check_timeout(
-                self.schain_record.failed_rpc_count
-            )
+            timeout = get_endpoint_alive_check_timeout(self.schain_record.failed_rpc_count)
             res = check_endpoint_alive(http_endpoint, timeout=timeout)
         return CheckRes(res)
 
@@ -430,7 +417,7 @@ def __init__(
         *,
         econfig: Optional[ExternalConfig] = None,
         dutils: DockerUtils = None,
-        sync_node: bool = False
+        sync_node: bool = False,
     ):
         self._subjects = [
             ConfigChecks(
@@ -442,7 +429,7 @@ def __init__(
                 current_nodes=current_nodes,
                 estate=estate,
                 econfig=econfig,
-                sync_node=sync_node
+                sync_node=sync_node,
             ),
             SkaledChecks(
                 schain_name=schain_name,
@@ -450,8 +437,8 @@ def __init__(
                 rule_controller=rule_controller,
                 econfig=econfig,
                 dutils=dutils,
-                sync_node=sync_node
-            )
+                sync_node=sync_node,
+            ),
         ]
 
     def __getattr__(self, attr: str) -> Any:
@@ -469,11 +456,7 @@ def get_all(self, log: bool = True, save: bool = False, needed: Optional[List[st
         plain_checks = {}
         for subj in self._subjects:
             logger.debug('Running checks for %s', subj)
-            subj_checks = subj.get_all(
-                log=False,
-                save=False,
-                needed=needed
-            )
+            subj_checks = subj.get_all(log=False, save=False, needed=needed)
             plain_checks.update(subj_checks)
         if not self.estate or not self.estate.ima_linked:
             if 'ima_container' in plain_checks:
@@ -492,13 +475,9 @@ def get_api_checks_status(status: Dict, allowed: List = API_ALLOWED_CHECKS) -> D
 
 def save_checks_dict(schain_name, checks_dict):
     schain_check_path = get_schain_check_filepath(schain_name)
-    logger.info(
-        f'Saving checks for the chain {schain_name}: {schain_check_path}')
+    logger.info(f'Saving checks for the chain {schain_name}: {schain_check_path}')
     try:
-        write_json(schain_check_path, {
-            'time': time.time(),
-            'checks': checks_dict
-        })
+        write_json(schain_check_path, {'time': time.time(), 'checks': checks_dict})
     except Exception:
         logger.exception(f'Failed to save checks: {schain_check_path}')
 
@@ -510,14 +489,12 @@ def log_checks_dict(schain_name, checks_dict):
         if not checks_dict[check]:
             failed_checks.append(check)
     if len(failed_checks) != 0:
-        failed_checks_str = ", ".join(failed_checks)
+        failed_checks_str = ', '.join(failed_checks)
         logger.info(
             arguments_list_string(
-                {
-                    'sChain name': schain_name,
-                    'Failed checks': failed_checks_str
-                },
-                'Failed sChain checks', 'error'
+                {'sChain name': schain_name, 'Failed checks': failed_checks_str},
+                'Failed sChain checks',
+                'error',
             )
         )
 
diff --git a/core/schains/monitor/config_monitor.py b/core/schains/monitor/config_monitor.py
index 1bd54841d..47587a1bc 100644
--- a/core/schains/monitor/config_monitor.py
+++ b/core/schains/monitor/config_monitor.py
@@ -29,11 +29,7 @@
 
 
 class BaseConfigMonitor(IMonitor):
-    def __init__(
-        self,
-        action_manager: ConfigActionManager,
-        checks: ConfigChecks
-    ) -> None:
+    def __init__(self, action_manager: ConfigActionManager, checks: ConfigChecks) -> None:
         self.am = action_manager
         self.checks = checks
 
@@ -73,7 +69,7 @@ def execute(self) -> None:
             self.am.config_dir()
         if not self.checks.external_state:
             self.am.external_state()
-        if not self.checks.upstream_config:
+        if self.checks.last_dkg_successful and not self.checks.upstream_config:
             self.am.upstream_config()
-        self.am.update_reload_ts(self.checks.skaled_node_ips, sync_node=True)
+            self.am.update_reload_ts(self.checks.skaled_node_ips, sync_node=True)
         self.am.reset_config_record()
diff --git a/core/schains/monitor/main.py b/core/schains/monitor/main.py
index ba57ba2a1..dcbbb5e22 100644
--- a/core/schains/monitor/main.py
+++ b/core/schains/monitor/main.py
@@ -81,6 +81,7 @@ def run_config_pipeline(
     rotation_data = skale.node_rotation.get_rotation(name)
     allowed_ranges = get_sync_agent_ranges(skale)
     ima_linked = not SYNC_NODE and skale_ima.linker.has_schain(name)
+    last_dkg_successful = skale.dkg.is_last_dkg_successfull(name)
     current_nodes = get_current_nodes(skale, name)
 
     estate = ExternalState(
@@ -96,6 +97,7 @@ def run_config_pipeline(
         stream_version=stream_version,
         rotation_id=rotation_data['rotation_id'],
         current_nodes=current_nodes,
+        last_dkg_successful=last_dkg_successful,
         econfig=econfig,
         estate=estate
     )
@@ -117,6 +119,8 @@ def run_config_pipeline(
 
     if SYNC_NODE:
         logger.info('Sync node mode, running config monitor')
+        if not last_dkg_successful:
+            logger.info('Latest DKG has not been completed for rotation_id %s', rotation_data)
         mon = SyncConfigMonitor(config_am, config_checks)
     else:
         logger.info('Regular node mode, running config monitor')
diff --git a/tests/schains/monitor/config_monitor_test.py b/tests/schains/monitor/config_monitor_test.py
index d7c211f65..3dc13584f 100644
--- a/tests/schains/monitor/config_monitor_test.py
+++ b/tests/schains/monitor/config_monitor_test.py
@@ -10,7 +10,7 @@
 from core.schains.config.directory import schain_config_dir
 
 from core.schains.monitor.action import ConfigActionManager
-from core.schains.monitor.config_monitor import RegularConfigMonitor
+from core.schains.monitor.config_monitor import RegularConfigMonitor, SyncConfigMonitor
 from core.schains.external_config import ExternalConfig
 
 from web.models.schain import SChainRecord
@@ -42,6 +42,7 @@ def config_checks(
         rotation_id=rotation_data['rotation_id'],
         stream_version=CONFIG_STREAM,
         current_nodes=current_nodes,
+        last_dkg_successful=True,
         estate=estate
     )
 
@@ -84,9 +85,18 @@ def regular_config_monitor(config_am, config_checks):
     )
 
 
+@pytest.fixture
+def sync_config_monitor(config_am, config_checks):
+    return SyncConfigMonitor(
+        action_manager=config_am,
+        checks=config_checks
+    )
+
+
 def test_regular_config_monitor(schain_db, regular_config_monitor, rotation_data):
     name = schain_db
     rotation_id = rotation_data['rotation_id']
+
     regular_config_monitor.run()
     config_dir = schain_config_dir(name)
 
@@ -130,3 +140,76 @@ def test_regular_config_monitor_change_ip(
     regular_config_monitor.am.cfm.sync_skaled_config_with_upstream()
     regular_config_monitor.run()
     assert econfig.reload_ts is None
+
+
+def test_sync_config_monitor(
+    skale,
+    schain_db,
+    config_am,
+    config_checks,
+    econfig,
+    estate,
+    rotation_data
+):
+    name = schain_db
+    config_dir = schain_config_dir(name)
+
+    rotation_id = rotation_data['rotation_id']
+    config_pattern = os.path.join(
+        config_dir,
+        f'schain_{name}_{rotation_id}_*.json'
+    )
+    assert len(glob.glob(config_pattern)) == 0
+
+    assert econfig.synced(estate)
+
+    estate.chain_id = 1
+    config_checks.estate = estate
+    config_am.estate = estate
+    assert not econfig.synced(estate)
+
+    sync_config_monitor = SyncConfigMonitor(
+        action_manager=config_am,
+        checks=config_checks
+    )
+    sync_config_monitor.run()
+    assert econfig.synced(estate)
+    config_filename = glob.glob(config_pattern)
+    assert os.path.isfile(config_filename[0])
+
+
+def test_sync_config_monitor_dkg_not_completed(
+    skale,
+    schain_db,
+    config_am,
+    config_checks,
+    econfig,
+    estate,
+    rotation_data
+):
+    name = schain_db
+    config_dir = schain_config_dir(name)
+
+    rotation_id = rotation_data['rotation_id']
+    config_pattern = os.path.join(
+        config_dir,
+        f'schain_{name}_{rotation_id}_*.json'
+    )
+    assert len(glob.glob(config_pattern)) == 0
+
+    assert econfig.synced(estate)
+
+    estate.chain_id = 1
+    config_checks.estate = estate
+    config_am.estate = estate
+    config_checks.last_dkg_successful = False
+    assert not econfig.synced(estate)
+
+    sync_config_monitor = SyncConfigMonitor(
+        action_manager=config_am,
+        checks=config_checks
+    )
+    sync_config_monitor.run()
+    assert econfig.synced(estate)
+    # config generation was not triggered because dkg has not been completed
+    assert len(glob.glob(config_pattern)) == 0

From a94d7ac4227c1ad2c932c11713992b23c34d31eb Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Wed, 10 Jul 2024 15:44:14 +0000
Subject: [PATCH 32/42] Improve logging

---
 core/schains/monitor/main.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/core/schains/monitor/main.py b/core/schains/monitor/main.py
index dcbbb5e22..3d50e0493 100644
--- a/core/schains/monitor/main.py
+++ b/core/schains/monitor/main.py
@@ -118,9 +118,7 @@ def run_config_pipeline(
     logger.info('Config checks: %s', status)
 
     if SYNC_NODE:
-        logger.info('Sync node mode, running config monitor')
-        if not last_dkg_successful:
-            logger.info('Latest DKG has not been completed for rotation_id %s', rotation_data)
+        logger.info('Sync node last_dkg_successful %s, rotation_data %s', last_dkg_successful, rotation_data)
         mon = SyncConfigMonitor(config_am, config_checks)
     else:
         logger.info('Regular node mode, running config monitor')

From 3b6a55174cfb6ec988e5c6ae0959ea379e5d5149 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Wed, 10 Jul 2024 16:37:01 +0000
Subject: [PATCH 33/42] Update gunicorn

---
 requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 664cd2b48..b39330d11 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
 peewee==3.9.5
 Flask==2.3.3
-Werkzeug==2.3.7
-gunicorn==20.1.0
+Werkzeug==3.0.3
+gunicorn==22.0.0
 
 Jinja2==3.1.2
 

From 352ec6d06d390f6caed6516681aa28e047dfe613 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Wed, 10 Jul 2024 18:16:06 +0000
Subject: [PATCH 34/42] Fix last_dkg_successful check

---
 core/schains/checks.py       |  1 +
 core/schains/monitor/main.py | 90 +++++++++++++++---------------------
 2 files changed, 38 insertions(+), 53 deletions(-)

diff --git a/core/schains/checks.py b/core/schains/checks.py
index 8cde844fd..71a1946e1 100644
--- a/core/schains/checks.py
+++ b/core/schains/checks.py
@@ -185,6 +185,7 @@ def config_dir(self) -> CheckRes:
         dir_path = self.cfm.dirname
         return CheckRes(os.path.isdir(dir_path))
 
+    @property
     def last_dkg_successful(self) -> CheckRes:
         """Checks that last dkg was successfuly completed"""
         return CheckRes(self._last_dkg_successful)
diff --git a/core/schains/monitor/main.py b/core/schains/monitor/main.py
index 3d50e0493..5030b8c7a 100644
--- a/core/schains/monitor/main.py
+++ b/core/schains/monitor/main.py
@@ -31,20 +31,11 @@
 
 from core.node import get_skale_node_version
 from core.node_config import NodeConfig
-from core.schains.checks import (
-    ConfigChecks,
-    get_api_checks_status,
-    TG_ALLOWED_CHECKS,
-    SkaledChecks
-)
+from core.schains.checks import ConfigChecks, get_api_checks_status, TG_ALLOWED_CHECKS, SkaledChecks
 from core.schains.config.file_manager import ConfigFileManager
 from core.schains.firewall import get_default_rule_controller
 from core.schains.firewall.utils import get_sync_agent_ranges
-from core.schains.monitor import (
-    get_skaled_monitor,
-    RegularConfigMonitor,
-    SyncConfigMonitor
-)
+from core.schains.monitor import get_skaled_monitor, RegularConfigMonitor, SyncConfigMonitor
 from core.schains.monitor.action import ConfigActionManager, SkaledActionManager
 from core.schains.external_config import ExternalConfig, ExternalState
 from core.schains.task import keep_tasks_running, Task
@@ -70,24 +61,19 @@
 
 
 def run_config_pipeline(
-    skale: Skale,
-    skale_ima: SkaleIma,
-    schain: Dict,
-    node_config: NodeConfig,
-    stream_version: str
+    skale: Skale, skale_ima: SkaleIma, schain: Dict, node_config: NodeConfig, stream_version: str
 ) -> None:
     name = schain['name']
     schain_record = SChainRecord.get_by_name(name)
     rotation_data = skale.node_rotation.get_rotation(name)
     allowed_ranges = get_sync_agent_ranges(skale)
     ima_linked = not SYNC_NODE and skale_ima.linker.has_schain(name)
-    last_dkg_successful = skale.dkg.is_last_dkg_successfull(name)
+    group_index = skale.schains.name_to_group_id(name)
+    last_dkg_successful = skale.dkg.is_last_dkg_successful(group_index)
     current_nodes = get_current_nodes(skale, name)
 
     estate = ExternalState(
-        ima_linked=ima_linked,
-        chain_id=skale_ima.web3.eth.chain_id,
-        ranges=allowed_ranges
+        ima_linked=ima_linked, chain_id=skale_ima.web3.eth.chain_id, ranges=allowed_ranges
     )
     econfig = ExternalConfig(name)
     config_checks = ConfigChecks(
@@ -99,7 +85,7 @@ def run_config_pipeline(
         current_nodes=current_nodes,
         last_dkg_successful=last_dkg_successful,
         econfig=econfig,
-        estate=estate
+        estate=estate,
     )
 
     config_am = ConfigActionManager(
@@ -111,14 +97,18 @@ def run_config_pipeline(
         checks=config_checks,
         current_nodes=current_nodes,
         estate=estate,
-        econfig=econfig
+        econfig=econfig,
     )
 
     status = config_checks.get_all(log=False, expose=True)
     logger.info('Config checks: %s', status)
 
     if SYNC_NODE:
-        logger.info('Sync node last_dkg_successful %s, rotation_data %s', last_dkg_successful, rotation_data)
+        logger.info(
+            'Sync node last_dkg_successful %s, rotation_data %s',
+            last_dkg_successful,
+            rotation_data
+        )
         mon = SyncConfigMonitor(config_am, config_checks)
     else:
         logger.info('Regular node mode, running config monitor')
@@ -132,10 +122,7 @@ def run_config_pipeline(
 
 
 def run_skaled_pipeline(
-    skale: Skale,
-    schain: Dict,
-    node_config: NodeConfig,
-    dutils: DockerUtils
+    skale: Skale, schain: Dict, node_config: NodeConfig, dutils: DockerUtils
 ) -> None:
     name = schain['name']
     schain_record = SChainRecord.get_by_name(name)
@@ -149,7 +136,7 @@ def run_skaled_pipeline(
         schain_record=schain_record,
         rule_controller=rc,
         dutils=dutils,
-        sync_node=SYNC_NODE
+        sync_node=SYNC_NODE,
     )
 
     skaled_status = get_skaled_status(name)
@@ -160,12 +147,11 @@ def run_skaled_pipeline(
         checks=skaled_checks,
         node_config=node_config,
         econfig=ExternalConfig(name),
-        dutils=dutils
+        dutils=dutils,
     )
     status = skaled_checks.get_all(log=False, expose=True)
     automatic_repair = get_automatic_repair_option()
-    api_status = get_api_checks_status(
-        status=status, allowed=TG_ALLOWED_CHECKS)
+    api_status = get_api_checks_status(status=status, allowed=TG_ALLOWED_CHECKS)
     notify_checks(name, node_config.all(), api_status)
 
     logger.info('Skaled status: %s', status)
@@ -177,7 +163,7 @@ def run_skaled_pipeline(
         status=status,
         schain_record=schain_record,
         skaled_status=skaled_status,
-        automatic_repair=automatic_repair
+        automatic_repair=automatic_repair,
     )
 
     statsd_client = get_statsd_client()
@@ -188,11 +174,9 @@ def run_skaled_pipeline(
 
 def post_monitor_sleep():
     schain_monitor_sleep = random.randint(
-        MIN_SCHAIN_MONITOR_SLEEP_INTERVAL,
-        MAX_SCHAIN_MONITOR_SLEEP_INTERVAL
+        MIN_SCHAIN_MONITOR_SLEEP_INTERVAL, MAX_SCHAIN_MONITOR_SLEEP_INTERVAL
     )
-    logger.info('Monitor iteration completed, sleeping for %d',
-                schain_monitor_sleep)
+    logger.info('Monitor iteration completed, sleeping for %d', schain_monitor_sleep)
     time.sleep(schain_monitor_sleep)
 
 
@@ -205,7 +189,7 @@ def create_and_execute_tasks(
     schain_record,
     executor,
     futures,
-    dutils
+    dutils,
 ):
     reload(web3_request)
     name = schain['name']
@@ -219,7 +203,9 @@ def create_and_execute_tasks(
 
     logger.info(
         'sync_config_run %s, config_version %s, stream_version %s',
-        schain_record.sync_config_run, schain_record.config_version, stream_version
+        schain_record.sync_config_run,
+        schain_record.config_version,
+        stream_version,
     )
 
     statsd_client = get_statsd_client()
@@ -239,12 +225,14 @@ def create_and_execute_tasks(
                     skale_ima=skale_ima,
                     schain=schain,
                     node_config=node_config,
-                    stream_version=stream_version
+                    stream_version=stream_version,
                 ),
-                sleep=CONFIG_PIPELINE_SLEEP
-            ))
-    if schain_record.config_version != stream_version or \
-       (schain_record.sync_config_run and schain_record.first_run):
+                sleep=CONFIG_PIPELINE_SLEEP,
+            )
+        )
+    if schain_record.config_version != stream_version or (
+        schain_record.sync_config_run and schain_record.first_run
+    ):
         ConfigFileManager(name).remove_skaled_config()
     else:
         logger.info('Adding skaled task to the pool')
@@ -256,10 +244,11 @@ def create_and_execute_tasks(
                     skale=skale,
                     schain=schain,
                     node_config=node_config,
-                    dutils=dutils
+                    dutils=dutils,
                 ),
-                sleep=SKALED_PIPELINE_SLEEP
-            ))
+                sleep=SKALED_PIPELINE_SLEEP,
+            )
+        )
 
     if len(tasks) == 0:
         logger.warning('No tasks to run')
@@ -267,12 +256,7 @@ def create_and_execute_tasks(
 
 
 def run_monitor_for_schain(
-    skale,
-    skale_ima,
-    node_config: NodeConfig,
-    schain,
-    dutils=None,
-    once=False
+    skale, skale_ima, node_config: NodeConfig, schain, dutils=None, once=False
 ):
     stream_version = get_skale_node_version()
     tasks_number = 2
@@ -290,7 +274,7 @@ def run_monitor_for_schain(
                     schain_record,
                     executor,
                     futures,
-                    dutils
+                    dutils,
                 )
                 if once:
                     return True

From 05872f6955766d8e8539610520ec930194f532d1 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 11 Jul 2024 11:14:48 +0000
Subject: [PATCH 35/42] Fix config monitor tests

---
 tests/schains/monitor/config_monitor_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/schains/monitor/config_monitor_test.py b/tests/schains/monitor/config_monitor_test.py
index 3dc13584f..71fbc2285 100644
--- a/tests/schains/monitor/config_monitor_test.py
+++ b/tests/schains/monitor/config_monitor_test.py
@@ -202,7 +202,7 @@ def test_sync_config_monitor_dkg_not_completed(
     estate.chain_id = 1
     config_checks.estate = estate
     config_am.estate = estate
-    config_checks.last_dkg_successful = False
+    config_checks._last_dkg_successful = False
     assert not econfig.synced(estate)
 
     sync_config_monitor = SyncConfigMonitor(

From b89c4ee03b330fe2725cf99a3ccaa16ba9e79efc Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 11 Jul 2024 11:17:44 +0000
Subject: [PATCH 36/42] Add tests for no_hyphens

---
 tests/helper_test.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/helper_test.py b/tests/helper_test.py
index 762225232..3000af75c 100644
--- a/tests/helper_test.py
+++ b/tests/helper_test.py
@@ -1,4 +1,4 @@
-from tools.helper import is_address_contract
+from tools.helper import is_address_contract, no_hyphens
 from tools.configs.web3 import ZERO_ADDRESS
 
 
@@ -6,3 +6,10 @@ def test_is_address_contract(skale):
     assert not is_address_contract(skale.web3, ZERO_ADDRESS)
     assert is_address_contract(skale.web3, skale.manager.address)
     assert is_address_contract(skale.web3, skale.nodes.address)
+
+
+def test_no_hyphen():
+    no_hyphens('too') == 'too'
+    no_hyphens('too-boo') == 'too_boo'
+    no_hyphens('too-boo_goo') == 'too_boo_goo'
+    no_hyphens('too_goo') == 'too_goo'

From 8d376382949fb5948fa969948988261b980d80d8 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 11 Jul 2024 15:01:53 +0000
Subject: [PATCH 37/42] Fix flake8

---
 core/schains/dkg/client.py | 12 ++++++------
 tests/helper_test.py       |  8 ++++----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/core/schains/dkg/client.py b/core/schains/dkg/client.py
index 584bad0b6..2edb9b54e 100644
--- a/core/schains/dkg/client.py
+++ b/core/schains/dkg/client.py
@@ -89,7 +89,7 @@ def convert_hex_to_g2_array(data):
     g2_array = []
     while len(data) > 0:
         cur = data[:256]
-        g2_array.append([str(x) for x in [int(cur[64 * i : 64 * i + 64], 16) for i in range(4)]])
+        g2_array.append([str(x) for x in [int(cur[64 * i: 64 * i + 64], 16) for i in range(4)]])
         data = data[256:]
     return g2_array
 
@@ -97,14 +97,14 @@ def convert_hex_to_g2_array(data):
 def convert_str_to_key_share(sent_secret_key_contribution, n):
     return_value = []
     for i in range(n):
-        public_key = sent_secret_key_contribution[i * 192 + 64 : (i + 1) * 192]
-        key_share = bytes.fromhex(sent_secret_key_contribution[i * 192 : i * 192 + 64])
+        public_key = sent_secret_key_contribution[i * 192 + 64: (i + 1) * 192]
+        key_share = bytes.fromhex(sent_secret_key_contribution[i * 192: i * 192 + 64])
         return_value.append(KeyShare(public_key, key_share).tuple)
     return return_value
 
 
 def convert_key_share_to_str(data, n):
-    return ''.join(to_verify(s) for s in [data[i * 192 : (i + 1) * 192] for i in range(n)])
+    return ''.join(to_verify(s) for s in [data[i * 192: (i + 1) * 192] for i in range(n)])
 
 
 def to_verify(share):
@@ -197,7 +197,7 @@ def check_complaint_logs(self, logs):
 
     def store_broadcasted_data(self, data, from_node):
         self.incoming_secret_key_contribution[from_node] = data[1][
-            192 * self.node_id_dkg : 192 * (self.node_id_dkg + 1)
+            192 * self.node_id_dkg: 192 * (self.node_id_dkg + 1)
         ]
         if from_node == self.node_id_dkg:
             self.incoming_verification_vector[from_node] = convert_hex_to_g2_array(data[0])
@@ -222,7 +222,7 @@ def secret_key_contribution(self):
             self.poly_name, self.public_keys
         )
         self.incoming_secret_key_contribution[self.node_id_dkg] = self.sent_secret_key_contribution[
-            self.node_id_dkg * 192 : (self.node_id_dkg + 1) * 192
+            self.node_id_dkg * 192: (self.node_id_dkg + 1) * 192
         ]
         return convert_str_to_key_share(self.sent_secret_key_contribution, self.n)
 
diff --git a/tests/helper_test.py b/tests/helper_test.py
index 3000af75c..538690185 100644
--- a/tests/helper_test.py
+++ b/tests/helper_test.py
@@ -9,7 +9,7 @@ def test_is_address_contract(skale):
 
 
 def test_no_hyphen():
-    no_hyphens('too') == 'too'
-    no_hyphens('too-boo') == 'too_boo'
-    no_hyphens('too-boo_goo') == 'too_boo_goo'
-    no_hyphens('too_goo') == 'too_goo'
+    assert no_hyphens('too') == 'too'
+    assert no_hyphens('too-boo') == 'too_boo'
+    assert no_hyphens('too-boo_goo') == 'too_boo_goo'
+    assert no_hyphens('too_goo') == 'too_goo'

From dff661c1ce9e7cd855c0bc53c5eb4933feb89699 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 11 Jul 2024 15:40:47 +0000
Subject: [PATCH 38/42] Rename metrics

---
 core/schains/checks.py         |  2 +-
 core/schains/dkg/client.py     |  2 +-
 core/schains/monitor/action.py |  8 ++++----
 core/schains/monitor/main.py   | 10 +++++-----
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/core/schains/checks.py b/core/schains/checks.py
index 288c3ed4b..08c005645 100644
--- a/core/schains/checks.py
+++ b/core/schains/checks.py
@@ -524,5 +524,5 @@ def log_checks_dict(schain_name, checks_dict):
 
 def send_to_statsd(statsd_client: statsd.StatsClient, schain_name: str, checks_dict: dict) -> None:
     for check, result in checks_dict.items():
-        mname = f'admin.checks.{check}.{no_hyphens(schain_name)}'
+        mname = f'admin.schain_checks.{check}.{no_hyphens(schain_name)}'
         statsd_client.gauge(mname, int(result))
diff --git a/core/schains/dkg/client.py b/core/schains/dkg/client.py
index 2edb9b54e..bf8c0db3a 100644
--- a/core/schains/dkg/client.py
+++ b/core/schains/dkg/client.py
@@ -185,7 +185,7 @@ def last_completed_step(self) -> DKGStep:
     @last_completed_step.setter
     def last_completed_step(self, step: DKGStep):
         self.statsd_client.gauge(
-            f'admin.dkg.last_completed_step.{no_hyphens(self.schain_name)}', step.value
+            f'admin.schains.dkg.last_completed_step.{no_hyphens(self.schain_name)}', step.value
         )
         self._last_completed_step = step
 
diff --git a/core/schains/monitor/action.py b/core/schains/monitor/action.py
index 2560680f1..fb09d98c5 100644
--- a/core/schains/monitor/action.py
+++ b/core/schains/monitor/action.py
@@ -179,7 +179,7 @@ def config_dir(self) -> bool:
     @BaseActionManager.monitor_block
     def dkg(self) -> bool:
         initial_status = self.checks.dkg.status
-        with self.statsd_client.timer(f'admin.dkg.{no_hyphens(self.name)}'):
+        with self.statsd_client.timer(f'admin.action.dkg.{no_hyphens(self.name)}'):
             if not initial_status:
                 logger.info('Initing dkg client')
                 dkg_client = get_dkg_client(
@@ -213,7 +213,7 @@ def dkg(self) -> bool:
 
     @BaseActionManager.monitor_block
     def upstream_config(self) -> bool:
-        with self.statsd_client.timer(f'admin.upstream_config.{no_hyphens(self.name)}'):
+        with self.statsd_client.timer(f'admin.action.upstream_config.{no_hyphens(self.name)}'):
             logger.info(
                 'Creating new upstream_config rotation_id: %s, stream: %s',
                 self.rotation_data.get('rotation_id'), self.stream_version
@@ -349,7 +349,7 @@ def firewall_rules(self, upstream: bool = False) -> bool:
 
             ranges = self.econfig.ranges
             logger.info('Adding ranges %s', ranges)
-            with self.statsd_client.timer(f'admin.firewall.{no_hyphens(self.name)}'):
+            with self.statsd_client.timer(f'admin.action.firewall.{no_hyphens(self.name)}'):
                 self.rc.configure(
                     base_port=base_port,
                     own_ip=own_ip,
@@ -357,7 +357,7 @@ def firewall_rules(self, upstream: bool = False) -> bool:
                     sync_ip_ranges=ranges
                 )
                 self.statsd_client.gauge(
-                    f'admin.expected_rules.{no_hyphens(self.name)}',
+                    f'admin.action.expected_rules.{no_hyphens(self.name)}',
                     len(self.rc.expected_rules())
                 )
                 self.rc.sync()
diff --git a/core/schains/monitor/main.py b/core/schains/monitor/main.py
index 9dd3a46c2..c66cfef50 100644
--- a/core/schains/monitor/main.py
+++ b/core/schains/monitor/main.py
@@ -108,11 +108,11 @@ def run_config_pipeline(
         mon = RegularConfigMonitor(config_am, config_checks)
     statsd_client = get_statsd_client()
 
-    statsd_client.incr(f'admin.config.pipeline.{mon.__class__.__name__}.{no_hyphens(name)}')
+    statsd_client.incr(f'admin.config_pipeline.{mon.__class__.__name__}.{no_hyphens(name)}')
     statsd_client.gauge(
-        f'admin.schain.rotation_id.{no_hyphens(name)}', rotation_data['rotation_id']
+        f'admin.config_pipeline.rotation_id.{no_hyphens(name)}', rotation_data['rotation_id']
     )
-    with statsd_client.timer(f'admin.config.pipeline.duration.{no_hyphens(name)}'):
+    with statsd_client.timer(f'admin.config_pipeline.duration.{no_hyphens(name)}'):
         mon.run()
 
 
@@ -161,8 +161,8 @@ def run_skaled_pipeline(
     )
 
     statsd_client = get_statsd_client()
-    statsd_client.incr(f'schain.skaled.pipeline.{mon.__name__}.{no_hyphens(name)}')
-    with statsd_client.timer(f'admin.skaled.pipeline.duration.{no_hyphens(name)}'):
+    statsd_client.incr(f'schain.skaled_pipeline.{mon.__name__}.{no_hyphens(name)}')
+    with statsd_client.timer(f'admin.skaled_pipeline.duration.{no_hyphens(name)}'):
         mon(skaled_am, skaled_checks).run()
 
 

From 4f8e1979bdfdd5225f6e9c29a5b6fe436e4c45d6 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 11 Jul 2024 16:31:27 +0000
Subject: [PATCH 39/42] Fix tests

---
 core/schains/checks.py                             |  2 ++
 core/schains/cleaner.py                            |  2 ++
 tests/conftest.py                                  |  1 +
 tests/schains/checks_test.py                       | 13 +++++++++++++
 tests/schains/cleaner_test.py                      |  1 +
 tests/schains/monitor/action/config_action_test.py |  1 +
 web/routes/health.py                               |  1 +
 7 files changed, 21 insertions(+)

diff --git a/core/schains/checks.py b/core/schains/checks.py
index 71a1946e1..d3a171c01 100644
--- a/core/schains/checks.py
+++ b/core/schains/checks.py
@@ -414,6 +414,7 @@ def __init__(
         stream_version: str,
         estate: ExternalState,
         current_nodes: list[ExtendedManagerNodeInfo],
+        last_dkg_successful: bool,
         rotation_id: int = 0,
         *,
         econfig: Optional[ExternalConfig] = None,
@@ -428,6 +429,7 @@ def __init__(
                 rotation_id=rotation_id,
                 stream_version=stream_version,
                 current_nodes=current_nodes,
+                last_dkg_successful=last_dkg_successful,
                 estate=estate,
                 econfig=econfig,
                 sync_node=sync_node,
diff --git a/core/schains/cleaner.py b/core/schains/cleaner.py
index 387f1861d..985089db6 100644
--- a/core/schains/cleaner.py
+++ b/core/schains/cleaner.py
@@ -224,6 +224,7 @@ def cleanup_schain(
     schain_name,
     sync_agent_ranges,
     rotation_id,
+    last_dkg_successful,
     current_nodes,
     estate,
     dutils=None
@@ -245,6 +246,7 @@ def cleanup_schain(
         current_nodes=current_nodes,
         rotation_id=rotation_id,
         estate=estate,
+        last_dkg_successful=last_dkg_successful,
         dutils=dutils,
         sync_node=SYNC_NODE
     )
diff --git a/tests/conftest.py b/tests/conftest.py
index 807884c44..8b34c172f 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -513,6 +513,7 @@ def schain_checks(schain_config, schain_db, current_nodes, rule_controller, esta
         rule_controller=rule_controller,
         stream_version=CONFIG_STREAM,
         current_nodes=current_nodes,
+        last_dkg_successful=True,
         estate=estate,
         dutils=dutils
     )
diff --git a/tests/schains/checks_test.py b/tests/schains/checks_test.py
index 95bb85950..b2cf591e2 100644
--- a/tests/schains/checks_test.py
+++ b/tests/schains/checks_test.py
@@ -96,6 +96,7 @@ def sample_false_checks(schain_config, schain_db, rule_controller, current_nodes
         schain_record=schain_record,
         rule_controller=rule_controller,
         stream_version=CONFIG_STREAM,
+        last_dkg_successful=True,
         current_nodes=current_nodes,
         estate=estate,
         dutils=dutils
@@ -120,6 +121,7 @@ def rules_unsynced_checks(
         rule_controller=uninited_rule_controller,
         stream_version=CONFIG_STREAM,
         current_nodes=current_nodes,
+        last_dkg_successful=True,
         estate=estate,
         dutils=dutils
     )
@@ -288,6 +290,7 @@ def test_init_checks(skale, schain_db, current_nodes, uninited_rule_controller,
         rule_controller=uninited_rule_controller,
         stream_version=CONFIG_STREAM,
         current_nodes=current_nodes,
+        last_dkg_successful=True,
         estate=estate,
         dutils=dutils
     )
@@ -316,6 +319,7 @@ def test_exit_code(skale, rule_controller, schain_db, current_nodes, estate, dut
             rule_controller=rule_controller,
             stream_version=CONFIG_STREAM,
             current_nodes=current_nodes,
+            last_dkg_successful=True,
             estate=estate,
             dutils=dutils
         )
@@ -335,6 +339,7 @@ def test_process(skale, rule_controller, schain_db, current_nodes, estate, dutil
         rule_controller=rule_controller,
         stream_version=CONFIG_STREAM,
         current_nodes=current_nodes,
+        last_dkg_successful=True,
         estate=estate,
         dutils=dutils
     )
@@ -359,6 +364,7 @@ def test_get_all(schain_config, rule_controller, dutils, current_nodes, schain_d
         rule_controller=rule_controller,
         stream_version=CONFIG_STREAM,
         current_nodes=current_nodes,
+        last_dkg_successful=True,
         estate=estate,
         dutils=dutils
     )
@@ -381,6 +387,7 @@ def test_get_all(schain_config, rule_controller, dutils, current_nodes, schain_d
         rule_controller=rule_controller,
         stream_version=CONFIG_STREAM,
         current_nodes=current_nodes,
+        last_dkg_successful=True,
         estate=estate,
         dutils=dutils
     )
@@ -408,6 +415,7 @@ def test_get_all_with_save(node_config, rule_controller, current_nodes, dutils,
         rule_controller=rule_controller,
         stream_version=CONFIG_STREAM,
         current_nodes=current_nodes,
+        last_dkg_successful=True,
         estate=estate,
         dutils=dutils
     )
@@ -432,9 +440,11 @@ def test_config_updated(skale, rule_controller, schain_db, current_nodes, estate
         rule_controller=rule_controller,
         stream_version=CONFIG_STREAM,
         current_nodes=current_nodes,
+        last_dkg_successful=True,
         estate=estate,
         dutils=dutils
     )
+    assert checks.last_dkg_successful is False
     assert checks.config_updated
 
     upstream_path = UpstreamConfigFilename(
@@ -453,6 +463,7 @@ def test_config_updated(skale, rule_controller, schain_db, current_nodes, estate
         rule_controller=rule_controller,
         stream_version=CONFIG_STREAM,
         current_nodes=current_nodes,
+        last_dkg_successful=True,
         estate=estate,
         dutils=dutils
     )
@@ -466,7 +477,9 @@ def test_config_updated(skale, rule_controller, schain_db, current_nodes, estate
         rule_controller=rule_controller,
         stream_version=CONFIG_STREAM,
         current_nodes=current_nodes,
+        last_dkg_successful=True,
         estate=estate,
         dutils=dutils
     )
+    assert checks.last_dkg_successful is True
     assert not checks.config_updated
diff --git a/tests/schains/cleaner_test.py b/tests/schains/cleaner_test.py
index e8319474c..4db2e9f45 100644
--- a/tests/schains/cleaner_test.py
+++ b/tests/schains/cleaner_test.py
@@ -255,6 +255,7 @@ def test_cleanup_schain(
         schain_name,
         current_nodes=current_nodes,
         sync_agent_ranges=[],
+        last_dkg_successful=True,
         rotation_id=0,
         estate=estate,
         dutils=dutils
diff --git a/tests/schains/monitor/action/config_action_test.py b/tests/schains/monitor/action/config_action_test.py
index 771769727..57c904ade 100644
--- a/tests/schains/monitor/action/config_action_test.py
+++ b/tests/schains/monitor/action/config_action_test.py
@@ -37,6 +37,7 @@ def config_checks(
         schain_record=schain_record,
         rotation_id=rotation_data['rotation_id'],
         stream_version=CONFIG_STREAM,
+        last_dkg_successful=True,
         current_nodes=current_nodes,
         estate=estate
     )
diff --git a/web/routes/health.py b/web/routes/health.py
index 56c5258e5..2503d674b 100644
--- a/web/routes/health.py
+++ b/web/routes/health.py
@@ -112,6 +112,7 @@ def schains_checks():
                     rotation_id=rotation_id,
                     stream_version=stream_version,
                     current_nodes=current_nodes,
+                    last_dkg_successful=True,
                     estate=estate,
                     sync_node=False
                 ).get_all(needed=checks_filter)

From 6310af162ab56819f9f6acb1450ea226280aa878 Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 11 Jul 2024 18:08:11 +0000
Subject: [PATCH 40/42] Fix config updated check

---
 tests/schains/checks_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/schains/checks_test.py b/tests/schains/checks_test.py
index b2cf591e2..2e86f4dca 100644
--- a/tests/schains/checks_test.py
+++ b/tests/schains/checks_test.py
@@ -440,11 +440,11 @@ def test_config_updated(skale, rule_controller, schain_db, current_nodes, estate
         rule_controller=rule_controller,
         stream_version=CONFIG_STREAM,
         current_nodes=current_nodes,
-        last_dkg_successful=True,
+        last_dkg_successful=False,
         estate=estate,
         dutils=dutils
     )
-    assert checks.last_dkg_successful is False
+    assert checks.last_dkg_successful.status is False
     assert checks.config_updated
 
     upstream_path = UpstreamConfigFilename(
@@ -481,5 +481,5 @@ def test_config_updated(skale, rule_controller, schain_db, current_nodes, estate
         estate=estate,
         dutils=dutils
     )
-    assert checks.last_dkg_successful is True
+    assert checks.last_dkg_successful.status is True
     assert not checks.config_updated

From 7a87e7c6d3d45fbac7cc0f99a4a8a266ced84f6e Mon Sep 17 00:00:00 2001
From: badrogger <ivan@skalelabs.com>
Date: Thu, 11 Jul 2024 18:26:22 +0000
Subject: [PATCH 41/42] Fix skaled_pipeline metric name

---
 core/schains/monitor/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/schains/monitor/main.py b/core/schains/monitor/main.py
index c66cfef50..c60e09c49 100644
--- a/core/schains/monitor/main.py
+++ b/core/schains/monitor/main.py
@@ -161,7 +161,7 @@ def run_skaled_pipeline(
     )
 
     statsd_client = get_statsd_client()
-    statsd_client.incr(f'schain.skaled_pipeline.{mon.__name__}.{no_hyphens(name)}')
+    statsd_client.incr(f'admin.skaled_pipeline.{mon.__name__}.{no_hyphens(name)}')
     with statsd_client.timer(f'admin.skaled_pipeline.duration.{no_hyphens(name)}'):
         mon(skaled_am, skaled_checks).run()
 

From 584b2e49f8eed0cd8fd5d46ba63de5eb0c26907a Mon Sep 17 00:00:00 2001
From: Dmytro Nazarenko <dmytro.nazarenko@skalelabs.com>
Date: Thu, 11 Jul 2024 21:31:44 +0100
Subject: [PATCH 42/42] Bump config-controller

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 664cd2b48..595de5f91 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,7 +14,7 @@ requests==2.31
 ima-predeployed==2.1.0b0
 etherbase-predeployed==1.1.0b3
 marionette-predeployed==2.0.0b2
-config-controller-predeployed==1.1.0b0
+config-controller-predeployed==1.1.0
 filestorage-predeployed==1.1.0.dev8
 multisigwallet-predeployed==1.1.0a8