diff --git a/core/schains/checks.py b/core/schains/checks.py index d3a171c0..8f4b0d9e 100644 --- a/core/schains/checks.py +++ b/core/schains/checks.py @@ -59,7 +59,7 @@ from tools.configs.containers import IMA_CONTAINER, SCHAIN_CONTAINER from tools.docker_utils import DockerUtils -from tools.helper import write_json +from tools.helper import no_hyphens, write_json from tools.resources import get_statsd_client from tools.str_formatters import arguments_list_string @@ -504,5 +504,5 @@ def log_checks_dict(schain_name, checks_dict): def send_to_statsd(statsd_client: statsd.StatsClient, schain_name: str, checks_dict: dict) -> None: for check, result in checks_dict.items(): - mname = f'admin.checks.{schain_name}.{check}' + mname = f'admin.schain_checks.{check}.{no_hyphens(schain_name)}' statsd_client.gauge(mname, int(result)) diff --git a/core/schains/dkg/client.py b/core/schains/dkg/client.py index 28c00159..bf8c0db3 100644 --- a/core/schains/dkg/client.py +++ b/core/schains/dkg/client.py @@ -29,6 +29,7 @@ from core.schains.dkg.broadcast_filter import Filter from core.schains.dkg.structures import ComplaintReason, DKGStep +from tools.helper import no_hyphens from tools.configs import NODE_DATA_PATH, SGX_CERTIFICATES_FOLDER from tools.resources import get_statsd_client from tools.sgx_utils import sgx_unreachable_retry @@ -78,7 +79,7 @@ def convert_g2_point_to_hex(data): data_hexed = '' for coord in data: temp = hex(int(coord))[2:] - while (len(temp) < 64): + while len(temp) < 64: temp = '0' + temp data_hexed += temp return data_hexed @@ -88,7 +89,7 @@ def convert_hex_to_g2_array(data): g2_array = [] while len(data) > 0: cur = data[:256] - g2_array.append([str(x) for x in [int(cur[64 * i:64 * i + 64], 16) for i in range(4)]]) + g2_array.append([str(x) for x in [int(cur[64 * i: 64 * i + 64], 16) for i in range(4)]]) data = data[256:] return g2_array @@ -103,7 +104,7 @@ def convert_str_to_key_share(sent_secret_key_contribution, n): def convert_key_share_to_str(data, n): - return "".join(to_verify(s) for s in [data[i * 192:(i + 1) * 192] for i in range(n)]) + return ''.join(to_verify(s) for s in [data[i * 192: (i + 1) * 192] for i in range(n)]) def to_verify(share): @@ -112,24 +113,24 @@ def to_verify(share): def generate_poly_name(group_index_str, node_id, dkg_id): return ( - "POLY:SCHAIN_ID:" - f"{group_index_str}" - ":NODE_ID:" - f"{str(node_id)}" - ":DKG_ID:" - f"{str(dkg_id)}" - ) + 'POLY:SCHAIN_ID:' + f'{group_index_str}' + ':NODE_ID:' + f'{str(node_id)}' + ':DKG_ID:' + f'{str(dkg_id)}' + ) def generate_bls_key_name(group_index_str, node_id, dkg_id): return ( - "BLS_KEY:SCHAIN_ID:" - f"{group_index_str}" - ":NODE_ID:" - f"{str(node_id)}" - ":DKG_ID:" - f"{str(dkg_id)}" - ) + 'BLS_KEY:SCHAIN_ID:' + f'{group_index_str}' + ':NODE_ID:' + f'{str(node_id)}' + ':DKG_ID:' + f'{str(dkg_id)}' + ) class DKGClient: @@ -146,10 +147,11 @@ def __init__( node_ids_contract, eth_key_name, rotation_id, - step: DKGStep = DKGStep.NONE + step: DKGStep = DKGStep.NONE, ): - self.sgx = SgxClient(os.environ['SGX_SERVER_URL'], n=n, t=t, - path_to_cert=SGX_CERTIFICATES_FOLDER) + self.sgx = SgxClient( + os.environ['SGX_SERVER_URL'], n=n, t=t, path_to_cert=SGX_CERTIFICATES_FOLDER + ) self.schain_name = schain_name self.group_index = skale.schains.name_to_group_id(schain_name) self.node_id_contract = node_id_contract @@ -169,9 +171,9 @@ def __init__( self.node_ids_contract = node_ids_contract self.dkg_contract_functions = self.skale.dkg.contract.functions self.dkg_timeout = self.skale.constants_holder.get_dkg_timeout() - self.complaint_error_event_hash = self.skale.web3.to_hex(self.skale.web3.keccak( - text="ComplaintError(string)" - )) + self.complaint_error_event_hash = self.skale.web3.to_hex( + self.skale.web3.keccak(text='ComplaintError(string)') + ) self.statsd_client = get_statsd_client() self._last_completed_step = step # last step logger.info(f'sChain: {self.schain_name}. DKG timeout is {self.dkg_timeout}') @@ -181,9 +183,11 @@ def last_completed_step(self) -> DKGStep: return self._last_completed_step @last_completed_step.setter - def last_completed_step(self, value: DKGStep): - self.statsd_client.gauge(f'admin.dkg.last_completed_step.{self.schain_name}', value) - self._last_completed_step = value + def last_completed_step(self, step: DKGStep): + self.statsd_client.gauge( + f'admin.schains.dkg.last_completed_step.{no_hyphens(self.schain_name)}', step.value + ) + self._last_completed_step = step def is_channel_opened(self): return self.skale.dkg.is_channel_opened(self.group_index) @@ -214,9 +218,9 @@ def verification_vector(self): @sgx_unreachable_retry def secret_key_contribution(self): - self.sent_secret_key_contribution = self.sgx.get_secret_key_contribution_v2(self.poly_name, - self.public_keys - ) + self.sent_secret_key_contribution = self.sgx.get_secret_key_contribution_v2( + self.poly_name, self.public_keys + ) self.incoming_secret_key_contribution[self.node_id_dkg] = self.sent_secret_key_contribution[ self.node_id_dkg * 192: (self.node_id_dkg + 1) * 192 ] @@ -233,12 +237,14 @@ def broadcast(self): ) is_broadcast_possible = self.skale.dkg.contract.functions.isBroadcastPossible( - self.group_index, self.node_id_contract).call({'from': self.skale.wallet.address}) + self.group_index, self.node_id_contract + ).call({'from': self.skale.wallet.address}) channel_opened = self.is_channel_opened() if not is_broadcast_possible or not channel_opened: - logger.info(f'sChain: {self.schain_name}. ' - f'{self.node_id_dkg} node could not sent broadcast') + logger.info( + f'sChain: {self.schain_name}. ' f'{self.node_id_dkg} node could not sent broadcast' + ) return verification_vector = self.verification_vector() @@ -249,7 +255,7 @@ def broadcast(self): self.node_id_contract, verification_vector, secret_key_contribution, - self.rotation_id + self.rotation_id, ) self.last_completed_step = DKGStep.BROADCAST logger.info('Everything is sent from %d node', self.node_id_dkg) @@ -262,28 +268,31 @@ def receive_from_node(self, from_node, broadcasted_data): try: if not self.verification(from_node): raise DkgVerificationError( - f"sChain: {self.schain_name}. " - f"Fatal error : user {str(from_node + 1)} " + f'sChain: {self.schain_name}. ' + f'Fatal error : user {str(from_node + 1)} ' f"hasn't passed verification by user {str(self.node_id_dkg + 1)}" ) - logger.info(f'sChain: {self.schain_name}. ' - f'All data from {from_node} was received and verified') + logger.info( + f'sChain: {self.schain_name}. ' + f'All data from {from_node} was received and verified' + ) except SgxUnreachableError as e: raise SgxUnreachableError( - f"sChain: {self.schain_name}. " - f"Fatal error : user {str(from_node + 1)} " - f"hasn't passed verification by user {str(self.node_id_dkg + 1)}" - f"with SgxUnreachableError: ", e - ) + f'sChain: {self.schain_name}. ' + f'Fatal error : user {str(from_node + 1)} ' + f"hasn't passed verification by user {str(self.node_id_dkg + 1)}" + f'with SgxUnreachableError: ', + e, + ) @sgx_unreachable_retry def verification(self, from_node): - return self.sgx.verify_secret_share_v2(self.incoming_verification_vector[from_node], - self.eth_key_name, - to_verify( - self.incoming_secret_key_contribution[from_node] - ), - self.node_id_dkg) + return self.sgx.verify_secret_share_v2( + self.incoming_verification_vector[from_node], + self.eth_key_name, + to_verify(self.incoming_secret_key_contribution[from_node]), + self.node_id_dkg, + ) @sgx_unreachable_retry def is_bls_key_generated(self): @@ -298,17 +307,20 @@ def is_bls_key_generated(self): @sgx_unreachable_retry def generate_bls_key(self): - received_secret_key_contribution = "".join(to_verify( - self.incoming_secret_key_contribution[j] - ) - for j in range(self.sgx.n)) - logger.info(f'sChain: {self.schain_name}. ' - f'DKGClient is going to create BLS private key with name {self.bls_name}') - bls_private_key = self.sgx.create_bls_private_key_v2(self.poly_name, self.bls_name, - self.eth_key_name, - received_secret_key_contribution) - logger.info(f'sChain: {self.schain_name}. ' - 'DKGClient is going to fetch BLS public key with name {self.bls_name}') + received_secret_key_contribution = ''.join( + to_verify(self.incoming_secret_key_contribution[j]) for j in range(self.sgx.n) + ) + logger.info( + f'sChain: {self.schain_name}. ' + f'DKGClient is going to create BLS private key with name {self.bls_name}' + ) + bls_private_key = self.sgx.create_bls_private_key_v2( + self.poly_name, self.bls_name, self.eth_key_name, received_secret_key_contribution + ) + logger.info( + f'sChain: {self.schain_name}. ' + 'DKGClient is going to fetch BLS public key with name {self.bls_name}' + ) self.public_key = self.sgx.get_bls_public_key(self.bls_name) return bls_private_key @@ -326,72 +338,70 @@ def get_bls_public_keys(self): def alright(self): logger.info(f'sChain {self.schain_name} sending alright transaction') is_alright_possible = self.skale.dkg.is_alright_possible( - self.group_index, self.node_id_contract, self.skale.wallet.address) + self.group_index, self.node_id_contract, self.skale.wallet.address + ) if not is_alright_possible or not self.is_channel_opened(): - logger.info(f'sChain: {self.schain_name}. ' - f'{self.node_id_dkg} node could not sent an alright note') + logger.info( + f'sChain: {self.schain_name}. ' + f'{self.node_id_dkg} node could not sent an alright note' + ) return self.skale.dkg.alright( - self.group_index, - self.node_id_contract, - gas_limit=ALRIGHT_GAS_LIMIT, - multiplier=2 + self.group_index, self.node_id_contract, gas_limit=ALRIGHT_GAS_LIMIT, multiplier=2 ) self.last_completed_step = DKGStep.ALRIGHT logger.info(f'sChain: {self.schain_name}. {self.node_id_dkg} node sent an alright note') def send_complaint(self, to_node: int, reason: ComplaintReason): - logger.info(f'sChain: {self.schain_name}. ' - f'{self.node_id_dkg} node is trying to sent a {reason} on {to_node} node') + logger.info( + f'sChain: {self.schain_name}. ' + f'{self.node_id_dkg} node is trying to sent a {reason} on {to_node} node' + ) is_complaint_possible = self.skale.dkg.is_complaint_possible( - self.group_index, self.node_id_contract, self.node_ids_dkg[to_node], - self.skale.wallet.address + self.group_index, + self.node_id_contract, + self.node_ids_dkg[to_node], + self.skale.wallet.address, ) is_channel_opened = self.is_channel_opened() logger.info( - 'Complaint possible %s, channel opened %s', - is_complaint_possible, - is_channel_opened + 'Complaint possible %s, channel opened %s', is_complaint_possible, is_channel_opened ) if not is_complaint_possible or not is_channel_opened: - logger.info( - '%d node could not sent a complaint on %d node', - self.node_id_dkg, - to_node - ) + logger.info('%d node could not sent a complaint on %d node', self.node_id_dkg, to_node) return False reason_to_step = { ComplaintReason.NO_BROADCAST: DKGStep.COMPLAINT_NO_BROADCAST, ComplaintReason.BAD_DATA: DKGStep.COMPLAINT_BAD_DATA, ComplaintReason.NO_ALRIGHT: DKGStep.COMPLAINT_NO_ALRIGHT, - ComplaintReason.NO_RESPONSE: DKGStep.COMPLAINT_NO_RESPONSE + ComplaintReason.NO_RESPONSE: DKGStep.COMPLAINT_NO_RESPONSE, } try: if reason == ComplaintReason.BAD_DATA: tx_res = self.skale.dkg.complaint_bad_data( - self.group_index, - self.node_id_contract, - self.node_ids_dkg[to_node] + self.group_index, self.node_id_contract, self.node_ids_dkg[to_node] ) else: tx_res = self.skale.dkg.complaint( - self.group_index, - self.node_id_contract, - self.node_ids_dkg[to_node] + self.group_index, self.node_id_contract, self.node_ids_dkg[to_node] ) if self.check_complaint_logs(tx_res.receipt['logs'][0]): - logger.info(f'sChain: {self.schain_name}. ' - f'{self.node_id_dkg} node sent a complaint on {to_node} node') + logger.info( + f'sChain: {self.schain_name}. ' + f'{self.node_id_dkg} node sent a complaint on {to_node} node' + ) self.last_completed_step = reason_to_step[reason] return True else: - logger.info(f'sChain: {self.schain_name}. Complaint from {self.node_id_dkg} on ' - f'{to_node} node was rejected') + logger.info( + f'sChain: {self.schain_name}. Complaint from {self.node_id_dkg} on ' + f'{to_node} node was rejected' + ) return False except TransactionFailedError as e: logger.error(f'DKG complaint failed: sChain {self.schain_name}') @@ -400,8 +410,7 @@ def send_complaint(self, to_node: int, reason: ComplaintReason): @sgx_unreachable_retry def get_complaint_response(self, to_node_index): response = self.sgx.complaint_response( - self.poly_name, - self.node_ids_contract[to_node_index] + self.poly_name, self.node_ids_contract[to_node_index] ) share, dh_key = response.share, response.dh_key verification_vector_mult = response.verification_vector_mult @@ -413,11 +422,13 @@ def get_complaint_response(self, to_node_index): def response(self, to_node_index): is_pre_response_possible = self.skale.dkg.is_pre_response_possible( - self.group_index, self.node_id_contract, self.skale.wallet.address) + self.group_index, self.node_id_contract, self.skale.wallet.address + ) if not is_pre_response_possible or not self.is_channel_opened(): - logger.info(f'sChain: {self.schain_name}. ' - f'{self.node_id_dkg} node could not sent a response') + logger.info( + f'sChain: {self.schain_name}. ' f'{self.node_id_dkg} node could not sent a response' + ) return share, dh_key, verification_vector_mult = self.get_complaint_response(to_node_index) @@ -428,24 +439,22 @@ def response(self, to_node_index): self.node_id_contract, convert_g2_points_to_array(self.incoming_verification_vector[self.node_id_dkg]), convert_g2_points_to_array(verification_vector_mult), - convert_str_to_key_share(self.sent_secret_key_contribution, self.n) + convert_str_to_key_share(self.sent_secret_key_contribution, self.n), ) self.last_completed_step = DKGStep.PRE_RESPONSE is_response_possible = self.skale.dkg.is_response_possible( - self.group_index, self.node_id_contract, self.skale.wallet.address) + self.group_index, self.node_id_contract, self.skale.wallet.address + ) if not is_response_possible or not self.is_channel_opened(): - logger.info(f'sChain: {self.schain_name}. ' - f'{self.node_id_dkg} node could not sent a response') + logger.info( + f'sChain: {self.schain_name}. ' + f'{self.node_id_dkg} node could not sent a response' + ) return - self.skale.dkg.response( - self.group_index, - self.node_id_contract, - int(dh_key, 16), - share - ) + self.skale.dkg.response(self.group_index, self.node_id_contract, int(dh_key, 16), share) self.last_completed_step = DKGStep.RESPONSE logger.info(f'sChain: {self.schain_name}. {self.node_id_dkg} node sent a response') except TransactionFailedError as e: @@ -461,8 +470,7 @@ def fetch_all_broadcasted_data(self): broadcasted_data = [event.verificationVector, event.secretKeyContribution] self.store_broadcasted_data(broadcasted_data, from_node) logger.info( - f'sChain: {self.schain_name}. Received by {self.node_id_dkg} from ' - f'{from_node}' + f'sChain: {self.schain_name}. Received by {self.node_id_dkg} from ' f'{from_node}' ) def is_all_data_received(self, from_node): diff --git a/core/schains/monitor/action.py b/core/schains/monitor/action.py index 88871e46..fb09d98c 100644 --- a/core/schains/monitor/action.py +++ b/core/schains/monitor/action.py @@ -78,6 +78,7 @@ from tools.configs import SYNC_NODE from tools.configs.containers import IMA_CONTAINER, SCHAIN_CONTAINER from tools.docker_utils import DockerUtils +from tools.helper import no_hyphens from tools.node_options import NodeOptions from tools.notifications.messages import notify_repair_mode from tools.resources import get_statsd_client @@ -178,7 +179,7 @@ def config_dir(self) -> bool: @BaseActionManager.monitor_block def dkg(self) -> bool: initial_status = self.checks.dkg.status - with self.statsd_client.timer(f'admin.dkg.{self.name}'): + with self.statsd_client.timer(f'admin.action.dkg.{no_hyphens(self.name)}'): if not initial_status: logger.info('Initing dkg client') dkg_client = get_dkg_client( @@ -212,7 +213,7 @@ def dkg(self) -> bool: @BaseActionManager.monitor_block def upstream_config(self) -> bool: - with self.statsd_client.timer(f'admin.upstream_config.{self.name}'): + with self.statsd_client.timer(f'admin.action.upstream_config.{no_hyphens(self.name)}'): logger.info( 'Creating new upstream_config rotation_id: %s, stream: %s', self.rotation_data.get('rotation_id'), self.stream_version @@ -348,7 +349,7 @@ def firewall_rules(self, upstream: bool = False) -> bool: ranges = self.econfig.ranges logger.info('Adding ranges %s', ranges) - with self.statsd_client.timer(f'admin.firewall.{self.name}'): + with self.statsd_client.timer(f'admin.action.firewall.{no_hyphens(self.name)}'): self.rc.configure( base_port=base_port, own_ip=own_ip, @@ -356,7 +357,7 @@ def firewall_rules(self, upstream: bool = False) -> bool: sync_ip_ranges=ranges ) self.statsd_client.gauge( - f'admin.expected_rules.{self.name}', + f'admin.action.expected_rules.{no_hyphens(self.name)}', len(self.rc.expected_rules()) ) self.rc.sync() diff --git a/core/schains/monitor/main.py b/core/schains/monitor/main.py index 5030b8c7..58010d34 100644 --- a/core/schains/monitor/main.py +++ b/core/schains/monitor/main.py @@ -46,7 +46,7 @@ from tools.docker_utils import DockerUtils from tools.configs import SYNC_NODE from tools.notifications.messages import notify_checks -from tools.helper import is_node_part_of_chain +from tools.helper import is_node_part_of_chain, no_hyphens from tools.resources import get_statsd_client from web.models.schain import SChainRecord @@ -115,9 +115,11 @@ def run_config_pipeline( mon = RegularConfigMonitor(config_am, config_checks) statsd_client = get_statsd_client() - statsd_client.incr(f'admin.config.pipeline.{name}.{mon.__class__.__name__}') - statsd_client.gauge(f'admin.schain.rotation_id.{name}', rotation_data['rotation_id']) - with statsd_client.timer(f'admin.config.pipeline.{name}.duration'): + statsd_client.incr(f'admin.config_pipeline.{mon.__class__.__name__}.{no_hyphens(name)}') + statsd_client.gauge( + f'admin.config_pipeline.rotation_id.{no_hyphens(name)}', rotation_data['rotation_id'] + ) + with statsd_client.timer(f'admin.config_pipeline.duration.{no_hyphens(name)}'): mon.run() @@ -167,8 +169,8 @@ def run_skaled_pipeline( ) statsd_client = get_statsd_client() - statsd_client.incr(f'schain.skaled.pipeline.{name}.{mon.__name__}') - with statsd_client.timer(f'admin.skaled.pipeline.{name}.duration'): + statsd_client.incr(f'admin.skaled_pipeline.{mon.__name__}.{no_hyphens(name)}') + with statsd_client.timer(f'admin.skaled_pipeline.duration.{no_hyphens(name)}'): mon(skaled_am, skaled_checks).run() @@ -210,8 +212,8 @@ def create_and_execute_tasks( statsd_client = get_statsd_client() monitor_last_seen_ts = schain_record.monitor_last_seen.timestamp() - statsd_client.incr(f'admin.schain.monitor.{name}') - statsd_client.gauge(f'admin.schain.monitor_last_seen.{name}', monitor_last_seen_ts) + statsd_client.incr(f'admin.schain.monitor.{no_hyphens(name)}') + statsd_client.gauge(f'admin.schain.monitor_last_seen.{no_hyphens(name)}', monitor_last_seen_ts) tasks = [] if not leaving_chain: diff --git a/core/schains/monitor/post_rotation_monitor.py b/core/schains/monitor/post_rotation_monitor.py deleted file mode 100644 index 8200ab5a..00000000 --- a/core/schains/monitor/post_rotation_monitor.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of SKALE Admin -# -# Copyright (C) 2021 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import logging - -from core.schains.monitor.base_monitor import BaseMonitor - - -logger = logging.getLogger(__name__) - - -class PostRotationMonitor(BaseMonitor): - """ - PostRotationMonitor be executed for the sChain on the staying node when rotation is complete. - This type of monitor reloads skaled container. - """ - @BaseMonitor.monitor_runner - def run(self): - logger.info(f'{self.p} was stopped after rotation. Going to restart') - self.config(overwrite=True) - self.firewall_rules() - self.recreated_schain_containers() diff --git a/requirements.txt b/requirements.txt index 664cd2b4..b39330d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ peewee==3.9.5 Flask==2.3.3 -Werkzeug==2.3.7 -gunicorn==20.1.0 +Werkzeug==3.0.3 +gunicorn==22.0.0 Jinja2==3.1.2 diff --git a/tests/helper_test.py b/tests/helper_test.py index 76222523..53869018 100644 --- a/tests/helper_test.py +++ b/tests/helper_test.py @@ -1,4 +1,4 @@ -from tools.helper import is_address_contract +from tools.helper import is_address_contract, no_hyphens from tools.configs.web3 import ZERO_ADDRESS @@ -6,3 +6,10 @@ def test_is_address_contract(skale): assert not is_address_contract(skale.web3, ZERO_ADDRESS) assert is_address_contract(skale.web3, skale.manager.address) assert is_address_contract(skale.web3, skale.nodes.address) + + +def test_no_hyphen(): + assert no_hyphens('too') == 'too' + assert no_hyphens('too-boo') == 'too_boo' + assert no_hyphens('too-boo_goo') == 'too_boo_goo' + assert no_hyphens('too_goo') == 'too_goo' diff --git a/tools/helper.py b/tools/helper.py index 8cfc28c3..71b788f4 100644 --- a/tools/helper.py +++ b/tools/helper.py @@ -184,3 +184,7 @@ def is_zero_address(address: str) -> bool: def is_address_contract(web3, address) -> bool: """Returns true if contract is deployed at the requested address""" return web3.eth.get_code(address) != b'' + + +def no_hyphens(name: str) -> str: + return name.replace('-', '_')