diff --git a/core/schains/cleaner.py b/core/schains/cleaner.py index 6b99f3ec0..dcf4b52b8 100644 --- a/core/schains/cleaner.py +++ b/core/schains/cleaner.py @@ -238,14 +238,16 @@ def cleanup_schain( dutils=dutils, sync_node=SYNC_NODE, ) - status = checks.get_all() - if status['skaled_container'] or is_exited( - schain_name, container_type=ContainerType.schain, dutils=dutils + check_status = checks.get_all() + if check_status['skaled_container'] or is_exited( + schain_name, + container_type=ContainerType.schain, + dutils=dutils ): remove_schain_container(schain_name, dutils=dutils) - if status['volume']: + if check_status['volume']: remove_schain_volume(schain_name, dutils=dutils) - if status['firewall_rules']: + if check_status['firewall_rules']: conf = ConfigFileManager(schain_name).skaled_config base_port = get_base_port_from_config(conf) own_ip = get_own_ip_from_config(conf) @@ -256,11 +258,13 @@ def cleanup_schain( rc.configure(base_port=base_port, own_ip=own_ip, node_ips=node_ips, sync_ip_ranges=ranges) rc.cleanup() if estate is not None and estate.ima_linked: - if status.get('ima_container', False) or is_exited( - schain_name, container_type=ContainerType.ima, dutils=dutils + if check_status.get('ima_container', False) or is_exited( + schain_name, + container_type=ContainerType.ima, + dutils=dutils ): remove_ima_container(schain_name, dutils=dutils) - if status['config_dir']: + if check_status['config_dir']: remove_config_dir(schain_name) mark_schain_deleted(schain_name) diff --git a/core/schains/cmd.py b/core/schains/cmd.py index c96e2432e..3df08f628 100644 --- a/core/schains/cmd.py +++ b/core/schains/cmd.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Optional + from core.schains.config.file_manager import ConfigFileManager from core.schains.config.helper import get_schain_ports_from_config from core.schains.config.main import get_skaled_container_config_path @@ -34,7 +36,7 @@ def get_schain_container_cmd( download_snapshot: bool = False, enable_ssl: bool = True, sync_node: bool = False, - snapshot_from: str = '' + snapshot_from: Optional[str] = None ) -> str: """Returns parameters that will be passed to skaled binary in the sChain container""" opts = get_schain_container_base_opts(schain_name, enable_ssl=enable_ssl, sync_node=sync_node) diff --git a/core/schains/config/directory.py b/core/schains/config/directory.py index 8d2c7a66d..ce11c8196 100644 --- a/core/schains/config/directory.py +++ b/core/schains/config/directory.py @@ -25,10 +25,11 @@ from tools.configs.schains import ( BASE_SCHAIN_CONFIG_FILEPATH, + NODE_CLI_STATUS_FILENAME, SCHAINS_DIR_PATH, SCHAINS_DIR_PATH_HOST, SCHAIN_SCHECKS_FILENAME, - SKALED_STATUS_FILENAME + SKALED_STATUS_FILENAME, ) @@ -58,6 +59,10 @@ def skaled_status_filepath(name: str) -> str: return os.path.join(schain_config_dir(name), SKALED_STATUS_FILENAME) +def node_cli_status_filepath(name: str) -> str: + return os.path.join(schain_config_dir(name), NODE_CLI_STATUS_FILENAME) + + def get_schain_check_filepath(schain_name): schain_dir_path = schain_config_dir(schain_name) return os.path.join(schain_dir_path, SCHAIN_SCHECKS_FILENAME) diff --git a/core/schains/config/static_params.py b/core/schains/config/static_params.py index 83d140c12..62e83761d 100644 --- a/core/schains/config/static_params.py +++ b/core/schains/config/static_params.py @@ -21,13 +21,15 @@ from core.schains.config.helper import get_static_params from tools.configs import ENV_TYPE +from typing import Optional + def get_static_schain_cmd(env_type: str = ENV_TYPE) -> list: static_params = get_static_params(env_type) return static_params['schain_cmd'] -def get_static_schain_info(schain_name: str, env_type: str = ENV_TYPE) -> dict | None: +def get_static_schain_info(schain_name: str, env_type: str = ENV_TYPE) -> Optional[dict]: static_params = get_static_params(env_type) static_params_schain = static_params['schain'] processed_params = {} @@ -36,7 +38,7 @@ def get_static_schain_info(schain_name: str, env_type: str = ENV_TYPE) -> dict | return processed_params -def get_schain_static_param(static_param_schain: dict | int, schain_name: str) -> int: +def get_schain_static_param(static_param_schain: dict, schain_name: str) -> int: if isinstance(static_param_schain, int): return static_param_schain elif isinstance(static_param_schain, dict) and schain_name in static_param_schain: diff --git a/core/schains/monitor/action.py b/core/schains/monitor/action.py index 1002bf8a9..b02cbab44 100644 --- a/core/schains/monitor/action.py +++ b/core/schains/monitor/action.py @@ -35,20 +35,18 @@ run_dkg, save_dkg_results ) -from core.schains.ima import get_migration_ts as get_ima_migration_ts from core.schains.cleaner import ( remove_ima_container, remove_schain_container, remove_schain_volume ) +from core.schains.ima import get_migration_ts as get_ima_migration_ts, ImaData +from core.schains.status import NodeCliStatus from core.schains.firewall.types import IRuleController - from core.schains.volume import init_data_volume from core.schains.exit_scheduler import ExitScheduleFileManager - from core.schains.limits import get_schain_type - from core.schains.monitor.containers import monitor_schain_container, monitor_ima_container from core.schains.monitor.rpc import handle_failed_schain_rpc from core.schains.runner import ( @@ -70,9 +68,8 @@ get_node_ips_from_config, get_own_ip_from_config ) -from core.schains.ima import ImaData from core.schains.external_config import ExternalConfig, ExternalState -from core.schains.skaled_status import init_skaled_status +from core.schains.status import init_skaled_status from core.schains.ssl import update_ssl_change_date from tools.configs import SYNC_NODE @@ -299,6 +296,7 @@ def __init__( rule_controller: IRuleController, checks: SkaledChecks, node_config: NodeConfig, + ncli_status: NodeCliStatus, econfig: Optional[ExternalConfig] = None, dutils: DockerUtils = None, node_options: NodeOptions = None @@ -321,6 +319,7 @@ def __init__( self.statsd_client = get_statsd_client() self.node_options = node_options or NodeOptions() + self.ncli_status = ncli_status super().__init__(name=schain.name) @@ -375,11 +374,13 @@ def skaled_container( download_snapshot, start_ts ) + snapshot_from = self.ncli_status.snapshot_from if self.ncli_status else None monitor_schain_container( self.schain, schain_record=self.schain_record, skaled_status=self.skaled_status, download_snapshot=download_snapshot, + snapshot_from=snapshot_from, start_ts=start_ts, abort_on_exit=abort_on_exit, dutils=self.dutils, @@ -556,4 +557,11 @@ def notify_repair_mode(self) -> None: @BaseActionManager.monitor_block def disable_repair_mode(self) -> None: logger.info('Switching off repair mode') - self.schain_record.set_repair_mode(False) + if self.schain_record.repair_mode: + self.schain_record.set_repair_mode(False) + + @BaseActionManager.monitor_block + def update_repair_ts(self, new_ts: int) -> None: + logger.info('Setting repair_ts to %d', new_ts) + new_dt = datetime.utcfromtimestamp(new_ts) + self.schain_record.set_repair_date(new_dt) diff --git a/core/schains/monitor/containers.py b/core/schains/monitor/containers.py index 311525766..d3c2142cd 100644 --- a/core/schains/monitor/containers.py +++ b/core/schains/monitor/containers.py @@ -56,6 +56,7 @@ def monitor_schain_container( skaled_status, download_snapshot=False, start_ts=None, + snapshot_from: Optional[str] = None, abort_on_exit: bool = True, dutils: Optional[DockerUtils] = None, sync_node: bool = False, @@ -83,7 +84,7 @@ def monitor_schain_container( download_snapshot=download_snapshot, start_ts=start_ts, dutils=dutils, - snapshot_from=schain_record.snapshot_from, + snapshot_from=snapshot_from, sync_node=sync_node, historic_state=historic_state, ) diff --git a/core/schains/monitor/main.py b/core/schains/monitor/main.py index 4d44b74bc..0975ab23c 100644 --- a/core/schains/monitor/main.py +++ b/core/schains/monitor/main.py @@ -41,7 +41,7 @@ from core.schains.external_config import ExternalConfig, ExternalState from core.schains.task import keep_tasks_running, Task from core.schains.config.static_params import get_automatic_repair_option -from core.schains.skaled_status import get_skaled_status +from core.schains.status import get_node_cli_status, get_skaled_status from core.node import get_current_nodes from tools.docker_utils import DockerUtils @@ -141,30 +141,33 @@ def run_skaled_pipeline( ) skaled_status = get_skaled_status(name) + ncli_status = get_node_cli_status(name) skaled_am = SkaledActionManager( schain=schain, rule_controller=rc, checks=skaled_checks, node_config=node_config, + ncli_status=ncli_status, econfig=ExternalConfig(name), dutils=dutils, ) - status = skaled_checks.get_all(log=False, expose=True) + check_status = skaled_checks.get_all(log=False, expose=True) automatic_repair = get_automatic_repair_option() - api_status = get_api_checks_status(status=status, allowed=TG_ALLOWED_CHECKS) + api_status = get_api_checks_status(status=check_status, allowed=TG_ALLOWED_CHECKS) notify_checks(name, node_config.all(), api_status) - logger.info('Skaled status: %s', status) + logger.info('Skaled check status: %s', check_status) logger.info('Upstream config %s', skaled_am.upstream_config_path) mon = get_skaled_monitor( action_manager=skaled_am, - status=status, + check_status=check_status, schain_record=schain_record, skaled_status=skaled_status, - automatic_repair=automatic_repair, + ncli_status=ncli_status, + automatic_repair=automatic_repair ) statsd_client = get_statsd_client() diff --git a/core/schains/monitor/skaled_monitor.py b/core/schains/monitor/skaled_monitor.py index aac40e16a..a946ca2be 100644 --- a/core/schains/monitor/skaled_monitor.py +++ b/core/schains/monitor/skaled_monitor.py @@ -26,7 +26,7 @@ from core.schains.checks import SkaledChecks from core.schains.monitor.action import SkaledActionManager from core.schains.config.main import get_number_of_secret_shares -from core.schains.skaled_status import SkaledStatus +from core.schains.status import NodeCliStatus, SkaledStatus from core.schains.ssl import ssl_reload_needed from tools.configs import SYNC_NODE from tools.resources import get_statsd_client @@ -37,11 +37,7 @@ class BaseSkaledMonitor(IMonitor): - def __init__( - self, - action_manager: SkaledActionManager, - checks: SkaledChecks - ) -> None: + def __init__(self, action_manager: SkaledActionManager, checks: SkaledChecks) -> None: self.am = action_manager self.checks = checks self.statsd_client = get_statsd_client() @@ -64,7 +60,6 @@ def run(self): class RegularSkaledMonitor(BaseSkaledMonitor): - def execute(self) -> None: if not self.checks.firewall_rules: self.am.firewall_rules() @@ -90,7 +85,7 @@ def execute(self) -> None: logger.warning( 'Repair mode execution, record: %s, exit_code_ok: %s', self.checks.schain_record.repair_mode, - self.checks.exit_code_ok.status + self.checks.exit_code_ok.status, ) self.am.notify_repair_mode() self.am.cleanup_schain_docker_entity() @@ -102,7 +97,7 @@ def execute(self) -> None: self.am.skaled_container(download_snapshot=True) else: self.am.reset_restart_count() - self.am.disable_repair_mode() + self.am.update_repair_ts(new_ts=int(time.time())) class BackupSkaledMonitor(BaseSkaledMonitor): @@ -223,10 +218,7 @@ def execute(self): if not self.checks.firewall_rules: self.am.firewall_rules() if not self.checks.skaled_container: - self.am.skaled_container( - download_snapshot=True, - start_ts=self.am.finish_ts - ) + self.am.skaled_container(download_snapshot=True, start_ts=self.am.finish_ts) else: self.am.reset_restart_counter() if not self.checks.ima_container: @@ -239,36 +231,34 @@ def is_backup_mode(schain_record: SChainRecord) -> bool: def is_repair_mode( schain_record: SChainRecord, - status: Dict, + check_status: Dict, skaled_status: Optional[SkaledStatus], - automatic_repair: bool + ncli_status: Optional[NodeCliStatus], + automatic_repair: bool, ) -> bool: - if schain_record.repair_mode: + repair_ts = int(schain_record.repair_date.timestamp()) + if ncli_status is not None and ncli_status.repair_ts > repair_ts: return True - else: - return automatic_repair and is_skaled_repair_status(status, skaled_status) + return automatic_repair and is_skaled_repair_internal(check_status, skaled_status) -def is_reload_group_mode(status: Dict, finish_ts: Optional[int]) -> bool: +def is_reload_group_mode(check_status: Dict, finish_ts: Optional[int]) -> bool: ts = int(time.time()) if finish_ts is None: return False - return finish_ts > ts and status['config'] and not status['config_updated'] + return finish_ts > ts and check_status['config'] and not check_status['config_updated'] -def is_reload_ip_mode(status: Dict, reload_ts: Optional[int]) -> bool: +def is_reload_ip_mode(check_status: Dict, reload_ts: Optional[int]) -> bool: if reload_ts is None: return False - return status['config'] and not status['config_updated'] + return check_status['config'] and not check_status['config_updated'] -def is_config_update_time( - status: Dict, - skaled_status: Optional[SkaledStatus] -) -> bool: +def is_config_update_time(check_status: Dict, skaled_status: Optional[SkaledStatus]) -> bool: if not skaled_status: return False - return not status['skaled_container'] and skaled_status.exit_time_reached + return not check_status['skaled_container'] and skaled_status.exit_time_reached def is_recreate_mode(status: Dict, schain_record: SChainRecord) -> bool: @@ -283,24 +273,25 @@ def is_new_node_mode(schain_record: SChainRecord, finish_ts: Optional[int]) -> b return finish_ts > ts and secret_shares_number == 1 -def is_skaled_repair_status(status: Dict, skaled_status: Optional[SkaledStatus]) -> bool: +def is_skaled_repair_internal(check_status: Dict, skaled_status: Optional[SkaledStatus]) -> bool: if skaled_status is None: return False skaled_status.log() needs_repair = skaled_status.clear_data_dir and skaled_status.start_from_snapshot - return not status['skaled_container'] and needs_repair + return not check_status['skaled_container'] and needs_repair -def no_config(status: Dict) -> bool: - return not status['config'] +def no_config(check_status: Dict) -> bool: + return not check_status['config'] def get_skaled_monitor( action_manager: SkaledActionManager, - status: Dict, + check_status: Dict, schain_record: SChainRecord, skaled_status: SkaledStatus, - automatic_repair: bool = True + ncli_status: NodeCliStatus, + automatic_repair: bool = True, ) -> Type[BaseSkaledMonitor]: logger.info('Choosing skaled monitor') if skaled_status: @@ -309,32 +300,32 @@ def get_skaled_monitor( mon_type: Type[BaseSkaledMonitor] = RegularSkaledMonitor if SYNC_NODE: - if no_config(status): + if no_config(check_status): mon_type = NoConfigSkaledMonitor - if is_recreate_mode(status, schain_record): + if is_recreate_mode(check_status, schain_record): mon_type = RecreateSkaledMonitor - elif is_config_update_time(status, skaled_status): + elif is_config_update_time(check_status, skaled_status): mon_type = UpdateConfigSkaledMonitor - elif is_reload_group_mode(status, action_manager.upstream_finish_ts): + elif is_reload_group_mode(check_status, action_manager.upstream_finish_ts): mon_type = ReloadGroupSkaledMonitor - elif is_reload_ip_mode(status, action_manager.econfig.reload_ts): + elif is_reload_ip_mode(check_status, action_manager.econfig.reload_ts): mon_type = ReloadIpSkaledMonitor return mon_type - if no_config(status): + if no_config(check_status): mon_type = NoConfigSkaledMonitor elif is_backup_mode(schain_record): mon_type = BackupSkaledMonitor - elif is_repair_mode(schain_record, status, skaled_status, automatic_repair): + elif is_repair_mode(schain_record, check_status, skaled_status, ncli_status, automatic_repair): mon_type = RepairSkaledMonitor - elif is_recreate_mode(status, schain_record): + elif is_recreate_mode(check_status, schain_record): mon_type = RecreateSkaledMonitor elif is_new_node_mode(schain_record, action_manager.finish_ts): mon_type = NewNodeSkaledMonitor - elif is_config_update_time(status, skaled_status): + elif is_config_update_time(check_status, skaled_status): mon_type = UpdateConfigSkaledMonitor - elif is_reload_group_mode(status, action_manager.upstream_finish_ts): + elif is_reload_group_mode(check_status, action_manager.upstream_finish_ts): mon_type = ReloadGroupSkaledMonitor - elif is_reload_ip_mode(status, action_manager.econfig.reload_ts): + elif is_reload_ip_mode(check_status, action_manager.econfig.reload_ts): mon_type = ReloadIpSkaledMonitor return mon_type diff --git a/core/schains/runner.py b/core/schains/runner.py index 14d0304e2..50b5c3867 100644 --- a/core/schains/runner.py +++ b/core/schains/runner.py @@ -19,6 +19,7 @@ import copy import logging +from typing import Optional from docker.types import LogConfig, Ulimit from skale.contracts.manager.schains import SchainStructure @@ -183,7 +184,7 @@ def run_schain_container( volume_mode=None, ulimit_check=True, enable_ssl=True, - snapshot_from: str = '', + snapshot_from: Optional[str] = None, sync_node=False, historic_state=False ): diff --git a/core/schains/skaled_status.py b/core/schains/status.py similarity index 65% rename from core/schains/skaled_status.py rename to core/schains/status.py index 02186a4a9..2b40b2271 100644 --- a/core/schains/skaled_status.py +++ b/core/schains/status.py @@ -22,22 +22,52 @@ import logging from json.decoder import JSONDecodeError from typing import Optional +from abc import ABCMeta, abstractmethod -from core.schains.config.directory import skaled_status_filepath +from core.schains.config.directory import node_cli_status_filepath, skaled_status_filepath from tools.config_utils import config_getter, log_broken_status_file from tools.helper import read_json logger = logging.getLogger(__name__) -class SkaledStatus: - def __init__(self, filepath: str): +class IStatus(metaclass=ABCMeta): + @abstractmethod + def __init__(self, filepath: str) -> None: + pass + + @property + @abstractmethod + def filepath(self) -> str: + pass + + @property + def all(self) -> dict: + if not os.path.isfile(self.filepath): + logger.warning("File %s is not found", self.filepath) + return + try: + return read_json(self.filepath) + except JSONDecodeError: + log_broken_status_file(self.filepath) + return {} + + def log(self) -> None: + logger.info(f'{self.__class__.__name__}: \n' + json.dumps(self.all, indent=4)) + + +class SkaledStatus(IStatus): + def __init__(self, filepath: str) -> None: """ Read-only wrapper for skaled.status file, reads from the file each time. Returns dict for top-level keys, True or False for second-level keys. Returns None for all keys if file is not found. """ - self.filepath = filepath + self._filepath = filepath + + @property + def filepath(self) -> str: + return self._filepath @property @config_getter @@ -84,28 +114,48 @@ def start_from_snapshot(self) -> bool: return return exit_state['StartFromSnapshot'] + +class NodeCliStatus(IStatus): + def __init__(self, filepath: str) -> None: + """ + Read-only wrapper for node_cli.status file, reads from the file each time. + """ + self._filepath = filepath + @property - def all(self) -> dict: - if not os.path.isfile(self.filepath): - logger.warning("File %s is not found", self.filepath) - return - try: - return read_json(self.filepath) - except JSONDecodeError: - log_broken_status_file(self.filepath) - return {} + @config_getter + def repair_ts(self) -> int: + return 'repair_ts', self.filepath - def log(self) -> None: - logger.info('skaled status file: \n' + json.dumps(self.all, indent=4)) + @property + @config_getter + def snapshot_from(self) -> int: + return 'snapshot_from', self.filepath + + @property + def filepath(self) -> str: + return self._filepath -def init_skaled_status(schain_name) -> SkaledStatus: +def init_skaled_status(schain_name: str) -> SkaledStatus: status_filepath = skaled_status_filepath(schain_name) return SkaledStatus(status_filepath) -def get_skaled_status(schain_name) -> Optional[SkaledStatus]: +def get_skaled_status(schain_name: str) -> Optional[SkaledStatus]: status_path = skaled_status_filepath(schain_name) if os.path.isfile(status_path): return SkaledStatus(status_path) return None + + +def init_node_cli_status(schain_name: str) -> SkaledStatus: + status_filepath = node_cli_status_filepath(schain_name) + return NodeCliStatus(status_filepath) + + +def get_node_cli_status(schain_name: str) -> Optional[SkaledStatus]: + status_path = node_cli_status_filepath(schain_name) + if os.path.isfile(status_path): + return NodeCliStatus(status_path) + return None diff --git a/tests/conftest.py b/tests/conftest.py index 8b34c172f..691671fdd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,9 +15,7 @@ from skale import SkaleManager from skale.wallets import Web3Wallet from skale.utils.account_tools import generate_account, send_eth -from skale.utils.contracts_provision.fake_multisig_contract import ( - deploy_fake_multisig_contract -) +from skale.utils.contracts_provision.fake_multisig_contract import deploy_fake_multisig_contract from skale.utils.contracts_provision.main import ( add_test_permissions, add_test2_schain_type, @@ -27,7 +25,7 @@ create_nodes, create_schain, link_nodes_to_validator, - setup_validator + setup_validator, ) from skale.utils.web3_utils import init_web3 @@ -38,13 +36,18 @@ from core.schains.config.helper import ( get_base_port_from_config, get_node_ips_from_config, - get_own_ip_from_config + get_own_ip_from_config, ) from core.schains.config.directory import schain_config_dir, skaled_status_filepath from core.schains.cleaner import remove_schain_container, remove_schain_volume from core.schains.ima import ImaData from core.schains.external_config import ExternalConfig, ExternalState -from core.schains.skaled_status import init_skaled_status, SkaledStatus +from core.schains.status import ( + init_node_cli_status, + init_skaled_status, + node_cli_status_filepath, + SkaledStatus, +) from core.schains.config.skale_manager_opts import SkaleManagerOpts from tools.configs import CONFIG_FOLDER, ENV_TYPE, META_FILEPATH, SSL_CERTIFICATES_FILEPATH @@ -60,6 +63,7 @@ from tests.utils import ( ALLOWED_RANGES, CONFIG_STREAM, + CURRENT_TS, ENDPOINT, ETH_AMOUNT_PER_NODE, ETH_PRIVATE_KEY, @@ -69,7 +73,7 @@ IMA_MIGRATION_TS, init_skale_from_wallet, init_skale_ima, - upsert_schain_record_with_config + upsert_schain_record_with_config, ) NUMBER_OF_NODES = 2 @@ -81,14 +85,8 @@ def images(): cinfo = {} with open(CONTAINERS_FILEPATH, 'r') as cf: json.load(cinfo, cf) - schain_image = '{}/{}'.format( - cinfo['schain']['name'], - cinfo['schain']['version'] - ) - ima_image = '{}/{}'.format( - cinfo['ima']['name'], - cinfo['ima']['version'] - ) + schain_image = '{}/{}'.format(cinfo['schain']['name'], cinfo['schain']['version']) + ima_image = '{}/{}'.format(cinfo['ima']['name'], cinfo['ima']['version']) dclient.images.pull(schain_image) dclient.images.pull(ima_image) @@ -104,14 +102,14 @@ def predeployed_ima(): @pytest.fixture(scope='session') def web3(): - """ Returns a SKALE Manager instance with provider from config """ + """Returns a SKALE Manager instance with provider from config""" w3 = init_web3(ENDPOINT) return w3 @pytest.fixture(scope='session') def skale(web3): - """ Returns a SKALE Manager instance with provider from config """ + """Returns a SKALE Manager instance with provider from config""" wallet = Web3Wallet(ETH_PRIVATE_KEY, web3) skale_obj = init_skale_from_wallet(wallet) add_test_permissions(skale_obj) @@ -139,7 +137,7 @@ def node_wallets(skale): web3=skale.web3, wallet=skale.wallet, receiver_address=wallet.address, - amount=ETH_AMOUNT_PER_NODE + amount=ETH_AMOUNT_PER_NODE, ) wallets.append(wallet) return wallets @@ -147,10 +145,7 @@ def node_wallets(skale): @pytest.fixture def node_skales(skale, node_wallets): - return [ - SkaleManager(ENDPOINT, ABI_FILEPATH, wallet) - for wallet in node_wallets - ] + return [SkaleManager(ENDPOINT, ABI_FILEPATH, wallet) for wallet in node_wallets] @pytest.fixture @@ -171,10 +166,7 @@ def skale_ima(): @pytest.fixture def ssl_folder(): - pathlib.Path(SSL_CERTIFICATES_FILEPATH).mkdir( - parents=True, - exist_ok=True - ) + pathlib.Path(SSL_CERTIFICATES_FILEPATH).mkdir(parents=True, exist_ok=True) try: yield SSL_CERTIFICATES_FILEPATH finally: @@ -203,63 +195,63 @@ def get_skaled_status_dict( exit_time_reached=False, clear_data_dir=False, start_from_snapshot=False, - start_again=False + start_again=False, ): return { - "subsystemRunning": { - "SnapshotDownloader": snapshot_downloader, - "Blockchain": False, - "Rpc": False + 'subsystemRunning': { + 'SnapshotDownloader': snapshot_downloader, + 'Blockchain': False, + 'Rpc': False, + }, + 'exitState': { + 'ClearDataDir': clear_data_dir, + 'StartAgain': start_again, + 'StartFromSnapshot': start_from_snapshot, + 'ExitTimeReached': exit_time_reached, }, - "exitState": { - "ClearDataDir": clear_data_dir, - "StartAgain": start_again, - "StartFromSnapshot": start_from_snapshot, - "ExitTimeReached": exit_time_reached - } } SECRET_KEY = { - "common_public_key": [ + 'common_public_key': [ 11111111111111111111111111111111111111111111111111111111111111111111111111111, 1111111111111111111111111111111111111111111111111111111111111111111111111111, 1111111111111111111111111111111111111111111111111111111111111111111111111111, - 11111111111111111111111111111111111111111111111111111111111111111111111111111 + 11111111111111111111111111111111111111111111111111111111111111111111111111111, ], - "public_key": [ - "1111111111111111111111111111111111111111111111111111111111111111111111111111", - "1111111111111111111111111111111111111111111111111111111111111111111111111111", - "1111111111111111111111111111111111111111111111111111111111111111111111111111", - "11111111111111111111111111111111111111111111111111111111111111111111111111111" + 'public_key': [ + '1111111111111111111111111111111111111111111111111111111111111111111111111111', + '1111111111111111111111111111111111111111111111111111111111111111111111111111', + '1111111111111111111111111111111111111111111111111111111111111111111111111111', + '11111111111111111111111111111111111111111111111111111111111111111111111111111', ], - "bls_public_keys": [ - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111" # noqa + 'bls_public_keys': [ + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa + '1111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111:11111111111111111111111111111111111111111111111111111111111111111111111111111', # noqa ], - "t": 11, - "n": 16, - "key_share_name": "BLS_KEY:SCHAIN_ID:33333333333333333333333333333333333333333333333333333333333333333333333333333:NODE_ID:0:DKG_ID:0" # noqa + 't': 11, + 'n': 16, + 'key_share_name': 'BLS_KEY:SCHAIN_ID:33333333333333333333333333333333333333333333333333333333333333333333333333333:NODE_ID:0:DKG_ID:0', # noqa } @pytest.fixture def _schain_name(): - """ Generates default schain name """ + """Generates default schain name""" return get_random_string() @@ -295,8 +287,7 @@ def secret_keys(_schain_name): @pytest.fixture def schain_config(_schain_name, secret_key, predeployed_ima): schain_dir_path = os.path.join(SCHAINS_DIR_PATH, _schain_name) - config_path = os.path.join(schain_dir_path, - f'schain_{_schain_name}.json') + config_path = os.path.join(schain_dir_path, f'schain_{_schain_name}.json') try: pathlib.Path(schain_dir_path).mkdir(parents=True, exist_ok=True) schain_config = generate_schain_config(_schain_name) @@ -349,8 +340,7 @@ def skaled_status_exit_time_reached(_schain_name): @pytest.fixture def skaled_status_repair(_schain_name): - generate_schain_skaled_status_file( - _schain_name, clear_data_dir=True, start_from_snapshot=True) + generate_schain_skaled_status_file(_schain_name, clear_data_dir=True, start_from_snapshot=True) try: yield init_skaled_status(_schain_name) finally: @@ -371,7 +361,7 @@ def skaled_status_broken_file(_schain_name): schain_dir_path = os.path.join(SCHAINS_DIR_PATH, _schain_name) pathlib.Path(schain_dir_path).mkdir(parents=True, exist_ok=True) status_filepath = skaled_status_filepath(_schain_name) - with open(status_filepath, "w") as text_file: + with open(status_filepath, 'w') as text_file: text_file.write('abcd') try: yield SkaledStatus(status_filepath) @@ -390,18 +380,14 @@ def db(): @pytest.fixture def schain_db(db, _schain_name, meta_file): - """ Database with default schain inserted """ + """Database with default schain inserted""" upsert_schain_record_with_config(_schain_name) return _schain_name @pytest.fixture def meta_file(): - meta_info = { - "version": "0.0.0", - "config_stream": CONFIG_STREAM, - "docker_lvmpy_stream": "1.1.1" - } + meta_info = {'version': '0.0.0', 'config_stream': CONFIG_STREAM, 'docker_lvmpy_stream': '1.1.1'} with open(META_FILEPATH, 'w') as meta_file: json.dump(meta_info, meta_file) try: @@ -416,7 +402,7 @@ def schain_on_contracts(skale, nodes, _schain_name): yield create_schain( skale, schain_type=1, # test2 should have 1 index - schain_name=_schain_name + schain_name=_schain_name, ) finally: cleanup_nodes_schains(skale) @@ -424,25 +410,14 @@ def schain_on_contracts(skale, nodes, _schain_name): @pytest.fixture def dutils(): - return DockerUtils( - volume_driver='local', - host='unix://var/run/docker.sock' - ) + return DockerUtils(volume_driver='local', host='unix://var/run/docker.sock') @pytest.fixture def skaled_mock_image(scope='module'): - dutils = DockerUtils( - volume_driver='local', - host='unix://var/run/docker.sock' - ) + dutils = DockerUtils(volume_driver='local', host='unix://var/run/docker.sock') name = 'skaled-mock' - dutils.client.images.build( - tag=name, - rm=True, - nocache=True, - path='tests/skaled-mock' - ) + dutils.client.images.build(tag=name, rm=True, nocache=True, path='tests/skaled-mock') yield name dutils.client.images.remove(name, force=True) @@ -515,18 +490,14 @@ def schain_checks(schain_config, schain_db, current_nodes, rule_controller, esta current_nodes=current_nodes, last_dkg_successful=True, estate=estate, - dutils=dutils + dutils=dutils, ) @pytest.fixture def schain_struct(schain_config): schain_name = schain_config['skaleConfig']['sChain']['schainName'] - return { - 'name': schain_name, - 'partOfNode': 0, - 'generation': 0 - } + return {'name': schain_name, 'partOfNode': 0, 'generation': 0} @pytest.fixture @@ -540,10 +511,7 @@ def rule_controller(_schain_name, schain_db, schain_config): own_ip = get_own_ip_from_config(schain_config) node_ips = get_node_ips_from_config(schain_config) return get_test_rule_controller( - name=_schain_name, - base_port=base_port, - own_ip=own_ip, - node_ips=node_ips + name=_schain_name, base_port=base_port, own_ip=own_ip, node_ips=node_ips ) @@ -560,10 +528,7 @@ def uninited_rule_controller(_schain_name): @pytest.fixture def skale_manager_opts(): - return SkaleManagerOpts( - schains_internal_address='0x1656', - nodes_address='0x7742' - ) + return SkaleManagerOpts(schains_internal_address='0x1656', nodes_address='0x7742') @pytest.fixture @@ -580,11 +545,7 @@ def new_upstream(schain_db): @pytest.fixture def estate(skale): - return ExternalState( - ima_linked=True, - chain_id=skale.web3.eth.chain_id, - ranges=ALLOWED_RANGES - ) + return ExternalState(ima_linked=True, chain_id=skale.web3.eth.chain_id, ranges=ALLOWED_RANGES) @pytest.fixture @@ -610,7 +571,7 @@ def upstreams(schain_db, schain_config): f'schain_{name}_9_1687183335.json', f'schain_{name}_11_1687183336.json', f'schain_{name}_11_1687183337.json', - f'schain_{name}_11_1687183339.json' + f'schain_{name}_11_1687183339.json', ] try: for fname in files: @@ -632,3 +593,19 @@ def ima_migration_schedule(schain_db): yield migration_schedule_path finally: os.remove(migration_schedule_path) + + +NCLI_STATUS_DICT = {'repair_ts': CURRENT_TS, 'snapshot_from': '127.0.0.1'} + + +@pytest.fixture +def ncli_status(_schain_name): + schain_dir_path = os.path.join(SCHAINS_DIR_PATH, _schain_name) + pathlib.Path(schain_dir_path).mkdir(parents=True, exist_ok=True) + ncli_status_path = node_cli_status_filepath(_schain_name) + write_json(ncli_status_path, NCLI_STATUS_DICT) + + try: + yield init_node_cli_status(_schain_name) + finally: + shutil.rmtree(schain_dir_path, ignore_errors=True) diff --git a/tests/db_test.py b/tests/db_test.py index 40ede0ca5..9c0f7417d 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -7,8 +7,6 @@ get_schains_statuses, mark_schain_deleted, set_schains_first_run, - switch_off_repair_mode, - toggle_schain_repair_mode, SChainRecord, upsert_schain_record ) @@ -67,44 +65,6 @@ def test_schains_first_run(db, upsert_db): SChainRecord.first_run == True).count() == RECORDS_NUMBER # noqa: E712 -def test_toggle_repair_mode(db, upsert_db): - result = toggle_schain_repair_mode('schain-0') - assert result - assert SChainRecord.select().where( - SChainRecord.repair_mode == True).count() == 1 # noqa: E712 - cursor = SChainRecord.select().where( - SChainRecord.repair_mode == True).execute() # noqa: E712 - records = list(cursor) - assert len(records) == 1 - assert records[0].name == 'schain-0' - assert records[0].snapshot_from == '' - - result = toggle_schain_repair_mode('schain-0', '1.1.1.1') - cursor = SChainRecord.select().where( - SChainRecord.repair_mode == True).execute() # noqa: E712 - records = list(cursor) - assert len(records) == 1 - assert records[0].name == 'schain-0' - assert records[0].snapshot_from == '1.1.1.1' - - switch_off_repair_mode('schain-0') - assert SChainRecord.select().where( - SChainRecord.repair_mode == True).count() == 0 # noqa: E712 - cursor = SChainRecord.select().where( - SChainRecord.name == 'schain-0').execute() # noqa: E712 - records = list(cursor) - assert records[0].name == 'schain-0' - assert not records[0].repair_mode - assert records[0].snapshot_from == '' - - -def test_toggle_repair_mode_schain_not_exists(db, upsert_db): - result = toggle_schain_repair_mode('undefined-schain') - assert not result - assert SChainRecord.select().where( - SChainRecord.repair_mode == True).count() == 0 # noqa: E712 - - def test_get_schains_names(db, upsert_db): mark_schain_deleted('schain-0') result = get_schains_names() diff --git a/tests/migrations_test.py b/tests/migrations_test.py index c744e6250..4a084919f 100644 --- a/tests/migrations_test.py +++ b/tests/migrations_test.py @@ -16,7 +16,8 @@ add_monitor_id_field, add_config_version_field, add_restart_count_field, - add_ssl_change_date_field + add_ssl_change_date_field, + add_repair_date_field ) @@ -118,3 +119,9 @@ def test_add_ssl_change_date_field(upserted_db, migrator, model): add_ssl_change_date_field(upserted_db, migrator) for r in model.select().execute(): r.ssl_change_date < datetime.now() + + +def test_add_repair_date_field(upserted_db, migrator, model): + add_repair_date_field(upserted_db, migrator) + for r in model.select().execute(): + r.repair_date < datetime.now() diff --git a/tests/routes/schains_test.py b/tests/routes/schains_test.py index 329987985..9395005dd 100644 --- a/tests/routes/schains_test.py +++ b/tests/routes/schains_test.py @@ -10,7 +10,7 @@ from core.node_config import NodeConfig from core.schains.config.file_manager import ConfigFileManager -from tests.utils import get_bp_data, get_test_rule_controller, post_bp_data +from tests.utils import get_bp_data, get_test_rule_controller from web.models.schain import SChainRecord, upsert_schain_record from web.routes.schains import schains_bp from web.helper import get_api_url @@ -97,39 +97,6 @@ def test_firewall_rules_route(skale_bp, schain_config): } -def test_enable_repair_mode(skale_bp, schain_db): - schain_name = schain_db - data = post_bp_data(skale_bp, get_api_url(BLUEPRINT_NAME, 'repair'), - params={'schain_name': schain_name}) - assert data == { - 'payload': {}, - 'status': 'ok' - } - r = upsert_schain_record(schain_name) - assert r.repair_mode - assert r.snapshot_from == '' - - data = post_bp_data( - skale_bp, - get_api_url(BLUEPRINT_NAME, 'repair'), - params={'schain_name': schain_name, 'snapshot_from': '1.1.1.1'} - ) - assert data == { - 'payload': {}, - 'status': 'ok' - } - r = upsert_schain_record(schain_name) - assert r.repair_mode - assert r.snapshot_from == '1.1.1.1' - - data = post_bp_data(skale_bp, get_api_url(BLUEPRINT_NAME, 'repair'), - params={'schain_name': 'undefined-schain'}) - assert data == { - 'payload': 'No schain with name undefined-schain', - 'status': 'error' - } - - def test_get_schain( skale_bp, skale, diff --git a/tests/schains/monitor/action/skaled_action_test.py b/tests/schains/monitor/action/skaled_action_test.py index d02b18037..970dc04cc 100644 --- a/tests/schains/monitor/action/skaled_action_test.py +++ b/tests/schains/monitor/action/skaled_action_test.py @@ -46,6 +46,7 @@ def monitor_schain_container_mock( schain_record, skaled_status, download_snapshot=False, + snapshot_from='', start_ts=None, abort_on_exit=True, dutils=None, @@ -92,6 +93,7 @@ def skaled_am( secret_key, ssl_folder, ima_migration_schedule, + ncli_status, dutils, skaled_checks ): @@ -102,6 +104,7 @@ def skaled_am( rule_controller=rule_controller, checks=skaled_checks, node_config=node_config, + ncli_status=ncli_status, dutils=dutils ) @@ -145,6 +148,7 @@ def test_skaled_container_with_snapshot_action(skaled_am): schain_record=skaled_am.schain_record, skaled_status=skaled_am.skaled_status, download_snapshot=True, + snapshot_from='127.0.0.1', start_ts=None, abort_on_exit=True, dutils=skaled_am.dutils, @@ -174,6 +178,7 @@ def test_skaled_container_snapshot_delay_start_action(skaled_am): start_ts=ts, abort_on_exit=True, dutils=skaled_am.dutils, + snapshot_from='127.0.0.1', sync_node=False, historic_state=False ) @@ -482,3 +487,12 @@ def test_firewall_rules_action(skaled_am, skaled_checks, rule_controller, econfi SChainRule(port=10009), SChainRule(port=10010, first_ip='127.0.0.2', last_ip='127.0.0.2') ] + + +def test_disable_repair_mode(skaled_am): + skaled_am.schain_record.set_repair_mode(True) + assert skaled_am.schain_record.repair_mode + skaled_am.disable_repair_mode() + assert not skaled_am.schain_record.repair_mode + skaled_am.disable_repair_mode() + assert not skaled_am.schain_record.repair_mode diff --git a/tests/schains/monitor/skaled_monitor_test.py b/tests/schains/monitor/skaled_monitor_test.py index 62d17d813..d9e296396 100644 --- a/tests/schains/monitor/skaled_monitor_test.py +++ b/tests/schains/monitor/skaled_monitor_test.py @@ -19,7 +19,7 @@ RecreateSkaledMonitor, RegularSkaledMonitor, RepairSkaledMonitor, - UpdateConfigSkaledMonitor + UpdateConfigSkaledMonitor, ) from core.schains.external_config import ExternalConfig from core.schains.exit_scheduler import ExitScheduleFileManager @@ -27,6 +27,8 @@ from tools.configs.containers import SCHAIN_CONTAINER, IMA_CONTAINER from web.models.schain import SChainRecord +from tests.utils import CURRENT_TS + CURRENT_TIMESTAMP = 1594903080 CURRENT_DATETIME = datetime.datetime.utcfromtimestamp(CURRENT_TIMESTAMP) @@ -39,7 +41,7 @@ def run_ima_container_mock(schain: dict, mainnet_chain_id: int, dutils=None): dutils.run_container( image_name=image_name, name=container_name, - entrypoint='bash -c "while true; do foo; sleep 2; done"' + entrypoint='bash -c "while true; do foo; sleep 2; done"', ) @@ -51,7 +53,7 @@ def monitor_schain_container_mock( start_ts=None, dutils=None, sync_node=False, - historic_state=False + historic_state=False, ): image_name, container_name, _, _ = get_container_info( SCHAIN_CONTAINER, schain.name) @@ -59,7 +61,7 @@ def monitor_schain_container_mock( dutils.run_container( image_name=image_name, name=container_name, - entrypoint='bash -c "while true; do foo; sleep 2; done"' + entrypoint='bash -c "while true; do foo; sleep 2; done"', ) @@ -69,12 +71,7 @@ def rotation_data(schain_db, skale): @pytest.fixture -def skaled_checks( - schain_db, - skale, - rule_controller, - dutils -): +def skaled_checks(schain_db, skale, rule_controller, dutils): name = schain_db schain_record = SChainRecord.get_by_name(name) return SkaledChecks( @@ -82,7 +79,7 @@ def skaled_checks( schain_record=schain_record, rule_controller=rule_controller, dutils=dutils, - sync_node=False + sync_node=False, ) @@ -98,8 +95,9 @@ def skaled_am( secret_key, ssl_folder, ima_migration_schedule, + ncli_status, dutils, - skaled_checks + skaled_checks, ): name = schain_db schain = skale.schains.get_by_name(name) @@ -107,8 +105,9 @@ def skaled_am( schain=schain, rule_controller=rule_controller, node_config=node_config, + ncli_status=ncli_status, checks=skaled_checks, - dutils=dutils + dutils=dutils, ) @@ -119,19 +118,14 @@ def config(self) -> CheckRes: @pytest.fixture -def skaled_checks_no_config( - schain_db, - skale, - rule_controller, - dutils -): +def skaled_checks_no_config(schain_db, skale, rule_controller, dutils): name = schain_db schain_record = SChainRecord.get_by_name(name) return SkaledChecksNoConfig( schain_name=name, schain_record=schain_record, rule_controller=rule_controller, - dutils=dutils + dutils=dutils, ) @@ -146,101 +140,76 @@ def rotation_id_updated(self) -> CheckRes: @pytest.fixture -def skaled_checks_outdated_config( - schain_db, - skale, - rule_controller, - dutils -): +def skaled_checks_outdated_config(schain_db, skale, rule_controller, dutils): name = schain_db schain_record = SChainRecord.get_by_name(name) return SkaledChecksConfigOutdated( schain_name=name, schain_record=schain_record, rule_controller=rule_controller, - dutils=dutils + dutils=dutils, ) -def test_get_skaled_monitor_no_config(skaled_am, skaled_checks_no_config, skaled_status, schain_db): +def test_get_skaled_monitor_no_config( + skaled_am, skaled_checks_no_config, skaled_status, schain_db, ncli_status +): name = schain_db schain_record = SChainRecord.get_by_name(name) mon = get_skaled_monitor( - skaled_am, - skaled_checks_no_config.get_all(), - schain_record, - skaled_status + skaled_am, skaled_checks_no_config.get_all(), schain_record, skaled_status, ncli_status ) assert mon == NoConfigSkaledMonitor -def test_get_skaled_monitor_regular_and_backup(skaled_am, skaled_checks, skaled_status, schain_db): +def test_get_skaled_monitor_regular_and_backup( + skaled_am, skaled_checks, skaled_status, schain_db, ncli_status +): name = schain_db schain_record = SChainRecord.get_by_name(name) mon = get_skaled_monitor( - skaled_am, - skaled_checks.get_all(), - schain_record, - skaled_status + skaled_am, skaled_checks.get_all(), schain_record, skaled_status, ncli_status ) assert mon == RegularSkaledMonitor schain_record.set_backup_run(True) mon = get_skaled_monitor( - skaled_am, - skaled_checks.get_all(), - schain_record, - skaled_status + skaled_am, skaled_checks.get_all(), schain_record, skaled_status, ncli_status ) assert mon == RegularSkaledMonitor schain_record.set_first_run(False) mon = get_skaled_monitor( - skaled_am, - skaled_checks.get_all(), - schain_record, - skaled_status + skaled_am, skaled_checks.get_all(), schain_record, skaled_status, ncli_status ) assert mon == RegularSkaledMonitor schain_record.set_new_schain(False) mon = get_skaled_monitor( - skaled_am, - skaled_checks.get_all(), - schain_record, - skaled_status + skaled_am, skaled_checks.get_all(), schain_record, skaled_status, ncli_status ) assert mon == BackupSkaledMonitor -def test_get_skaled_monitor_repair(skaled_am, skaled_checks, skaled_status, schain_db): +def test_get_skaled_monitor_repair(skaled_am, skaled_checks, skaled_status, schain_db, ncli_status): name = schain_db schain_record = SChainRecord.get_by_name(name) - schain_record.set_repair_mode(True) + schain_record.set_repair_date(datetime.datetime.utcfromtimestamp(CURRENT_TS - 10)) mon = get_skaled_monitor( - skaled_am, - skaled_checks.get_all(), - schain_record, - skaled_status + skaled_am, skaled_checks.get_all(), schain_record, skaled_status, ncli_status ) assert mon == RepairSkaledMonitor def test_get_skaled_monitor_repair_skaled_status( - skaled_am, - skaled_checks, - schain_db, - skaled_status_repair + skaled_am, skaled_checks, schain_db, skaled_status_repair, ncli_status ): name = schain_db schain_record = SChainRecord.get_by_name(name) mon = get_skaled_monitor( - skaled_am, - skaled_checks.get_all(), - schain_record, - skaled_status_repair + skaled_am, skaled_checks.get_all(), schain_record, skaled_status_repair, ncli_status ) assert mon == RepairSkaledMonitor @@ -249,7 +218,8 @@ def test_get_skaled_monitor_repair_skaled_status( skaled_checks.get_all(), schain_record, skaled_status_repair, - automatic_repair=False + ncli_status, + automatic_repair=False, ) assert mon == RegularSkaledMonitor @@ -277,19 +247,14 @@ def container(self) -> CheckRes: @pytest.fixture -def skaled_checks_new_config( - schain_db, - skale, - rule_controller, - dutils -): +def skaled_checks_new_config(schain_db, skale, rule_controller, dutils): name = schain_db schain_record = SChainRecord.get_by_name(name) return SkaledChecksWithConfig( schain_name=name, schain_record=schain_record, rule_controller=rule_controller, - dutils=dutils + dutils=dutils, ) @@ -308,6 +273,7 @@ def test_get_skaled_monitor_reload_group( secret_keys, ssl_folder, skaled_checks, + ncli_status, dutils ): name = schain_db @@ -319,8 +285,7 @@ def test_get_skaled_monitor_reload_group( schain = skale.schains.get_by_name(name) with mock.patch( - f'{__name__}.SkaledActionManager.upstream_finish_ts', - new_callable=mock.PropertyMock + f'{__name__}.SkaledActionManager.upstream_finish_ts', new_callable=mock.PropertyMock ) as finish_ts_mock: finish_ts_mock.return_value = CURRENT_TIMESTAMP - 10 skaled_am = SkaledActionManager( @@ -328,14 +293,10 @@ def test_get_skaled_monitor_reload_group( rule_controller=rule_controller, node_config=node_config, checks=skaled_checks, - dutils=dutils - ) - mon = get_skaled_monitor( - skaled_am, - state, - schain_record, - skaled_status + ncli_status=ncli_status, + dutils=dutils, ) + mon = get_skaled_monitor(skaled_am, state, schain_record, skaled_status, ncli_status) assert mon == RegularSkaledMonitor finish_ts_mock.return_value = CURRENT_TIMESTAMP + 10 skaled_am = SkaledActionManager( @@ -343,14 +304,10 @@ def test_get_skaled_monitor_reload_group( rule_controller=rule_controller, node_config=node_config, checks=skaled_checks, - dutils=dutils - ) - mon = get_skaled_monitor( - skaled_am, - state, - schain_record, - skaled_status + ncli_status=ncli_status, + dutils=dutils, ) + mon = get_skaled_monitor(skaled_am, state, schain_record, skaled_status, ncli_status) assert mon == ReloadGroupSkaledMonitor @@ -369,7 +326,8 @@ def test_get_skaled_monitor_reload_ip( secret_keys, ssl_folder, skaled_checks, - dutils + ncli_status, + dutils, ): name = schain_db schain_record = SChainRecord.get_by_name(name) @@ -386,26 +344,17 @@ def test_get_skaled_monitor_reload_ip( rule_controller=rule_controller, node_config=node_config, checks=skaled_checks, - dutils=dutils - ) - mon = get_skaled_monitor( - skaled_am, - state, - schain_record, - skaled_status + ncli_status=ncli_status, + dutils=dutils, ) + mon = get_skaled_monitor(skaled_am, state, schain_record, skaled_status, ncli_status) assert mon == RegularSkaledMonitor estate = econfig.read() estate['reload_ts'] = CURRENT_TIMESTAMP + 10 econfig.write(estate) - mon = get_skaled_monitor( - skaled_am, - state, - schain_record, - skaled_status - ) + mon = get_skaled_monitor(skaled_am, state, schain_record, skaled_status, ncli_status) assert mon == ReloadIpSkaledMonitor @@ -423,7 +372,8 @@ def test_get_skaled_monitor_new_node( skaled_status, skaled_checks, ima_migration_schedule, - dutils + ncli_status, + dutils, ): name = schain_db schain_record = SChainRecord.get_by_name(name) @@ -431,23 +381,20 @@ def test_get_skaled_monitor_new_node( finish_ts = CURRENT_TIMESTAMP + 10 with mock.patch( - f'{__name__}.SkaledActionManager.finish_ts', - new_callable=mock.PropertyMock + f'{__name__}.SkaledActionManager.finish_ts', new_callable=mock.PropertyMock ) as finish_ts_mock: skaled_am = SkaledActionManager( schain=schain, rule_controller=rule_controller, node_config=node_config, + ncli_status=ncli_status, checks=skaled_checks, - dutils=dutils + dutils=dutils, ) finish_ts_mock.return_value = finish_ts mon = get_skaled_monitor( - skaled_am, - skaled_checks.get_all(), - schain_record, - skaled_status + skaled_am, skaled_checks.get_all(), schain_record, skaled_status, ncli_status ) assert mon == NewNodeSkaledMonitor @@ -458,6 +405,7 @@ def test_get_skaled_monitor_update_config( skaled_checks_new_config, schain_db, skaled_status_exit_time_reached, + ncli_status, ): name = schain_db schain_record = SChainRecord.get_by_name(name) @@ -465,29 +413,20 @@ def test_get_skaled_monitor_update_config( status['skaled_container'] = False mon = get_skaled_monitor( - skaled_am, - status, - schain_record, - skaled_status_exit_time_reached + skaled_am, status, schain_record, skaled_status_exit_time_reached, ncli_status ) assert mon == UpdateConfigSkaledMonitor status = skaled_checks_new_config.get_all() status['skaled_container'] = False mon = get_skaled_monitor( - skaled_am, - status, - schain_record, - skaled_status_exit_time_reached + skaled_am, status, schain_record, skaled_status_exit_time_reached, ncli_status ) assert mon == UpdateConfigSkaledMonitor def test_get_skaled_monitor_recreate( - skaled_am, - skaled_checks, - schain_db, - skaled_status + skaled_am, skaled_checks, schain_db, skaled_status, ncli_status ): name = schain_db schain_record = SChainRecord.get_by_name(name) @@ -501,7 +440,8 @@ def test_get_skaled_monitor_recreate( skaled_am, status, schain_record, - skaled_status + skaled_status, + ncli_status ) assert mon == RegularSkaledMonitor status['skaled_container'] = True @@ -509,17 +449,13 @@ def test_get_skaled_monitor_recreate( skaled_am, status, schain_record, - skaled_status + skaled_status, + ncli_status ) assert mon == RecreateSkaledMonitor -def test_regular_skaled_monitor( - skaled_am, - skaled_checks, - clean_docker, - dutils -): +def test_regular_skaled_monitor(skaled_am, skaled_checks, clean_docker, dutils): mon = RegularSkaledMonitor(skaled_am, skaled_checks) mon.run() assert skaled_am.rc.is_rules_synced @@ -533,8 +469,7 @@ def test_backup_skaled_monitor(skaled_am, skaled_checks, clean_docker, dutils): mon.run() assert skaled_am.rc.is_rules_synced assert dutils.get_vol(skaled_am.name) - schain_container = dutils.safe_get_container( - f'skale_schain_{skaled_am.name}') + schain_container = dutils.safe_get_container(f'skale_schain_{skaled_am.name}') assert schain_container assert '--download-snapshot' in dutils.get_cmd(schain_container.id) assert dutils.safe_get_container(f'skale_ima_{skaled_am.name}') @@ -549,8 +484,7 @@ def test_repair_skaled_monitor(skaled_am, skaled_checks, clean_docker, dutils): assert dutils.get_vol(skaled_am.name) assert dutils.get_vol_created_ts(skaled_am.name) > ts_before - schain_container = dutils.safe_get_container( - f'skale_schain_{skaled_am.name}') + schain_container = dutils.safe_get_container(f'skale_schain_{skaled_am.name}') assert schain_container assert '--download-snapshot' in dutils.get_cmd(schain_container.id) assert dutils.get_container_created_ts(schain_container.id) > ts_before @@ -561,8 +495,9 @@ def test_group_reload_skaled_monitor(skaled_am, skaled_checks, clean_docker, dut mon = ReloadGroupSkaledMonitor(skaled_am, skaled_checks) ts = time.time() esfm = ExitScheduleFileManager(mon.am.name) - with mock.patch('core.schains.monitor.action.get_finish_ts_from_latest_upstream', - return_value=ts): + with mock.patch( + 'core.schains.monitor.action.get_finish_ts_from_latest_upstream', return_value=ts + ): mon.run() assert esfm.exit_ts == ts assert skaled_am.rc.is_rules_synced @@ -574,8 +509,9 @@ def test_group_reload_skaled_monitor(skaled_am, skaled_checks, clean_docker, dut @pytest.mark.skip def test_group_reload_skaled_monitor_failed_skaled(skaled_am, skaled_checks, clean_docker, dutils): mon = ReloadGroupSkaledMonitor(skaled_am, skaled_checks) - with mock.patch('core.schains.monitor.containers.run_schain_container') \ - as run_skaled_container_mock: + with mock.patch( + 'core.schains.monitor.containers.run_schain_container' + ) as run_skaled_container_mock: mon.run() assert skaled_am.rc.is_rules_synced assert run_skaled_container_mock.assert_not_called() @@ -586,19 +522,13 @@ def test_recreate_skaled_monitor(skaled_am, skaled_checks, clean_docker, dutils) ts_before = time.time() time.sleep(1) mon.run() - schain_container = dutils.safe_get_container( - f'skale_schain_{skaled_am.name}') + schain_container = dutils.safe_get_container(f'skale_schain_{skaled_am.name}') assert schain_container assert dutils.get_container_created_ts(schain_container.id) > ts_before def test_update_config_skaled_monitor( - skaled_am, - skaled_checks, - dutils, - clean_docker, - upstreams, - skaled_status_exit_time_reached + skaled_am, skaled_checks, dutils, clean_docker, upstreams, skaled_status_exit_time_reached ): name = skaled_checks.name ts_before = time.time() @@ -607,13 +537,10 @@ def test_update_config_skaled_monitor( mon.run() assert dutils.get_vol(name) assert dutils.get_vol_created_ts(name) > ts_before - schain_container = dutils.safe_get_container( - f'skale_schain_{name}' - ) + schain_container = dutils.safe_get_container(f'skale_schain_{name}') assert schain_container assert dutils.get_container_created_ts(schain_container.id) > ts_before - os.stat(os.path.join(schain_config_dir(name), - f'schain_{name}.json')).st_mtime > ts_before + os.stat(os.path.join(schain_config_dir(name), f'schain_{name}.json')).st_mtime > ts_before def test_no_config_monitor(skaled_am, skaled_checks, clean_docker, dutils): @@ -629,7 +556,6 @@ def test_new_node_monitor(skaled_am, skaled_checks, clean_docker, dutils): mon.run() assert skaled_am.rc.is_rules_synced assert dutils.get_vol(skaled_am.name) - schain_container = dutils.safe_get_container( - f'skale_schain_{skaled_am.name}') + schain_container = dutils.safe_get_container(f'skale_schain_{skaled_am.name}') assert schain_container assert '--download-snapshot' in dutils.get_cmd(schain_container.id) diff --git a/tests/schains/skaled_status_test.py b/tests/schains/skaled_status_test.py index 4981698ef..b11422fe1 100644 --- a/tests/schains/skaled_status_test.py +++ b/tests/schains/skaled_status_test.py @@ -1,6 +1,15 @@ -from core.schains.skaled_status import SkaledStatus +from core.schains.status import ( + get_node_cli_status, + node_cli_status_filepath, + NodeCliStatus, + SkaledStatus, +) from core.schains.config.directory import skaled_status_filepath +CURRENT_TS = 1594903080 + +NCLI_STATUS_DICT = {'repair_ts': CURRENT_TS, 'snapshot_from': '127.0.0.1'} + def test_skaled_status(skaled_status, _schain_name): status_filepath = skaled_status_filepath(_schain_name) @@ -9,14 +18,14 @@ def test_skaled_status(skaled_status, _schain_name): assert skaled_status.subsystem_running == { 'SnapshotDownloader': False, 'Blockchain': False, - 'Rpc': False + 'Rpc': False, } assert skaled_status.exit_state == { 'ClearDataDir': False, 'StartAgain': False, 'StartFromSnapshot': False, - 'ExitTimeReached': False + 'ExitTimeReached': False, } @@ -47,3 +56,21 @@ def test_log(skaled_status, _schain_name, caplog): status_filepath = skaled_status_filepath(_schain_name) skaled_status = SkaledStatus(filepath=status_filepath) skaled_status.log() + + +def test_node_cli_status_empty(_schain_name): + cli_status = get_node_cli_status(_schain_name) + assert cli_status is None + + status_filepath = node_cli_status_filepath(_schain_name) + cli_status = NodeCliStatus(filepath=status_filepath) + + assert cli_status.repair_ts is None + assert cli_status.snapshot_from is None + + +def test_node_cli_status_repair(_schain_name, ncli_status): + cli_status = get_node_cli_status(_schain_name) + + assert cli_status.repair_ts == CURRENT_TS + assert cli_status.snapshot_from == '127.0.0.1' diff --git a/tests/utils.py b/tests/utils.py index 5b429907c..e7744a592 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -39,6 +39,7 @@ from web.models.schain import upsert_schain_record +CURRENT_TS = 1594903080 DIR_PATH = os.path.dirname(os.path.realpath(__file__)) ENDPOINT = os.getenv('ENDPOINT') diff --git a/tools/configs/schains.py b/tools/configs/schains.py index 566709ca8..f08338d45 100644 --- a/tools/configs/schains.py +++ b/tools/configs/schains.py @@ -44,6 +44,7 @@ MAX_SCHAIN_FAILED_RPC_COUNT = int(os.getenv('MAX_SCHAIN_FAILED_RPC_COUNT', 5)) SKALED_STATUS_FILENAME = 'skaled.status' +NODE_CLI_STATUS_FILENAME = 'node_cli.status' STATIC_SCHAIN_DIR_NAME = 'schains' SCHAIN_STATE_PATH = os.path.join(SKALE_LIB_PATH, 'schains') diff --git a/web/migrations.py b/web/migrations.py index 3341a49bf..44ce37fef 100644 --- a/web/migrations.py +++ b/web/migrations.py @@ -65,6 +65,9 @@ def run_migrations(db, migrator): add_backup_run_field(db, migrator) add_sync_config_run_field(db, migrator) + # 2.7 -> 2.8 update fields + add_repair_date_field(db, migrator) + def add_new_schain_field(db, migrator): add_column( @@ -157,6 +160,13 @@ def add_dkg_step_field(db, migrator): ) +def add_repair_date_field(db, migrator): + add_column( + db, migrator, 'SChainRecord', 'repair_date', + DateTimeField(default=datetime.now()) + ) + + def find_column(db, table_name, column_name): columns = db.get_columns(table_name) return next((x for x in columns if x.name == column_name), None) diff --git a/web/models/schain.py b/web/models/schain.py index a7f67eb79..14a023a7a 100644 --- a/web/models/schain.py +++ b/web/models/schain.py @@ -54,6 +54,8 @@ class SChainRecord(BaseModel): ssl_change_date = DateTimeField(default=datetime.now()) + repair_date = DateTimeField(default=datetime.now()) + @classmethod def add(cls, name): try: @@ -211,6 +213,11 @@ def is_dkg_unsuccessful(self) -> bool: DKGStatus.FAILED ] + def set_repair_date(self, value: datetime) -> None: + logger.info(f'Changing repair_date for {self.name} to {value}') + self.repair_date = value + self.save() + def create_tables(): logger.info('Creating schainrecord table...') @@ -302,23 +309,3 @@ def get_schains_names(include_deleted=False): def get_schains_statuses(include_deleted=False): return [SChainRecord.to_dict(r) for r in SChainRecord.get_all_records(include_deleted)] - - -def toggle_schain_repair_mode(name, snapshot_from: str = ''): - logger.info(f'Toggling repair mode for schain {name}') - query = SChainRecord.update( - repair_mode=True, - snapshot_from=snapshot_from - ).where(SChainRecord.name == name) - count = query.execute() - return count > 0 - - -def switch_off_repair_mode(name): - logger.info(f'Disabling repair mode for schain {name}') - query = SChainRecord.update( - repair_mode=False, - snapshot_from='' - ).where(SChainRecord.name == name) - count = query.execute() - return count > 0 diff --git a/web/routes/schains.py b/web/routes/schains.py index 223649fb2..060a83ffd 100644 --- a/web/routes/schains.py +++ b/web/routes/schains.py @@ -31,11 +31,11 @@ get_default_rule_controller, get_sync_agent_ranges ) -from core.schains.skaled_status import init_skaled_status +from core.schains.status import init_skaled_status from core.schains.ima import get_ima_version_after_migration from core.schains.info import get_schain_info_by_name, get_skaled_version from core.schains.cleaner import get_schains_on_node -from web.models.schain import get_schains_statuses, toggle_schain_repair_mode +from web.models.schain import get_schains_statuses from web.helper import ( construct_ok_response, construct_err_response, @@ -132,21 +132,6 @@ def firewall_rules(): return construct_ok_response({'endpoints': endpoints}) -@schains_bp.route(get_api_url(BLUEPRINT_NAME, 'repair'), methods=['POST']) -def repair(): - logger.debug(request) - schain_name = request.json.get('schain_name') - snapshot_from = request.json.get('snapshot_from', '') - result = toggle_schain_repair_mode( - schain_name, snapshot_from=snapshot_from) - if result: - return construct_ok_response() - else: - return construct_err_response( - msg=f'No schain with name {schain_name}' - ) - - @schains_bp.route(get_api_url(BLUEPRINT_NAME, 'get'), methods=['GET']) @g_skale def get_schain():