From 633d59470460724ba5f649bf56c8f20e31cbc1d5 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 7 May 2024 16:54:23 +0000 Subject: [PATCH 1/5] Add IMA time frame migration --- core/schains/ima.py | 24 +++++++++++++++---- core/schains/monitor/containers.py | 6 ++++- core/schains/runner.py | 7 +++--- tests/schains/checks_test.py | 2 +- tests/schains/ima_test.py | 4 +++- .../monitor/action/skaled_action_test.py | 10 ++++++-- tests/skale-data/config/static_params.yaml | 5 ++++ tools/configs/ima.py | 2 +- 8 files changed, 46 insertions(+), 14 deletions(-) diff --git a/core/schains/ima.py b/core/schains/ima.py index d5c64f96d..17cc612a8 100644 --- a/core/schains/ima.py +++ b/core/schains/ima.py @@ -28,7 +28,7 @@ from core.schains.config.directory import schain_config_dir from core.schains.config.file_manager import ConfigFileManager -from core.schains.config.helper import get_schain_ports_from_config, get_chain_id +from core.schains.config.helper import get_chain_id, get_schain_ports_from_config, get_static_params from core.ima.schain import get_schain_ima_abi_filepath from tools.configs import ENV_TYPE, SGX_SSL_KEY_FILEPATH, SGX_SSL_CERT_FILEPATH, SGX_SERVER_URL from tools.configs.containers import IMA_MIGRATION_PATH, CONTAINERS_INFO @@ -36,8 +36,8 @@ from tools.configs.ima import ( MAINNET_IMA_ABI_FILEPATH, IMA_STATE_CONTAINER_PATH, - IMA_TIME_FRAMING, - IMA_NETWORK_BROWSER_FILEPATH + IMA_NETWORK_BROWSER_FILEPATH, + DEFAULT_TIME_FRAME_INTERVAL ) from tools.configs.schains import SCHAINS_DIR_PATH from tools.helper import safe_load_yml @@ -145,7 +145,7 @@ def schain_index_to_node_number(node): return int(node['schainIndex']) - 1 -def get_ima_env(schain_name: str, mainnet_chain_id: int) -> ImaEnv: +def get_ima_env(schain_name: str, mainnet_chain_id: int, time_frame_interval: int) -> ImaEnv: schain_config = ConfigFileManager(schain_name).skaled_config node_info = schain_config["skaleConfig"]["nodeInfo"] bls_key_name = node_info['wallets']['ima']['keyShareName'] @@ -180,7 +180,7 @@ def get_ima_env(schain_name: str, mainnet_chain_id: int) -> ImaEnv: cid_schain=schain_chain_id, monitoring_port=node_info['imaMonitoringPort'], rpc_port=get_ima_rpc_port(schain_name), - time_framing=IMA_TIME_FRAMING, + time_framing=time_frame_interval, network_browser_data_path=IMA_NETWORK_BROWSER_FILEPATH ) @@ -280,3 +280,17 @@ def get_migration_schedule() -> dict: def get_migration_ts(name: str) -> int: return get_migration_schedule().get(name, 0) + + +def get_ima_time_frame_interval(name: str, after: bool = False) -> int: + params = get_static_params() + if 'ima' not in params or 'time_frame_interval' not in params['ima']: + logger.debug( + 'IMA time frame intrerval is not set. Using default value %d', + DEFAULT_TIME_FRAME_INTERVAL + ) + return DEFAULT_TIME_FRAME_INTERVAL + if after: + return params['ima']['time_frame_interval']['after'] + else: + return params['ima']['time_frame_interval']['before'] diff --git a/core/schains/monitor/containers.py b/core/schains/monitor/containers.py index 0e67162a8..ddfaf338d 100644 --- a/core/schains/monitor/containers.py +++ b/core/schains/monitor/containers.py @@ -32,7 +32,7 @@ run_schain_container ) from core.ima.schain import copy_schain_ima_abi -from core.schains.ima import ImaData +from core.schains.ima import get_ima_time_frame_interval, ImaData from tools.configs import SYNC_NODE from tools.configs.containers import ( @@ -134,6 +134,7 @@ def monitor_ima_container( expected_image = get_image_name(image_type=IMA_CONTAINER) logger.debug('%s IMA image %s, expected %s', schain_name, container_image, expected_image) + time_frame_interval = get_ima_time_frame_interval(schain_name, after=False) if time.time() > migration_ts: logger.debug('%s IMA migration time passed', schain_name) @@ -143,6 +144,8 @@ def monitor_ima_container( '%s Removing old container as part of IMA migration', schain_name) remove_container(schain_name, IMA_CONTAINER, dutils) container_exists = False + time_frame_interval = get_ima_time_frame_interval(schain_name, after=True) + logger.debug('IMA time frame interval %d', time_frame_interval) if not container_exists: logger.info('%s No IMA container, creating, image %s', @@ -151,6 +154,7 @@ def monitor_ima_container( schain, ima_data.chain_id, image=expected_image, + time_frame_interval=time_frame_interval, dutils=dutils ) else: diff --git a/core/schains/runner.py b/core/schains/runner.py index 9e842ca81..1fb9fcc4e 100644 --- a/core/schains/runner.py +++ b/core/schains/runner.py @@ -226,11 +226,12 @@ def run_schain_container( def run_ima_container( schain: dict, mainnet_chain_id: int, + time_frame_interval: int, image: str, dutils: DockerUtils = None ) -> None: dutils = dutils or DockerUtils() - env = get_ima_env(schain['name'], mainnet_chain_id) + env = get_ima_env(schain['name'], mainnet_chain_id, time_frame_interval) schain_type = get_schain_type(schain['partOfNode']) cpu_limit = get_ima_limit(schain_type, MetricType.cpu_shares) @@ -299,8 +300,8 @@ def is_new_image_pulled(image_type: str, dutils: DockerUtils) -> bool: return dutils.pulled(image) -def remove_container(schain_name: str, type: str, dutils: DockerUtils): - container = get_container_name(type=type, schain_name=schain_name) +def remove_container(schain_name: str, image_type: str, dutils: DockerUtils): + container = get_container_name(image_type=type, schain_name=schain_name) dutils.safe_rm(container) diff --git a/tests/schains/checks_test.py b/tests/schains/checks_test.py index ff3b64907..d0704e2e4 100644 --- a/tests/schains/checks_test.py +++ b/tests/schains/checks_test.py @@ -231,7 +231,7 @@ def test_ima_container_check(schain_checks, cleanup_ima_containers, dutils): with mock.patch('core.schains.checks.get_ima_migration_ts', return_value=mts): assert not schain_checks.ima_container.status image = get_image_name(image_type=IMA_CONTAINER, new=True) - run_ima_container(schain, mainnet_chain_id=1, + run_ima_container(schain, mainnet_chain_id=1, time_frame_interval=900, image=image, dutils=dutils) assert schain_checks.ima_container.status diff --git a/tests/schains/ima_test.py b/tests/schains/ima_test.py index ff2fb0c19..1d3d42ac9 100644 --- a/tests/schains/ima_test.py +++ b/tests/schains/ima_test.py @@ -4,10 +4,12 @@ def test_get_ima_env(_schain_name, schain_config): ima_env = get_ima_env( schain_name=_schain_name, - mainnet_chain_id=123 + mainnet_chain_id=123, + time_frame_interval=100 ) ima_env_dict = ima_env.to_dict() assert len(ima_env_dict) == 23 assert ima_env_dict['CID_MAIN_NET'] == 123 assert ima_env_dict['RPC_PORT'] == 10010 + assert ima_env_dict['TIME_FRAMING'] == 100 isinstance(ima_env_dict['CID_SCHAIN'], str) diff --git a/tests/schains/monitor/action/skaled_action_test.py b/tests/schains/monitor/action/skaled_action_test.py index 66b6ee0e6..4b130dc81 100644 --- a/tests/schains/monitor/action/skaled_action_test.py +++ b/tests/schains/monitor/action/skaled_action_test.py @@ -21,7 +21,13 @@ CURRENT_DATETIME = datetime.datetime.utcfromtimestamp(CURRENT_TIMESTAMP) -def run_ima_container_mock(schain: dict, mainnet_chain_id: int, image: str, dutils=None): +def run_ima_container_mock( + schain: dict, + mainnet_chain_id: int, + image: str, + time_frame_interval: int, + dutils=None +): image_name, container_name, _, _ = get_container_info( IMA_CONTAINER, schain['name']) image = image or image_name @@ -279,7 +285,7 @@ def test_ima_container_action_new_chain( assert image == 'skalenetwork/ima:2.0.0-beta.9' -@pytest.mark.skip('Docker API GA issues need to be resolved') +# @pytest.mark.skip('Docker API GA issues need to be resolved') @mock.patch('core.schains.monitor.containers.run_ima_container', run_ima_container_mock) def test_ima_container_action_old_chain( skaled_am, diff --git a/tests/skale-data/config/static_params.yaml b/tests/skale-data/config/static_params.yaml index 728b884ba..8c459e197 100644 --- a/tests/skale-data/config/static_params.yaml +++ b/tests/skale-data/config/static_params.yaml @@ -43,6 +43,11 @@ envs: emptyBlockIntervalMs: 10000 snapshotDownloadTimeout: 18000 snapshotDownloadInactiveTimeout: 120 + ima: + before: + time_frame_interval: 1800 + after: + time_frame_interval: 900 schain_cmd: ["-v 3", "--web3-trace", "--enable-debug-behavior-apis", "--aa no"] diff --git a/tools/configs/ima.py b/tools/configs/ima.py index 18d48943f..dac9b453e 100644 --- a/tools/configs/ima.py +++ b/tools/configs/ima.py @@ -88,4 +88,4 @@ } } -IMA_TIME_FRAMING = 1800 # 30 min +DEFAULT_TIME_FRAME_INTERVAL = 1800 # 30 min From fb6f23a76e3dda4973b0a42bad9478fe552ecb34 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 9 May 2024 19:37:16 +0000 Subject: [PATCH 2/5] Update checks to handle time frame migration --- core/schains/checks.py | 29 ++++-- core/schains/ima.py | 18 ++-- core/schains/monitor/containers.py | 48 +++++----- core/schains/runner.py | 11 ++- tests/conftest.py | 16 +++- tests/schains/checks_test.py | 2 +- tests/schains/ima_test.py | 2 +- .../monitor/action/skaled_action_test.py | 91 +++++++++++++++---- .../config/ima_migration_schedule.yaml | 2 - tests/skale-data/config/static_params.yaml | 10 +- tests/utils.py | 2 + tools/configs/ima.py | 2 +- tools/docker_utils.py | 13 ++- 13 files changed, 173 insertions(+), 73 deletions(-) delete mode 100644 tests/skale-data/config/ima_migration_schedule.yaml diff --git a/core/schains/checks.py b/core/schains/checks.py index 3d6e14621..1a31c1488 100644 --- a/core/schains/checks.py +++ b/core/schains/checks.py @@ -38,7 +38,7 @@ ) from core.schains.dkg.utils import get_secret_key_share_filepath from core.schains.firewall.types import IRuleController -from core.schains.ima import get_migration_ts as get_ima_migration_ts +from core.schains.ima import get_ima_time_frame, get_migration_ts as get_ima_migration_ts from core.schains.process_manager_helper import is_monitor_process_alive from core.schains.rpc import ( check_endpoint_alive, @@ -46,7 +46,12 @@ get_endpoint_alive_check_timeout ) from core.schains.external_config import ExternalConfig, ExternalState -from core.schains.runner import get_container_name, get_image_name, is_new_image_pulled +from core.schains.runner import ( + get_container_name, + get_ima_container_time_frame, + get_image_name, + is_new_image_pulled +) from core.schains.skaled_exit_codes import SkaledExitCodes from core.schains.volume import is_volume_exists @@ -335,26 +340,36 @@ def ima_container(self) -> CheckRes: new_image_pulled = is_new_image_pulled(image_type=IMA_CONTAINER, dutils=self.dutils) migration_ts = get_ima_migration_ts(self.name) - new = time.time() > migration_ts + after = time.time() > migration_ts container_running = self.dutils.is_container_running(container_name) - updated_image = False + updated_image, updated_time_frame = False, False if container_running: - expected_image = get_image_name(image_type=IMA_CONTAINER, new=new) + expected_image = get_image_name(image_type=IMA_CONTAINER, new=after) image = self.dutils.get_container_image_name(container_name) updated_image = image == expected_image + time_frame = get_ima_time_frame(self.name, after=after) + container_time_frame = get_ima_container_time_frame(self.name, self.dutils) + + updated_time_frame = time_frame == container_time_frame + logger.debug( + 'IMA image %s, container image %s, time frame %d, container_time_frame %d', + expected_image, image, time_frame, container_time_frame + ) + data = { 'container_running': container_running, 'updated_image': updated_image, - 'new_image_pulled': new_image_pulled + 'new_image_pulled': new_image_pulled, + 'updated_time_frame': updated_time_frame } logger.debug( '%s, IMA check - %s', self.name, data ) - result: bool = container_running and updated_image and new_image_pulled + result: bool = all(data.values()) return CheckRes(result, data=data) @property diff --git a/core/schains/ima.py b/core/schains/ima.py index 17cc612a8..c95fd5142 100644 --- a/core/schains/ima.py +++ b/core/schains/ima.py @@ -37,7 +37,7 @@ MAINNET_IMA_ABI_FILEPATH, IMA_STATE_CONTAINER_PATH, IMA_NETWORK_BROWSER_FILEPATH, - DEFAULT_TIME_FRAME_INTERVAL + DEFAULT_TIME_FRAME ) from tools.configs.schains import SCHAINS_DIR_PATH from tools.helper import safe_load_yml @@ -145,7 +145,7 @@ def schain_index_to_node_number(node): return int(node['schainIndex']) - 1 -def get_ima_env(schain_name: str, mainnet_chain_id: int, time_frame_interval: int) -> ImaEnv: +def get_ima_env(schain_name: str, mainnet_chain_id: int, time_frame: int) -> ImaEnv: schain_config = ConfigFileManager(schain_name).skaled_config node_info = schain_config["skaleConfig"]["nodeInfo"] bls_key_name = node_info['wallets']['ima']['keyShareName'] @@ -180,7 +180,7 @@ def get_ima_env(schain_name: str, mainnet_chain_id: int, time_frame_interval: in cid_schain=schain_chain_id, monitoring_port=node_info['imaMonitoringPort'], rpc_port=get_ima_rpc_port(schain_name), - time_framing=time_frame_interval, + time_framing=time_frame, network_browser_data_path=IMA_NETWORK_BROWSER_FILEPATH ) @@ -282,15 +282,15 @@ def get_migration_ts(name: str) -> int: return get_migration_schedule().get(name, 0) -def get_ima_time_frame_interval(name: str, after: bool = False) -> int: +def get_ima_time_frame(name: str, after: bool = False) -> int: params = get_static_params() - if 'ima' not in params or 'time_frame_interval' not in params['ima']: + if 'ima' not in params or 'time_frame' not in params['ima']: logger.debug( 'IMA time frame intrerval is not set. Using default value %d', - DEFAULT_TIME_FRAME_INTERVAL + DEFAULT_TIME_FRAME ) - return DEFAULT_TIME_FRAME_INTERVAL + return DEFAULT_TIME_FRAME if after: - return params['ima']['time_frame_interval']['after'] + return params['ima']['time_frame']['after'] else: - return params['ima']['time_frame_interval']['before'] + return params['ima']['time_frame']['before'] diff --git a/core/schains/monitor/containers.py b/core/schains/monitor/containers.py index ddfaf338d..ef5c9b284 100644 --- a/core/schains/monitor/containers.py +++ b/core/schains/monitor/containers.py @@ -19,10 +19,12 @@ import logging import time +from typing import Optional from core.schains.volume import is_volume_exists from core.schains.runner import ( get_container_image, + get_ima_container_time_frame, get_image_name, is_container_exists, is_schain_container_failed, @@ -32,7 +34,7 @@ run_schain_container ) from core.ima.schain import copy_schain_ima_abi -from core.schains.ima import get_ima_time_frame_interval, ImaData +from core.schains.ima import get_ima_time_frame, ImaData from tools.configs import SYNC_NODE from tools.configs.containers import ( @@ -53,7 +55,7 @@ def monitor_schain_container( download_snapshot=False, start_ts=None, abort_on_exit: bool = True, - dutils=None, + dutils: Optional[DockerUtils] = None, sync_node: bool = False, historic_state: bool = False ) -> None: @@ -128,33 +130,35 @@ def monitor_ima_container( container_exists = is_container_exists( schain_name, container_type=IMA_CONTAINER, dutils=dutils) - container_image = get_container_image(schain_name, IMA_CONTAINER, dutils) - new_image = get_image_name(image_type=IMA_CONTAINER, new=True) - - expected_image = get_image_name(image_type=IMA_CONTAINER) - logger.debug('%s IMA image %s, expected %s', schain_name, - container_image, expected_image) - time_frame_interval = get_ima_time_frame_interval(schain_name, after=False) if time.time() > migration_ts: - logger.debug('%s IMA migration time passed', schain_name) - expected_image = new_image - if container_exists and expected_image != container_image: - logger.info( - '%s Removing old container as part of IMA migration', schain_name) - remove_container(schain_name, IMA_CONTAINER, dutils) - container_exists = False - time_frame_interval = get_ima_time_frame_interval(schain_name, after=True) - logger.debug('IMA time frame interval %d', time_frame_interval) + logger.debug('IMA migration time passed') + + image = get_image_name(image_type=IMA_CONTAINER, new=True) + time_frame = get_ima_time_frame(schain_name, after=True) + if container_exists: + container_image = get_container_image(schain_name, IMA_CONTAINER, dutils) + container_time_frame = get_ima_container_time_frame(schain_name, dutils) + + if image != container_image or time_frame != container_time_frame: + logger.info('Removing old container as part of IMA migration') + remove_container(schain_name, IMA_CONTAINER, dutils) + container_exists = False + else: + time_frame = get_ima_time_frame(schain_name, after=False) + image = get_image_name(image_type=IMA_CONTAINER, new=False) + logger.debug('IMA time frame %d', time_frame) if not container_exists: - logger.info('%s No IMA container, creating, image %s', - schain_name, expected_image) + logger.info( + '%s No IMA container, creating, image %s, time frame %d', + schain_name, image, time_frame + ) run_ima_container( schain, ima_data.chain_id, - image=expected_image, - time_frame_interval=time_frame_interval, + image=image, + time_frame=time_frame, dutils=dutils ) else: diff --git a/core/schains/runner.py b/core/schains/runner.py index 1fb9fcc4e..e65aa6394 100644 --- a/core/schains/runner.py +++ b/core/schains/runner.py @@ -226,12 +226,12 @@ def run_schain_container( def run_ima_container( schain: dict, mainnet_chain_id: int, - time_frame_interval: int, + time_frame: int, image: str, dutils: DockerUtils = None ) -> None: dutils = dutils or DockerUtils() - env = get_ima_env(schain['name'], mainnet_chain_id, time_frame_interval) + env = get_ima_env(schain['name'], mainnet_chain_id, time_frame) schain_type = get_schain_type(schain['partOfNode']) cpu_limit = get_ima_limit(schain_type, MetricType.cpu_shares) @@ -301,7 +301,7 @@ def is_new_image_pulled(image_type: str, dutils: DockerUtils) -> bool: def remove_container(schain_name: str, image_type: str, dutils: DockerUtils): - container = get_container_name(image_type=type, schain_name=schain_name) + container = get_container_name(image_type=image_type, schain_name=schain_name) dutils.safe_rm(container) @@ -310,3 +310,8 @@ def pull_new_image(image_type: str, dutils: DockerUtils) -> None: if not dutils.pulled(image): logger.info('Pulling new image %s', image) dutils.pull(image) + + +def get_ima_container_time_frame(schain_name: str, dutils: DockerUtils) -> int: + container_name = get_container_name(IMA_CONTAINER, schain_name) + return int(dutils.get_container_env_value(container_name, 'TIME_FRAMING')) diff --git a/tests/conftest.py b/tests/conftest.py index 41685e421..807884c44 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,7 @@ import docker import pytest +import yaml from skale import SkaleManager @@ -46,7 +47,7 @@ from core.schains.skaled_status import init_skaled_status, SkaledStatus from core.schains.config.skale_manager_opts import SkaleManagerOpts -from tools.configs import META_FILEPATH, SSL_CERTIFICATES_FILEPATH +from tools.configs import CONFIG_FOLDER, ENV_TYPE, META_FILEPATH, SSL_CERTIFICATES_FILEPATH from tools.configs.containers import CONTAINERS_FILEPATH from tools.configs.ima import SCHAIN_IMA_ABI_FILEPATH from tools.configs.schains import SCHAINS_DIR_PATH @@ -65,6 +66,7 @@ generate_cert, generate_schain_config, get_test_rule_controller, + IMA_MIGRATION_TS, init_skale_from_wallet, init_skale_ima, upsert_schain_record_with_config @@ -617,3 +619,15 @@ def upstreams(schain_db, schain_config): yield files finally: shutil.rmtree(config_folder, ignore_errors=True) + + +@pytest.fixture +def ima_migration_schedule(schain_db): + name = schain_db + try: + migration_schedule_path = os.path.join(CONFIG_FOLDER, 'ima_migration_schedule.yaml') + with open(migration_schedule_path, 'w') as migration_schedule_file: + yaml.dump({ENV_TYPE: {name: IMA_MIGRATION_TS}}, migration_schedule_file) + yield migration_schedule_path + finally: + os.remove(migration_schedule_path) diff --git a/tests/schains/checks_test.py b/tests/schains/checks_test.py index d0704e2e4..66f1964d0 100644 --- a/tests/schains/checks_test.py +++ b/tests/schains/checks_test.py @@ -231,7 +231,7 @@ def test_ima_container_check(schain_checks, cleanup_ima_containers, dutils): with mock.patch('core.schains.checks.get_ima_migration_ts', return_value=mts): assert not schain_checks.ima_container.status image = get_image_name(image_type=IMA_CONTAINER, new=True) - run_ima_container(schain, mainnet_chain_id=1, time_frame_interval=900, + run_ima_container(schain, mainnet_chain_id=1, time_frame=900, image=image, dutils=dutils) assert schain_checks.ima_container.status diff --git a/tests/schains/ima_test.py b/tests/schains/ima_test.py index 1d3d42ac9..18b3f584f 100644 --- a/tests/schains/ima_test.py +++ b/tests/schains/ima_test.py @@ -5,7 +5,7 @@ def test_get_ima_env(_schain_name, schain_config): ima_env = get_ima_env( schain_name=_schain_name, mainnet_chain_id=123, - time_frame_interval=100 + time_frame=100 ) ima_env_dict = ima_env.to_dict() assert len(ima_env_dict) == 23 diff --git a/tests/schains/monitor/action/skaled_action_test.py b/tests/schains/monitor/action/skaled_action_test.py index 4b130dc81..cc35736f0 100644 --- a/tests/schains/monitor/action/skaled_action_test.py +++ b/tests/schains/monitor/action/skaled_action_test.py @@ -17,6 +17,8 @@ from tools.configs.containers import SCHAIN_CONTAINER, IMA_CONTAINER from web.models.schain import SChainRecord +from tests.utils import IMA_MIGRATION_TS + CURRENT_TIMESTAMP = 1594903080 CURRENT_DATETIME = datetime.datetime.utcfromtimestamp(CURRENT_TIMESTAMP) @@ -25,7 +27,7 @@ def run_ima_container_mock( schain: dict, mainnet_chain_id: int, image: str, - time_frame_interval: int, + time_frame: int, dutils=None ): image_name, container_name, _, _ = get_container_info( @@ -263,42 +265,38 @@ def test_recreated_schain_containers( assert ima_ts > ima_created_ts -def test_ima_container_action_new_chain( +def test_ima_container_action_from_scratch( skaled_am, skaled_checks, schain_config, predeployed_ima, ima_linked, cleanup_ima, + ima_migration_schedule, dutils ): - with mock.patch( - 'core.schains.monitor.containers.run_ima_container', - run_ima_container_mock - ): - skaled_am.ima_container() - containers = dutils.get_all_ima_containers(all=True) - assert len(containers) == 1 - container_name = containers[0].name - assert container_name == f'skale_ima_{skaled_am.name}' - image = dutils.get_container_image_name(container_name) - assert image == 'skalenetwork/ima:2.0.0-beta.9' + skaled_am.ima_container() + containers = dutils.get_all_ima_containers(all=True) + assert len(containers) == 1 + container_name = containers[0].name + assert container_name == f'skale_ima_{skaled_am.name}' + image = dutils.get_container_image_name(container_name) + assert image == 'skalenetwork/ima:2.0.0-beta.9' # @pytest.mark.skip('Docker API GA issues need to be resolved') -@mock.patch('core.schains.monitor.containers.run_ima_container', run_ima_container_mock) -def test_ima_container_action_old_chain( +def test_ima_container_action_image_pulling( skaled_am, skaled_checks, schain_config, predeployed_ima, ima_linked, cleanup_ima, + ima_migration_schedule, dutils ): - ts = int(time.time()) - mts = ts + 3600 - with mock.patch('core.schains.monitor.action.get_ima_migration_ts', return_value=mts): + dt = datetime.datetime.utcfromtimestamp(IMA_MIGRATION_TS - 5) + with freezegun.freeze_time(dt): skaled_am.ima_container() containers = dutils.get_all_ima_containers(all=True) assert len(containers) == 1 @@ -309,8 +307,19 @@ def test_ima_container_action_old_chain( assert image == 'skalenetwork/ima:2.0.0-develop.3' assert dutils.pulled('skalenetwork/ima:2.0.0-beta.9') - mts = ts - 5 - with mock.patch('core.schains.monitor.action.get_ima_migration_ts', return_value=mts): + +def test_ima_container_action_image_migration( + skaled_am, + skaled_checks, + schain_config, + predeployed_ima, + ima_linked, + cleanup_ima, + ima_migration_schedule, + dutils +): + dt = datetime.datetime.utcfromtimestamp(IMA_MIGRATION_TS + 5) + with freezegun.freeze_time(dt): skaled_am.ima_container() containers = dutils.get_all_ima_containers(all=True) assert len(containers) == 1 @@ -320,12 +329,54 @@ def test_ima_container_action_old_chain( assert image == 'skalenetwork/ima:2.0.0-beta.9' +# @mock.patch('core.schains.monitor.containers.run_ima_container', run_ima_container_mock) +# @mock.patch('core.schains.monitor.containers.get_image_name', return_value="skalenetwork/ima:2) +def test_ima_container_action_time_frame_migration( + skaled_am, + skaled_checks, + schain_config, + predeployed_ima, + ima_linked, + cleanup_ima, + ima_migration_schedule, + dutils +): + dt = datetime.datetime.utcfromtimestamp(IMA_MIGRATION_TS - 5) + with freezegun.freeze_time(dt): + with mock.patch('core.schains.monitor.containers.get_image_name', + return_value='skalenetwork/ima:2.0.0-beta.9'): + skaled_am.ima_container() + containers = dutils.get_all_ima_containers(all=True) + assert len(containers) == 1 + container_name = containers[0].name + assert container_name == f'skale_ima_{skaled_am.name}' + image = dutils.get_container_image_name(container_name) + assert image == 'skalenetwork/ima:2.0.0-beta.9' + actual_time_frame = int(dutils.get_container_env_value(container_name, 'TIME_FRAMING')) + assert actual_time_frame == 1800 + + dt = datetime.datetime.utcfromtimestamp(IMA_MIGRATION_TS + 5) + with freezegun.freeze_time(dt): + with mock.patch('core.schains.monitor.containers.get_image_name', + return_value='skalenetwork/ima:2.0.0-beta.9'): + skaled_am.ima_container() + containers = dutils.get_all_ima_containers(all=True) + assert len(containers) == 1 + container_name = containers[0].name + assert container_name == f'skale_ima_{skaled_am.name}' + image = dutils.get_container_image_name(container_name) + assert image == 'skalenetwork/ima:2.0.0-beta.9' + actual_time_frame = int(dutils.get_container_env_value(container_name, 'TIME_FRAMING')) + assert actual_time_frame == 900 + + def test_ima_container_action_not_linked( skaled_am, skaled_checks, schain_db, _schain_name, cleanup_ima_containers, + ima_migration_schedule, dutils ): skaled_am.ima_container() diff --git a/tests/skale-data/config/ima_migration_schedule.yaml b/tests/skale-data/config/ima_migration_schedule.yaml deleted file mode 100644 index 9e1a770e7..000000000 --- a/tests/skale-data/config/ima_migration_schedule.yaml +++ /dev/null @@ -1,2 +0,0 @@ -devnet: - test_chain: 1688388551 diff --git a/tests/skale-data/config/static_params.yaml b/tests/skale-data/config/static_params.yaml index 8c459e197..3a3144fc9 100644 --- a/tests/skale-data/config/static_params.yaml +++ b/tests/skale-data/config/static_params.yaml @@ -43,11 +43,6 @@ envs: emptyBlockIntervalMs: 10000 snapshotDownloadTimeout: 18000 snapshotDownloadInactiveTimeout: 120 - ima: - before: - time_frame_interval: 1800 - after: - time_frame_interval: 900 schain_cmd: ["-v 3", "--web3-trace", "--enable-debug-behavior-apis", "--aa no"] @@ -274,6 +269,11 @@ envs: snapshotDownloadTimeout: 18000 snapshotDownloadInactiveTimeout: 120 + ima: + time_frame: + before: 1800 + after: 900 + schain_cmd: ["-v 3", "--web3-trace", "--enable-debug-behavior-apis", "--aa no"] diff --git a/tests/utils.py b/tests/utils.py index d136d78fa..29f37e745 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -54,6 +54,8 @@ IpRange('3.3.3.3', '4.4.4.4') ] +IMA_MIGRATION_TS = 1688388551 + class FailedAPICall(Exception): pass diff --git a/tools/configs/ima.py b/tools/configs/ima.py index dac9b453e..94ceefa42 100644 --- a/tools/configs/ima.py +++ b/tools/configs/ima.py @@ -88,4 +88,4 @@ } } -DEFAULT_TIME_FRAME_INTERVAL = 1800 # 30 min +DEFAULT_TIME_FRAME = 1800 # 30 min diff --git a/tools/docker_utils.py b/tools/docker_utils.py index b6b4548d3..1b13d062e 100644 --- a/tools/docker_utils.py +++ b/tools/docker_utils.py @@ -148,7 +148,7 @@ def get_containers_info(self, all=False, name_filter='*', format=False) -> list: def get_all_ima_containers(self, all=False, format=False) -> list: return self.client.containers.list(all=all, filters={'name': 'skale_ima_*'}) - def get_info(self, container_id: str) -> dict: + def get_info(self, container_id: str, raise_not_found: bool = False) -> dict: container_info = {} try: container = self.client.containers.get(container_id) @@ -157,6 +157,8 @@ def get_info(self, container_id: str) -> dict: container_info['stats'] = self.cli.inspect_container(container.id) container_info['status'] = container.status except docker.errors.NotFound: + if raise_not_found: + raise logger.debug( f'Can not get info - no such container: {container_id}') container_info['status'] = CONTAINER_NOT_FOUND @@ -391,6 +393,15 @@ def get_container_image_name(self, name: str) -> Optional[str]: return None return info['stats']['Config']['Image'] + def get_container_env_value(self, container_name: str, env_option: str) -> Optional[str]: + info = self.get_info(container_name, raise_not_found=True) + env = info['stats']['Config']['Env'] + try: + value = next(filter(lambda v: v.startswith(env_option), env)) + except StopIteration: + return None + return value.split('=')[1] + def wait_for_container_creation(self, name: str, timeout=CONTAINER_CREATION_TIMEOUT): start_ts = time.time() while time.time() - start_ts < timeout and not self.is_container_exists(name): From f8b34894ab84dbe6814932bb7ea4ca78154c9967 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 10 May 2024 12:08:50 +0000 Subject: [PATCH 3/5] Fix tests --- core/schains/ima.py | 12 ++++++------ tests/schains/monitor/action/skaled_action_test.py | 4 +--- tests/schains/monitor/skaled_monitor_test.py | 2 ++ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/schains/ima.py b/core/schains/ima.py index c95fd5142..ded3ce80a 100644 --- a/core/schains/ima.py +++ b/core/schains/ima.py @@ -274,12 +274,12 @@ def get_ima_log_checks(): return all_ima_healthchecks -def get_migration_schedule() -> dict: - return safe_load_yml(IMA_MIGRATION_PATH)[ENV_TYPE] - - -def get_migration_ts(name: str) -> int: - return get_migration_schedule().get(name, 0) +def get_migration_ts(name: str, path: str = IMA_MIGRATION_PATH, env_type: str = ENV_TYPE) -> int: + if os.path.isfile(path): + schedule = safe_load_yml(IMA_MIGRATION_PATH)[env_type] + return schedule.get(name, 0) + else: + return 0 def get_ima_time_frame(name: str, after: bool = False) -> int: diff --git a/tests/schains/monitor/action/skaled_action_test.py b/tests/schains/monitor/action/skaled_action_test.py index cc35736f0..68cfb816b 100644 --- a/tests/schains/monitor/action/skaled_action_test.py +++ b/tests/schains/monitor/action/skaled_action_test.py @@ -91,6 +91,7 @@ def skaled_am( predeployed_ima, secret_key, ssl_folder, + ima_migration_schedule, dutils, skaled_checks ): @@ -292,7 +293,6 @@ def test_ima_container_action_image_pulling( predeployed_ima, ima_linked, cleanup_ima, - ima_migration_schedule, dutils ): dt = datetime.datetime.utcfromtimestamp(IMA_MIGRATION_TS - 5) @@ -315,7 +315,6 @@ def test_ima_container_action_image_migration( predeployed_ima, ima_linked, cleanup_ima, - ima_migration_schedule, dutils ): dt = datetime.datetime.utcfromtimestamp(IMA_MIGRATION_TS + 5) @@ -338,7 +337,6 @@ def test_ima_container_action_time_frame_migration( predeployed_ima, ima_linked, cleanup_ima, - ima_migration_schedule, dutils ): dt = datetime.datetime.utcfromtimestamp(IMA_MIGRATION_TS - 5) diff --git a/tests/schains/monitor/skaled_monitor_test.py b/tests/schains/monitor/skaled_monitor_test.py index e333a4388..5f004cabf 100644 --- a/tests/schains/monitor/skaled_monitor_test.py +++ b/tests/schains/monitor/skaled_monitor_test.py @@ -97,6 +97,7 @@ def skaled_am( rotation_data, secret_key, ssl_folder, + ima_migration_schedule, dutils, skaled_checks ): @@ -421,6 +422,7 @@ def test_get_skaled_monitor_new_node( ssl_folder, skaled_status, skaled_checks, + ima_migration_schedule, dutils ): name = schain_db From 90d82b00262ae6ade301d805c80f2c295e4dac96 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 10 May 2024 14:42:37 +0000 Subject: [PATCH 4/5] Fix ima_container check test --- tests/schains/checks_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/schains/checks_test.py b/tests/schains/checks_test.py index 66f1964d0..95bb85950 100644 --- a/tests/schains/checks_test.py +++ b/tests/schains/checks_test.py @@ -202,7 +202,6 @@ def test_exit_code_ok_check(schain_checks, sample_false_checks): def test_ima_container_check(schain_checks, cleanup_ima_containers, dutils): - dutils.is_container_running = lambda *args: True ts = int(time.time()) mts = ts + 3600 name = schain_checks.name From 3dcc197e3b4711b153b08a1d78a69e1fb27ff05c Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 14 May 2024 18:29:29 +0000 Subject: [PATCH 5/5] Remove unnecessary comments --- tests/schains/monitor/action/skaled_action_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/schains/monitor/action/skaled_action_test.py b/tests/schains/monitor/action/skaled_action_test.py index 68cfb816b..3281ec6bd 100644 --- a/tests/schains/monitor/action/skaled_action_test.py +++ b/tests/schains/monitor/action/skaled_action_test.py @@ -328,8 +328,6 @@ def test_ima_container_action_image_migration( assert image == 'skalenetwork/ima:2.0.0-beta.9' -# @mock.patch('core.schains.monitor.containers.run_ima_container', run_ima_container_mock) -# @mock.patch('core.schains.monitor.containers.get_image_name', return_value="skalenetwork/ima:2) def test_ima_container_action_time_frame_migration( skaled_am, skaled_checks,