Merge pull request #1069 from skalenetwork/decrease-ima-frame

Decrease IMA time frame

core/schains/checks.py

@@ -38,15 +38,20 @@
 )
 from core.schains.dkg.utils import get_secret_key_share_filepath
 from core.schains.firewall.types import IRuleController
-from core.schains.ima import get_migration_ts as get_ima_migration_ts
+from core.schains.ima import get_ima_time_frame, get_migration_ts as get_ima_migration_ts
 from core.schains.process_manager_helper import is_monitor_process_alive
 from core.schains.rpc import (
     check_endpoint_alive,
     check_endpoint_blocks,
     get_endpoint_alive_check_timeout
 )
 from core.schains.external_config import ExternalConfig, ExternalState
-from core.schains.runner import get_container_name, get_image_name, is_new_image_pulled
+from core.schains.runner import (
+    get_container_name,
+    get_ima_container_time_frame,
+    get_image_name,
+    is_new_image_pulled
+)
 from core.schains.skaled_exit_codes import SkaledExitCodes
 from core.schains.volume import is_volume_exists
 
@@ -335,26 +340,36 @@ def ima_container(self) -> CheckRes:
         new_image_pulled = is_new_image_pulled(image_type=IMA_CONTAINER, dutils=self.dutils)
 
         migration_ts = get_ima_migration_ts(self.name)
-        new = time.time() > migration_ts
+        after = time.time() > migration_ts
 
         container_running = self.dutils.is_container_running(container_name)
 
-        updated_image = False
+        updated_image, updated_time_frame = False, False
         if container_running:
-            expected_image = get_image_name(image_type=IMA_CONTAINER, new=new)
+            expected_image = get_image_name(image_type=IMA_CONTAINER, new=after)
             image = self.dutils.get_container_image_name(container_name)
             updated_image = image == expected_image
 
+            time_frame = get_ima_time_frame(self.name, after=after)
+            container_time_frame = get_ima_container_time_frame(self.name, self.dutils)
+
+            updated_time_frame = time_frame == container_time_frame
+            logger.debug(
+                'IMA image %s, container image %s, time frame %d, container_time_frame %d',
+                expected_image, image, time_frame, container_time_frame
+            )
+
         data = {
             'container_running': container_running,
             'updated_image': updated_image,
-            'new_image_pulled': new_image_pulled
+            'new_image_pulled': new_image_pulled,
+            'updated_time_frame': updated_time_frame
         }
         logger.debug(
             '%s, IMA check - %s',
             self.name, data
         )
-        result: bool = container_running and updated_image and new_image_pulled
+        result: bool = all(data.values())
         return CheckRes(result, data=data)
 
     @property

core/schains/ima.py

@@ -28,16 +28,16 @@
 
 from core.schains.config.directory import schain_config_dir
 from core.schains.config.file_manager import ConfigFileManager
-from core.schains.config.helper import get_schain_ports_from_config, get_chain_id
+from core.schains.config.helper import get_chain_id, get_schain_ports_from_config, get_static_params
 from core.ima.schain import get_schain_ima_abi_filepath
 from tools.configs import ENV_TYPE, SGX_SSL_KEY_FILEPATH, SGX_SSL_CERT_FILEPATH, SGX_SERVER_URL
 from tools.configs.containers import IMA_MIGRATION_PATH, CONTAINERS_INFO
 from tools.configs.db import REDIS_URI
 from tools.configs.ima import (
     MAINNET_IMA_ABI_FILEPATH,
     IMA_STATE_CONTAINER_PATH,
-    IMA_TIME_FRAMING,
-    IMA_NETWORK_BROWSER_FILEPATH
+    IMA_NETWORK_BROWSER_FILEPATH,
+    DEFAULT_TIME_FRAME
 )
 from tools.configs.schains import SCHAINS_DIR_PATH
 from tools.helper import safe_load_yml
@@ -145,7 +145,7 @@ def schain_index_to_node_number(node):
     return int(node['schainIndex']) - 1
 
 
-def get_ima_env(schain_name: str, mainnet_chain_id: int) -> ImaEnv:
+def get_ima_env(schain_name: str, mainnet_chain_id: int, time_frame: int) -> ImaEnv:
     schain_config = ConfigFileManager(schain_name).skaled_config
     node_info = schain_config["skaleConfig"]["nodeInfo"]
     bls_key_name = node_info['wallets']['ima']['keyShareName']
@@ -180,7 +180,7 @@ def get_ima_env(schain_name: str, mainnet_chain_id: int) -> ImaEnv:
         cid_schain=schain_chain_id,
         monitoring_port=node_info['imaMonitoringPort'],
         rpc_port=get_ima_rpc_port(schain_name),
-        time_framing=IMA_TIME_FRAMING,
+        time_framing=time_frame,
         network_browser_data_path=IMA_NETWORK_BROWSER_FILEPATH
     )
 
@@ -274,9 +274,23 @@ def get_ima_log_checks():
     return all_ima_healthchecks
 
 
-def get_migration_schedule() -> dict:
-    return safe_load_yml(IMA_MIGRATION_PATH)[ENV_TYPE]
-
-
-def get_migration_ts(name: str) -> int:
-    return get_migration_schedule().get(name, 0)
+def get_migration_ts(name: str, path: str = IMA_MIGRATION_PATH, env_type: str = ENV_TYPE) -> int:
+    if os.path.isfile(path):
+        schedule = safe_load_yml(IMA_MIGRATION_PATH)[env_type]
+        return schedule.get(name, 0)
+    else:
+        return 0
+
+
+def get_ima_time_frame(name: str, after: bool = False) -> int:
+    params = get_static_params()
+    if 'ima' not in params or 'time_frame' not in params['ima']:
+        logger.debug(
+            'IMA time frame intrerval is not set. Using default value %d',
+            DEFAULT_TIME_FRAME
+        )
+        return DEFAULT_TIME_FRAME
+    if after:
+        return params['ima']['time_frame']['after']
+    else:
+        return params['ima']['time_frame']['before']

core/schains/monitor/containers.py

@@ -19,10 +19,12 @@
 
 import logging
 import time
+from typing import Optional
 
 from core.schains.volume import is_volume_exists
 from core.schains.runner import (
     get_container_image,
+    get_ima_container_time_frame,
     get_image_name,
     is_container_exists,
     is_schain_container_failed,
@@ -32,7 +34,7 @@
     run_schain_container
 )
 from core.ima.schain import copy_schain_ima_abi
-from core.schains.ima import ImaData
+from core.schains.ima import get_ima_time_frame, ImaData
 
 from tools.configs import SYNC_NODE
 from tools.configs.containers import (
@@ -53,7 +55,7 @@ def monitor_schain_container(
     download_snapshot=False,
     start_ts=None,
     abort_on_exit: bool = True,
-    dutils=None,
+    dutils: Optional[DockerUtils] = None,
     sync_node: bool = False,
     historic_state: bool = False
 ) -> None:
@@ -128,29 +130,35 @@ def monitor_ima_container(
 
     container_exists = is_container_exists(
         schain_name, container_type=IMA_CONTAINER, dutils=dutils)
-    container_image = get_container_image(schain_name, IMA_CONTAINER, dutils)
-    new_image = get_image_name(image_type=IMA_CONTAINER, new=True)
-
-    expected_image = get_image_name(image_type=IMA_CONTAINER)
-    logger.debug('%s IMA image %s, expected %s', schain_name,
-                 container_image, expected_image)
 
     if time.time() > migration_ts:
-        logger.debug('%s IMA migration time passed', schain_name)
-        expected_image = new_image
-        if container_exists and expected_image != container_image:
-            logger.info(
-                '%s Removing old container as part of IMA migration', schain_name)
-            remove_container(schain_name, IMA_CONTAINER, dutils)
-            container_exists = False
+        logger.debug('IMA migration time passed')
+
+        image = get_image_name(image_type=IMA_CONTAINER, new=True)
+        time_frame = get_ima_time_frame(schain_name, after=True)
+        if container_exists:
+            container_image = get_container_image(schain_name, IMA_CONTAINER, dutils)
+            container_time_frame = get_ima_container_time_frame(schain_name, dutils)
+
+            if image != container_image or time_frame != container_time_frame:
+                logger.info('Removing old container as part of IMA migration')
+                remove_container(schain_name, IMA_CONTAINER, dutils)
+                container_exists = False
+    else:
+        time_frame = get_ima_time_frame(schain_name, after=False)
+        image = get_image_name(image_type=IMA_CONTAINER, new=False)
+    logger.debug('IMA time frame %d', time_frame)
 
     if not container_exists:
-        logger.info('%s No IMA container, creating, image %s',
-                    schain_name, expected_image)
+        logger.info(
+            '%s No IMA container, creating, image %s, time frame %d',
+            schain_name, image, time_frame
+        )
         run_ima_container(
             schain,
             ima_data.chain_id,
-            image=expected_image,
+            image=image,
+            time_frame=time_frame,
             dutils=dutils
         )
     else:

core/schains/runner.py

@@ -226,11 +226,12 @@ def run_schain_container(
 def run_ima_container(
     schain: dict,
     mainnet_chain_id: int,
+    time_frame: int,
     image: str,
     dutils: DockerUtils = None
 ) -> None:
     dutils = dutils or DockerUtils()
-    env = get_ima_env(schain['name'], mainnet_chain_id)
+    env = get_ima_env(schain['name'], mainnet_chain_id, time_frame)
 
     schain_type = get_schain_type(schain['partOfNode'])
     cpu_limit = get_ima_limit(schain_type, MetricType.cpu_shares)
@@ -299,8 +300,8 @@ def is_new_image_pulled(image_type: str, dutils: DockerUtils) -> bool:
     return dutils.pulled(image)
 
 
-def remove_container(schain_name: str, type: str, dutils: DockerUtils):
-    container = get_container_name(type=type, schain_name=schain_name)
+def remove_container(schain_name: str, image_type: str, dutils: DockerUtils):
+    container = get_container_name(image_type=image_type, schain_name=schain_name)
     dutils.safe_rm(container)
 
 
@@ -309,3 +310,8 @@ def pull_new_image(image_type: str, dutils: DockerUtils) -> None:
     if not dutils.pulled(image):
         logger.info('Pulling new image %s', image)
         dutils.pull(image)
+
+
+def get_ima_container_time_frame(schain_name: str, dutils: DockerUtils) -> int:
+    container_name = get_container_name(IMA_CONTAINER, schain_name)
+    return int(dutils.get_container_env_value(container_name, 'TIME_FRAMING'))

tests/conftest.py

@@ -9,6 +9,7 @@
 
 import docker
 import pytest
+import yaml
 
 
 from skale import SkaleManager
@@ -46,7 +47,7 @@
 from core.schains.skaled_status import init_skaled_status, SkaledStatus
 from core.schains.config.skale_manager_opts import SkaleManagerOpts
 
-from tools.configs import META_FILEPATH, SSL_CERTIFICATES_FILEPATH
+from tools.configs import CONFIG_FOLDER, ENV_TYPE, META_FILEPATH, SSL_CERTIFICATES_FILEPATH
 from tools.configs.containers import CONTAINERS_FILEPATH
 from tools.configs.ima import SCHAIN_IMA_ABI_FILEPATH
 from tools.configs.schains import SCHAINS_DIR_PATH
@@ -65,6 +66,7 @@
     generate_cert,
     generate_schain_config,
     get_test_rule_controller,
+    IMA_MIGRATION_TS,
     init_skale_from_wallet,
     init_skale_ima,
     upsert_schain_record_with_config
@@ -617,3 +619,15 @@ def upstreams(schain_db, schain_config):
         yield files
     finally:
         shutil.rmtree(config_folder, ignore_errors=True)
+
+
+@pytest.fixture
+def ima_migration_schedule(schain_db):
+    name = schain_db
+    try:
+        migration_schedule_path = os.path.join(CONFIG_FOLDER, 'ima_migration_schedule.yaml')
+        with open(migration_schedule_path, 'w') as migration_schedule_file:
+            yaml.dump({ENV_TYPE: {name: IMA_MIGRATION_TS}}, migration_schedule_file)
+        yield migration_schedule_path
+    finally:
+        os.remove(migration_schedule_path)

tests/schains/checks_test.py

@@ -202,7 +202,6 @@ def test_exit_code_ok_check(schain_checks, sample_false_checks):
 
 
 def test_ima_container_check(schain_checks, cleanup_ima_containers, dutils):
-    dutils.is_container_running = lambda *args: True
     ts = int(time.time())
     mts = ts + 3600
     name = schain_checks.name
@@ -231,7 +230,7 @@ def test_ima_container_check(schain_checks, cleanup_ima_containers, dutils):
     with mock.patch('core.schains.checks.get_ima_migration_ts', return_value=mts):
         assert not schain_checks.ima_container.status
         image = get_image_name(image_type=IMA_CONTAINER, new=True)
-        run_ima_container(schain, mainnet_chain_id=1,
+        run_ima_container(schain, mainnet_chain_id=1, time_frame=900,
                           image=image, dutils=dutils)
         assert schain_checks.ima_container.status
 

tests/schains/ima_test.py

@@ -4,10 +4,12 @@
 def test_get_ima_env(_schain_name, schain_config):
     ima_env = get_ima_env(
         schain_name=_schain_name,
-        mainnet_chain_id=123
+        mainnet_chain_id=123,
+        time_frame=100
     )
     ima_env_dict = ima_env.to_dict()
     assert len(ima_env_dict) == 23
     assert ima_env_dict['CID_MAIN_NET'] == 123
     assert ima_env_dict['RPC_PORT'] == 10010
+    assert ima_env_dict['TIME_FRAMING'] == 100
     isinstance(ima_env_dict['CID_SCHAIN'], str)