Merge pull request #1119 from skalenetwork/develop

2.8.x beta

.flake8

@@ -1,3 +1,3 @@
 [flake8]
 max-line-length = 100
-exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,venv,node_modules,helper-scripts
+exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,venv,.venv,node_modules,helper-scripts

.github/workflows/test.yml

@@ -1,5 +1,5 @@
 name: Test
-on: [push, pull_request]
+on: [push]
 env:
   ETH_PRIVATE_KEY: ${{ secrets.ETH_PRIVATE_KEY }}
   SCHAIN_TYPE: ${{ secrets.SCHAIN_TYPE }}

VERSION

@@ -1 +1 @@
-2.7.2
+2.8.0

core/node.py

@@ -194,7 +194,7 @@ def get_exit_status(self):
             self.config.id)
         schain_statuses = [
             {
-                'name': schain['name'],
+                'name': schain.name,
                 'status': SchainExitStatus.ACTIVE.name
             }
             for schain in active_schains
@@ -207,7 +207,7 @@ def get_exit_status(self):
                 status = SchainExitStatus.LEFT
             else:
                 status = SchainExitStatus.LEAVING
-            schain_name = self.skale.schains.get(schain['schain_id'])['name']
+            schain_name = self.skale.schains.get(schain['schain_id']).name
             if not schain_name:
                 schain_name = '[REMOVED]'
             schain_statuses.append(

core/schains/checks.py

@@ -41,7 +41,7 @@
 from core.schains.dkg.utils import get_secret_key_share_filepath
 from core.schains.firewall.types import IRuleController
 from core.schains.ima import get_ima_time_frame, get_migration_ts as get_ima_migration_ts
-from core.schains.process_manager_helper import is_monitor_process_alive
+from core.schains.process import is_monitor_process_alive
 from core.schains.rpc import (
     check_endpoint_alive,
     check_endpoint_blocks,

core/schains/cleaner.py

@@ -37,7 +37,7 @@
     get_node_ips_from_config,
     get_own_ip_from_config,
 )
-from core.schains.process_manager_helper import terminate_schain_process
+from core.schains.process import ProcessReport, terminate_process
 from core.schains.runner import get_container_name, is_exited
 from core.schains.external_config import ExternalConfig
 from core.schains.types import ContainerType
@@ -109,18 +109,21 @@ def monitor(skale, node_config, dutils=None):
 
     for schain_name in schains_on_node:
         if schain_name not in schain_names_on_contracts:
-            logger.warning(f'sChain {schain_name} was found on node, but not on contracts: \
-{schain_names_on_contracts}, going to remove it!')
+            logger.warning(
+                '%s was found on node, but not on contracts: %s, trying to cleanup',
+                schain_name,
+                schain_names_on_contracts,
+            )
             try:
                 ensure_schain_removed(skale, schain_name, node_config.id, dutils=dutils)
             except Exception:
-                logger.exception(f'sChain removal {schain_name} failed')
+                logger.exception('%s removal failed', schain_name)
     logger.info('Cleanup procedure finished')
 
 
 def get_schain_names_from_contract(skale, node_id):
     schains_on_contract = skale.schains.get_schains_for_node(node_id)
-    return list(map(lambda schain: schain['name'], schains_on_contract))
+    return list(map(lambda schain: schain.name, schains_on_contract))
 
 
 def get_schains_with_containers(dutils=None):
@@ -185,9 +188,10 @@ def remove_schain(
     msg: str,
     dutils: Optional[DockerUtils] = None,
 ) -> None:
-    schain_record = upsert_schain_record(schain_name)
     logger.warning(msg)
-    terminate_schain_process(schain_record)
+    report = ProcessReport(name=schain_name)
+    if report.is_exist():
+        terminate_process(report.pid)
 
     delete_bls_keys(skale, schain_name)
     sync_agent_ranges = get_sync_agent_ranges(skale)
@@ -238,14 +242,14 @@ def cleanup_schain(
         dutils=dutils,
         sync_node=SYNC_NODE,
     )
-    status = checks.get_all()
-    if status['skaled_container'] or is_exited(
+    check_status = checks.get_all()
+    if check_status['skaled_container'] or is_exited(
         schain_name, container_type=ContainerType.schain, dutils=dutils
     ):
         remove_schain_container(schain_name, dutils=dutils)
-    if status['volume']:
+    if check_status['volume']:
         remove_schain_volume(schain_name, dutils=dutils)
-    if status['firewall_rules']:
+    if check_status['firewall_rules']:
         conf = ConfigFileManager(schain_name).skaled_config
         base_port = get_base_port_from_config(conf)
         own_ip = get_own_ip_from_config(conf)
@@ -256,11 +260,11 @@ def cleanup_schain(
         rc.configure(base_port=base_port, own_ip=own_ip, node_ips=node_ips, sync_ip_ranges=ranges)
         rc.cleanup()
     if estate is not None and estate.ima_linked:
-        if status.get('ima_container', False) or is_exited(
+        if check_status.get('ima_container', False) or is_exited(
             schain_name, container_type=ContainerType.ima, dutils=dutils
         ):
             remove_ima_container(schain_name, dutils=dutils)
-    if status['config_dir']:
+    if check_status['config_dir']:
         remove_config_dir(schain_name)
     mark_schain_deleted(schain_name)
 

core/schains/cmd.py

@@ -17,6 +17,8 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+from typing import Optional
+
 from core.schains.config.file_manager import ConfigFileManager
 from core.schains.config.helper import get_schain_ports_from_config
 from core.schains.config.main import get_skaled_container_config_path
@@ -34,7 +36,7 @@ def get_schain_container_cmd(
     download_snapshot: bool = False,
     enable_ssl: bool = True,
     sync_node: bool = False,
-    snapshot_from: str = ''
+    snapshot_from: Optional[str] = None
 ) -> str:
     """Returns parameters that will be passed to skaled binary in the sChain container"""
     opts = get_schain_container_base_opts(schain_name, enable_ssl=enable_ssl, sync_node=sync_node)

core/schains/config/directory.py

@@ -25,10 +25,11 @@
 
 from tools.configs.schains import (
     BASE_SCHAIN_CONFIG_FILEPATH,
+    NODE_CLI_STATUS_FILENAME,
     SCHAINS_DIR_PATH,
     SCHAINS_DIR_PATH_HOST,
     SCHAIN_SCHECKS_FILENAME,
-    SKALED_STATUS_FILENAME
+    SKALED_STATUS_FILENAME,
 )
 
 
@@ -58,6 +59,10 @@ def skaled_status_filepath(name: str) -> str:
     return os.path.join(schain_config_dir(name), SKALED_STATUS_FILENAME)
 
 
+def node_cli_status_filepath(name: str) -> str:
+    return os.path.join(schain_config_dir(name), NODE_CLI_STATUS_FILENAME)
+
+
 def get_schain_check_filepath(schain_name):
     schain_dir_path = schain_config_dir(schain_name)
     return os.path.join(schain_dir_path, SCHAIN_SCHECKS_FILENAME)

core/schains/config/generator.py

@@ -21,6 +21,7 @@
 from dataclasses import dataclass
 
 from skale import Skale
+from skale.contracts.manager.schains import SchainStructure
 from skale.schain_config.generator import get_schain_nodes_with_schains
 from skale.schain_config.ports_allocation import get_schain_base_port_on_node
 from skale.schain_config.rotation_history import get_previous_schain_groups
@@ -34,7 +35,7 @@
 from core.schains.config.predeployed import generate_predeployed_accounts
 from core.schains.config.precompiled import generate_precompiled_accounts
 from core.schains.config.generation import Gen
-from core.schains.config.static_accounts import is_static_accounts, static_accounts
+from core.schains.config.legacy_data import is_static_accounts, static_accounts, static_groups
 from core.schains.config.helper import get_chain_id, get_schain_id
 from core.schains.dkg.utils import get_common_bls_public_key
 from core.schains.limits import get_schain_type
@@ -88,26 +89,26 @@ def to_dict(self):
         }
 
 
-def get_on_chain_owner(schain: dict, generation: int, is_owner_contract: bool) -> str:
+def get_on_chain_owner(schain: SchainStructure, generation: int, is_owner_contract: bool) -> str:
     """
     Returns on-chain owner depending on sChain generation.
     """
     if not is_owner_contract:
-        return schain['mainnetOwner']
+        return schain.mainnet_owner
     if generation >= Gen.ONE:
         return MARIONETTE_ADDRESS
     if generation == Gen.ZERO:
-        return schain['mainnetOwner']
+        return schain.mainnet_owner
 
 
-def get_on_chain_etherbase(schain: dict, generation: int) -> str:
+def get_on_chain_etherbase(schain: SchainStructure, generation: int) -> str:
     """
     Returns on-chain owner depending on sChain generation.
     """
     if generation >= Gen.ONE:
         return ETHERBASE_ADDRESS
     if generation == Gen.ZERO:
-        return schain['mainnetOwner']
+        return schain.mainnet_owner
 
 
 def get_schain_id_for_chain(schain_name: str, generation: int) -> int:
@@ -120,17 +121,17 @@ def get_schain_id_for_chain(schain_name: str, generation: int) -> int:
         return 1
 
 
-def get_schain_originator(schain: dict):
+def get_schain_originator(schain: SchainStructure) -> str:
     """
     Returns address that will be used as an sChain originator
     """
-    if is_zero_address(schain['originator']):
-        return schain['mainnetOwner']
-    return schain['originator']
+    if is_zero_address(schain.originator):
+        return schain.mainnet_owner
+    return schain.originator
 
 
 def generate_schain_config(
-    schain: dict, node_id: int, node: dict, ecdsa_key_name: str,
+    schain: SchainStructure, node_id: int, node: dict, ecdsa_key_name: str,
     rotation_id: int, schain_nodes_with_schains: list,
     node_groups: list, generation: int, is_owner_contract: bool,
     skale_manager_opts: SkaleManagerOpts, schain_base_port: int, common_bls_public_keys: list[str],
@@ -139,7 +140,7 @@ def generate_schain_config(
 ) -> SChainConfig:
     """Main function that is used to generate sChain config"""
     logger.info(
-        f'Going to generate sChain config for {schain["name"]}, '
+        f'Going to generate sChain config for {schain.name}, '
         f'node_name: {node["name"]}, node_id: {node_id}, rotation_id: {rotation_id}'
     )
     if sync_node:
@@ -149,17 +150,23 @@ def generate_schain_config(
 
     on_chain_etherbase = get_on_chain_etherbase(schain, generation)
     on_chain_owner = get_on_chain_owner(schain, generation, is_owner_contract)
-    mainnet_owner = schain['mainnetOwner']
-    schain_type = get_schain_type(schain['partOfNode'])
+    mainnet_owner = schain.mainnet_owner
+    schain_type = get_schain_type(schain.part_of_node)
 
-    schain_id = get_schain_id_for_chain(schain['name'], generation)
+    schain_id = get_schain_id_for_chain(schain.name, generation)
 
     base_config = SChainBaseConfig(BASE_SCHAIN_CONFIG_FILEPATH)
 
     dynamic_params = {
-        'chainID': get_chain_id(schain['name'])
+        'chainID': get_chain_id(schain.name)
     }
 
+    legacy_groups = static_groups(schain.name)
+    logger.debug('Legacy node groups: %s', legacy_groups)
+    logger.debug('Vanilla node groups: %s', node_groups)
+    node_groups.update(legacy_groups)
+    logger.debug('Modified node groups: %s', node_groups)
+
     originator_address = get_schain_originator(schain)
 
     skale_config = generate_skale_section(
@@ -182,13 +189,14 @@ def generate_schain_config(
     )
 
     accounts = {}
-    if is_static_accounts(schain['name']):
-        logger.info(f'Found static account for {schain["name"]}, going to use in config')
-        accounts = static_accounts(schain['name'])['accounts']
+    if is_static_accounts(schain.name):
+        logger.info(f'Found static account for {schain.name}, going to use in config')
+        accounts = static_accounts(schain.name)['accounts']
     else:
         logger.info('Static accounts not found, generating regular accounts section')
         predeployed_accounts = generate_predeployed_accounts(
-            schain_name=schain['name'],
+            schain_name=schain.name,
+            allocation_type=schain.options.allocation_type,
             schain_type=schain_type,
             schain_nodes=schain_nodes_with_schains,
             on_chain_owner=on_chain_owner,
@@ -235,7 +243,7 @@ def generate_schain_config_with_skale(
     node = skale.nodes.get(node_config.id)
     node_groups = get_previous_schain_groups(skale, schain_name)
 
-    is_owner_contract = is_address_contract(skale.web3, schain['mainnetOwner'])
+    is_owner_contract = is_address_contract(skale.web3, schain.mainnet_owner)
 
     skale_manager_opts = init_skale_manager_opts(skale)
     group_index = skale.schains.name_to_id(schain_name)
@@ -246,7 +254,7 @@ def generate_schain_config_with_skale(
     else:
         schain_base_port = get_schain_base_port_on_node(
             schains_on_node,
-            schain['name'],
+            schain.name,
             node['port']
         )
 

core/schains/config/static_accounts.py → core/schains/config/legacy_data.py

@@ -19,8 +19,10 @@
 
 import os
 
+from skale.schain_config.rotation_history import RotationNodeData
+
 from tools.helper import read_json
-from tools.configs import STATIC_ACCOUNTS_FOLDER, ENV_TYPE
+from tools.configs import STATIC_ACCOUNTS_FOLDER, STATIC_GROUPS_FOLDER, ENV_TYPE
 
 
 def static_accounts(schain_name: str) -> dict:
@@ -36,3 +38,25 @@ def static_accounts_filepath(schain_name: str) -> str:
     if not os.path.isdir(static_accounts_env_path):
         return ''
     return os.path.join(static_accounts_env_path, f'schain-{schain_name}.json')
+
+
+def static_groups(schain_name: str) -> dict:
+    static_groups_env_path = static_groups_filepath(schain_name)
+    if not os.path.isfile(static_groups_env_path):
+        return {}
+    groups = read_json(static_groups_env_path)
+    prepared_groups = {}
+    for plain_rotation_id, data in groups.items():
+        rotation_id = int(plain_rotation_id)
+        prepared_groups[rotation_id] = data
+        prepared_nodes = prepared_groups[rotation_id]['nodes']
+        node_ids_string = list(data['nodes'].keys())
+        for node_id_string in node_ids_string:
+            node_info = prepared_nodes.pop(node_id_string)
+            prepared_nodes[int(node_id_string)] = RotationNodeData(*node_info)
+    return prepared_groups
+
+
+def static_groups_filepath(schain_name: str) -> str:
+    static_groups_env_path = os.path.join(STATIC_GROUPS_FOLDER, ENV_TYPE)
+    return os.path.join(static_groups_env_path, f'schain-{schain_name}.json')

core/schains/config/node_info.py

@@ -81,7 +81,7 @@ def generate_current_node_info(
     sync_node: bool = False, archive: bool = False, catchup: bool = False
 ) -> CurrentNodeInfo:
     wallets = generate_wallets_config(
-        schain['name'],
+        schain.name,
         rotation_id,
         sync_node,
         nodes_in_schain,

core/schains/config/predeployed.py

@@ -19,6 +19,7 @@
 
 import logging
 
+from skale.dataclasses.schain_options import AllocationType
 from skale.wallets.web3_wallet import public_key_to_address
 
 from etherbase_predeployed import (
@@ -60,6 +61,7 @@
 def generate_predeployed_accounts(
     schain_name: str,
     schain_type: SchainType,
+    allocation_type: AllocationType,
     schain_nodes: list,
     on_chain_owner: str,
     mainnet_owner: str,
@@ -80,6 +82,7 @@ def generate_predeployed_accounts(
     if generation >= Gen.ONE:
         v1_predeployed_contracts = generate_v1_predeployed_contracts(
             schain_type=schain_type,
+            allocation_type=allocation_type,
             on_chain_owner=on_chain_owner,
             mainnet_owner=mainnet_owner,
             originator_address=originator_address,
@@ -94,6 +97,7 @@ def generate_predeployed_accounts(
 
 def generate_v1_predeployed_contracts(
     schain_type: SchainType,
+    allocation_type: AllocationType,
     on_chain_owner: str,
     mainnet_owner: str,
     originator_address: str,
@@ -127,7 +131,7 @@ def generate_v1_predeployed_contracts(
         ima=message_proxy_for_schain_address,
     )
 
-    allocated_storage = get_fs_allocated_storage(schain_type)
+    allocated_storage = get_fs_allocated_storage(schain_type, allocation_type)
     filestorage_generator = UpgradeableFileStorageGenerator()
     filestorage_predeployed = filestorage_generator.generate_allocation(
         contract_address=FILESTORAGE_ADDRESS,