Rename stcd -> statsd_client. Extract telegraf mem_limit

core/monitoring.py

@@ -31,7 +31,8 @@
     TELEGRAF,
     TELEGRAF_CONTAINER_NAME, TELEGRAF_IMAGE,
     TELEGRAF_TEMPLATE_PATH,
-    TELEGRAF_CONFIG_PATH
+    TELEGRAF_CONFIG_PATH,
+    TELEGRAF_MEM_LIMIT
 )
 
 logger = logging.getLogger(__name__)
@@ -79,7 +80,7 @@ def ensure_telegraf_running(dutils: Optional[DockerUtils] = None) -> None:
                 f'{SKALE_DIR_HOST}/node_data/telegraf': {'bind': '/var/lib/telegraf', 'mode': 'rw'},
                 '/var/run/skale/': {'bind': '/var/run/skale', 'mode': 'rw'}
             },
-            mem_limit='1GB'
+            mem_limit=TELEGRAF_MEM_LIMIT
         )
 
 

core/schains/checks.py

@@ -127,7 +127,7 @@ def get_all(self,
                 logger.debug('Running check %s', name)
                 checks_status[name] = getattr(self, name).status
         if expose:
-            send_to_statsd(self.stcd, self.get_name(), checks_status)
+            send_to_statsd(self.statsd_client, self.get_name(), checks_status)
         if log:
             log_checks_dict(self.get_name(), checks_status)
         if save:
@@ -171,7 +171,7 @@ def __init__(self,
         self.cfm: ConfigFileManager = ConfigFileManager(
             schain_name=schain_name
         )
-        self.stcd = get_statsd_client()
+        self.statsd_client = get_statsd_client()
 
     def get_name(self) -> str:
         return self.name
@@ -262,7 +262,7 @@ def __init__(
         self.cfm: ConfigFileManager = ConfigFileManager(
             schain_name=schain_name
         )
-        self.stcd = get_statsd_client()
+        self.statsd_client = get_statsd_client()
 
     def get_name(self) -> str:
         return self.name
@@ -522,7 +522,7 @@ def log_checks_dict(schain_name, checks_dict):
         )
 
 
-def send_to_statsd(stcd: statsd.StatsClient, schain_name: str, checks_dict: dict) -> None:
+def send_to_statsd(statsd_client: statsd.StatsClient, schain_name: str, checks_dict: dict) -> None:
     for check, result in checks_dict.items():
         mname = f'admin.checks.{schain_name}.{check}'
-        stcd.gauge(mname, int(result))
+        statsd_client.gauge(mname, int(result))

core/schains/dkg/client.py

@@ -172,7 +172,7 @@ def __init__(
         self.complaint_error_event_hash = self.skale.web3.to_hex(self.skale.web3.keccak(
             text="ComplaintError(string)"
         ))
-        self.stcd = get_statsd_client()
+        self.statsd_client = get_statsd_client()
         self._last_completed_step = step  # last step
         logger.info(f'sChain: {self.schain_name}. DKG timeout is {self.dkg_timeout}')
 
@@ -182,7 +182,7 @@ def last_completed_step(self) -> DKGStep:
 
     @last_completed_step.setter
     def last_completed_step(self, value: DKGStep):
-        self.stcd.gauge(f'admin.dkg.last_completed_step.{self.schain_name}', value)
+        self.statsd_client.gauge(f'admin.dkg.last_completed_step.{self.schain_name}', value)
         self._last_completed_step = value
 
     def is_channel_opened(self):

core/schains/monitor/action.py

@@ -166,7 +166,7 @@ def __init__(
         self.cfm: ConfigFileManager = ConfigFileManager(
             schain_name=self.schain['name']
         )
-        self.stcd = get_statsd_client()
+        self.statsd_client = get_statsd_client()
         super().__init__(name=schain['name'])
 
     @BaseActionManager.monitor_block
@@ -178,7 +178,7 @@ def config_dir(self) -> bool:
     @BaseActionManager.monitor_block
     def dkg(self) -> bool:
         initial_status = self.checks.dkg.status
-        with self.stcd.timer(f'admin.dkg.{self.name}'):
+        with self.statsd_client.timer(f'admin.dkg.{self.name}'):
             if not initial_status:
                 logger.info('Initing dkg client')
                 dkg_client = get_dkg_client(
@@ -212,7 +212,7 @@ def dkg(self) -> bool:
 
     @BaseActionManager.monitor_block
     def upstream_config(self) -> bool:
-        with self.stcd.timer(f'admin.upstream_config.{self.name}'):
+        with self.statsd_client.timer(f'admin.upstream_config.{self.name}'):
             logger.info(
                 'Creating new upstream_config rotation_id: %s, stream: %s',
                 self.rotation_data.get('rotation_id'), self.stream_version
@@ -317,7 +317,7 @@ def __init__(
 
         self.esfm = ExitScheduleFileManager(schain['name'])
         self.dutils = dutils or DockerUtils()
-        self.stcd = get_statsd_client()
+        self.statsd_client = get_statsd_client()
 
         self.node_options = node_options or NodeOptions()
 
@@ -348,14 +348,17 @@ def firewall_rules(self, upstream: bool = False) -> bool:
 
             ranges = self.econfig.ranges
             logger.info('Adding ranges %s', ranges)
-            with self.stcd.timer(f'admin.firewall.{self.name}'):
+            with self.statsd_client.timer(f'admin.firewall.{self.name}'):
                 self.rc.configure(
                     base_port=base_port,
                     own_ip=own_ip,
                     node_ips=node_ips,
                     sync_ip_ranges=ranges
                 )
-                self.stcd.gauge(f'admin.expected_rules.{self.name}', len(self.rc.expected_rules()))
+                self.statsd_client.gauge(
+                    f'admin.expected_rules.{self.name}',
+                    len(self.rc.expected_rules())
+                )
                 self.rc.sync()
         return initial_status
 

core/schains/monitor/main.py

@@ -121,11 +121,11 @@ def run_config_pipeline(
     else:
         logger.info('Regular node mode, running config monitor')
         mon = RegularConfigMonitor(config_am, config_checks)
-    stcd = get_statsd_client()
+    statsd_client = get_statsd_client()
 
-    stcd.incr(f'admin.config.pipeline.{name}.{mon.__class__.__name__}')
-    stcd.gauge(f'admin.schain.rotation_id.{name}', rotation_data['rotation_id'])
-    with stcd.timer(f'admin.config.pipeline.{name}.duration'):
+    statsd_client.incr(f'admin.config.pipeline.{name}.{mon.__class__.__name__}')
+    statsd_client.gauge(f'admin.schain.rotation_id.{name}', rotation_data['rotation_id'])
+    with statsd_client.timer(f'admin.config.pipeline.{name}.duration'):
         mon.run()
 
 
@@ -177,9 +177,9 @@ def run_skaled_pipeline(
         automatic_repair=automatic_repair
     )
 
-    stcd = get_statsd_client()
-    stcd.incr(f'schain.skaled.pipeline.{name}.{mon.__name__}')
-    with stcd.timer(f'admin.skaled.pipeline.{name}.duration'):
+    statsd_client = get_statsd_client()
+    statsd_client.incr(f'schain.skaled.pipeline.{name}.{mon.__name__}')
+    with statsd_client.timer(f'admin.skaled.pipeline.{name}.duration'):
         mon(skaled_am, skaled_checks).run()
 
 
@@ -219,10 +219,10 @@ def create_and_execute_tasks(
         schain_record.sync_config_run, schain_record.config_version, stream_version
     )
 
-    stcd = get_statsd_client()
+    statsd_client = get_statsd_client()
     monitor_last_seen_ts = schain_record.monitor_last_seen.timestamp()
-    stcd.incr(f'admin.schain.monitor.{name}')
-    stcd.gauge(f'admin.schain.monitor_last_seen.{name}', monitor_last_seen_ts)
+    statsd_client.incr(f'admin.schain.monitor.{name}')
+    statsd_client.gauge(f'admin.schain.monitor_last_seen.{name}', monitor_last_seen_ts)
 
     tasks = []
     if not leaving_chain:

core/schains/monitor/skaled_monitor.py

@@ -44,7 +44,7 @@ def __init__(
     ) -> None:
         self.am = action_manager
         self.checks = checks
-        self.stcd = get_statsd_client()
+        self.statsd_client = get_statsd_client()
 
     @abstractmethod
     def execute(self) -> None:

tools/configs/monitoring.py

@@ -19,3 +19,4 @@
 TELEGRAF_CONTAINER_NAME = 'skale_telegraf'
 TELEGRAF_SERVICE_NAME = 'telegraf'
 TELEGRAF_IMAGE = 'telegraf:1.27.4'
+TELEGRAF_MEM_LIMIT = os.getenv('TELEGRAF_MEM_LIMIT', '1GB')

tools/resources.py

@@ -9,12 +9,12 @@
 db = SqliteDatabase(DB_FILE, DB_PRAGMAS)
 cpool: redis.ConnectionPool = redis.ConnectionPool.from_url(REDIS_URI)
 rs: redis.Redis = redis.Redis(connection_pool=cpool)
-stcd = statsd.StatsClient(STATSD_HOST, STATSD_PORT)
+statsd_client = statsd.StatsClient(STATSD_HOST, STATSD_PORT)
 
 
 def get_database():
     return db
 
 
 def get_statsd_client():
-    return stcd
+    return statsd_client