Detailed timeout settings (#212)

* Add detailed timeouts

* Edit format_error

Author: evgeny-stakewise

poetry.lock

@@ -10,7 +10,7 @@ python-decouple = "==3.8"
 sentry-sdk = "==1.32.0"
 py-ecc = "==6.0.0"
 multiproof = { git = "https://github.com/stakewise/multiproof.git", rev = "v0.1.5" }
-sw-utils = { git = "https://github.com/stakewise/sw-utils.git", rev = "v0.3.29" }
+sw-utils = { git = "https://github.com/stakewise/sw-utils.git", rev = "v0.3.30" }
 staking-deposit = { git = "https://github.com/ethereum/staking-deposit-cli.git", rev = "v2.4.0" }
 pycryptodomex = "==3.19.0"
 milagro-bls-binding = "==1.9.0"

pyproject.toml

@@ -14,7 +14,7 @@
 from web3 import AsyncWeb3
 
 from src.common.wallet import hot_wallet
-from src.config.settings import DEFAULT_RETRY_TIME, settings
+from src.config.settings import settings
 
 logger = logging.getLogger(__name__)
 
@@ -30,7 +30,11 @@ def create_db_dir(self):
 class ExecutionClient:
     @cached_property
     def client(self) -> AsyncWeb3:
-        w3 = get_execution_client(settings.execution_endpoints, retry_timeout=DEFAULT_RETRY_TIME)
+        w3 = get_execution_client(
+            settings.execution_endpoints,
+            timeout=settings.execution_timeout,
+            retry_timeout=settings.execution_retry_timeout,
+        )
         # Account is required when emitting transactions.
         # For read-only queries account may be omitted.
         if hot_wallet.can_load():
@@ -48,7 +52,11 @@ def __getattr__(self, item):
 class ConsensusClient:
     @cached_property
     def client(self) -> ExtendedAsyncBeacon:
-        return get_consensus_client(settings.consensus_endpoints, retry_timeout=DEFAULT_RETRY_TIME)
+        return get_consensus_client(
+            settings.consensus_endpoints,
+            timeout=settings.consensus_timeout,
+            retry_timeout=settings.consensus_retry_timeout,
+        )
 
     def __getattr__(self, item):
         return getattr(self.client, item)
@@ -57,15 +65,17 @@ def __getattr__(self, item):
 class IpfsFetchRetryClient:
     @cached_property
     def client(self) -> IpfsFetchClient:
-        return IpfsFetchClient(endpoints=settings.ipfs_fetch_endpoints)
+        return IpfsFetchClient(
+            endpoints=settings.ipfs_fetch_endpoints, timeout=settings.ipfs_timeout
+        )
 
-    @retry_ipfs_exception(delay=DEFAULT_RETRY_TIME)
     async def fetch_bytes(self, ipfs_hash: str) -> bytes:
-        return await self.client.fetch_bytes(ipfs_hash)
+        decorator = retry_ipfs_exception(delay=settings.ipfs_retry_timeout)
+        return await decorator(self.client.fetch_bytes)(ipfs_hash)
 
-    @retry_ipfs_exception(delay=DEFAULT_RETRY_TIME)
     async def fetch_json(self, ipfs_hash: str) -> dict | list:
-        return await self.client.fetch_json(ipfs_hash)
+        decorator = retry_ipfs_exception(delay=settings.ipfs_retry_timeout)
+        return await decorator(self.client.fetch_json)(ipfs_hash)
 
 
 db_client = Database()

src/common/clients.py

@@ -48,7 +48,7 @@ async def get_oracles() -> Oracles:
 
     # fetch IPFS record
     ipfs_hash = event['args']['configIpfsHash']
-    config: dict = await ipfs_fetch_client.fetch_json(ipfs_hash)
+    config: dict = await ipfs_fetch_client.fetch_json(ipfs_hash)  # type: ignore
     rewards_threshold = await keeper_contract.get_rewards_min_oracles()
     validators_threshold = await keeper_contract.get_validators_min_oracles()
     endpoints = []

src/common/execution.py

@@ -14,7 +14,7 @@ async def fetch_harvest_params(
     vault_address: ChecksumAddress, ipfs_hash: str, rewards_root: bytes
 ) -> HarvestParams | None:
     ipfs_data = await ipfs_fetch_client.fetch_json(ipfs_hash)
-    for vault_data in ipfs_data['vaults']:
+    for vault_data in ipfs_data['vaults']:  # type: ignore
         if vault_address == Web3.to_checksum_address(vault_data['vault']):
             unlocked_mev_reward = Wei(vault_data['unlocked_mev_reward'])
             reward = Wei(

src/common/ipfs.py

@@ -43,10 +43,6 @@ def log_verbose(e: Exception):
     if settings.verbose:
         logger.exception(e)
     else:
-        if isinstance(e, tenacity.RetryError):
-            # get original error
-            e = e.last_attempt.exception()  # type: ignore
-
         logger.error(format_error(e))
 
 
@@ -56,6 +52,10 @@ def warning_verbose(msg: str, *args: Any) -> None:
 
 
 def format_error(e: Exception) -> str:
+    if isinstance(e, tenacity.RetryError):
+        # get original error
+        e = e.last_attempt.exception()  # type: ignore
+
     if isinstance(e, asyncio.TimeoutError):
         # str(e) returns empty string
         return repr(e)

src/common/utils.py

@@ -29,7 +29,11 @@ class Settings(metaclass=Singleton):
     vault_dir: Path
     network: str
     consensus_endpoints: list[str]
+    consensus_timeout: int
+    consensus_retry_timeout: int
     execution_endpoints: list[str]
+    execution_timeout: int
+    execution_retry_timeout: int
 
     harvest_vault: bool
     verbose: bool
@@ -51,6 +55,8 @@ class Settings(metaclass=Singleton):
     database: Path
     log_level: str
     ipfs_fetch_endpoints: list[str]
+    ipfs_timeout: int
+    ipfs_retry_timeout: int
     validators_fetch_chunk_size: int
     sentry_dsn: str
     pool_size: int | None
@@ -146,13 +152,23 @@ def set(
             'http://cloudflare-ipfs.com,'
             'https://gateway.pinata.cloud,https://ipfs.io',
         )
+        self.ipfs_timeout = decouple_config('IPFS_TIMEOUT', default=60)
+        self.ipfs_retry_timeout = decouple_config('IPFS_RETRY_TIMEOUT', default=120)
         self.validators_fetch_chunk_size = decouple_config(
             'VALIDATORS_FETCH_CHUNK_SIZE', default=100, cast=int
         )
         self.sentry_dsn = decouple_config('SENTRY_DSN', default='')
         self.pool_size = decouple_config(
             'POOL_SIZE', default=None, cast=lambda x: int(x) if x else None
         )
+        self.execution_timeout = decouple_config('EXECUTION_TIMEOUT', default=30, cast=int)
+        self.execution_retry_timeout = decouple_config(
+            'EXECUTION_RETRY_TIMEOUT', default=60, cast=int
+        )
+        self.consensus_timeout = decouple_config('CONSENSUS_TIMEOUT', default=60, cast=int)
+        self.consensus_retry_timeout = decouple_config(
+            'CONSENSUS_RETRY_TIMEOUT', default=120, cast=int
+        )
 
     @property
     def network_config(self) -> NetworkConfig: