Update failed endpoints for exits log (#234)

Signed-off-by: cyc60 <avsysoev60@gmail.com>

src/common/execution.py

@@ -34,7 +34,8 @@ async def check_hot_wallet_balance() -> None:
 
     if hot_wallet_balance < hot_wallet_min_balance:
         logger.warning(
-            'Wallet balance is too low. At least %s %s is recommended.',
+            'Wallet %s balance is too low. At least %s %s is recommended.',
+            hot_wallet.address,
             Web3.from_wei(hot_wallet_min_balance, 'ether'),
             symbol,
         )

src/exits/tasks.py

@@ -7,10 +7,11 @@
 from web3.types import HexStr
 
 from src.common.contracts import keeper_contract
+from src.common.exceptions import NotEnoughOracleApprovalsError
 from src.common.execution import get_oracles
 from src.common.metrics import metrics
 from src.common.typings import Oracles
-from src.common.utils import get_current_timestamp, is_block_finalized, log_verbose
+from src.common.utils import get_current_timestamp, is_block_finalized
 from src.config.settings import settings
 from src.exits.consensus import get_validator_public_keys
 from src.exits.execution import submit_exit_signatures
@@ -139,8 +140,14 @@ async def _update_exit_signatures(
             oracles_approval = await send_signature_rotation_requests(oracles, oracles_request)
             logger.info('Fetched updated signature for validators: count=%d', len(validators))
             break
-        except Exception as e:
-            log_verbose(e)
+        except NotEnoughOracleApprovalsError as e:
+            logger.error(
+                'Failed to fetch oracle approvals. Received %d out of %d, '
+                'the oracles with endpoints %s have failed to respond.',
+                e.num_votes,
+                e.threshold,
+                ', '.join(e.failed_endpoints),
+            )
 
     tx_hash = await submit_exit_signatures(oracles_approval)
     if not tx_hash:

src/exits/utils.py

@@ -10,6 +10,7 @@
 from sw_utils.decorators import retry_aiohttp_errors
 from web3 import Web3
 
+from src.common.exceptions import NotEnoughOracleApprovalsError
 from src.common.typings import OracleApproval, Oracles, OraclesApproval
 from src.common.utils import format_error, process_oracles_approvals, warning_verbose
 from src.config.settings import (
@@ -32,6 +33,7 @@ async def send_signature_rotation_requests(
     random.shuffle(endpoints)
 
     approvals: dict[ChecksumAddress, OracleApproval] = {}
+    failed_endpoints: list[str] = []
     async with aiohttp.ClientSession() as session:
         for address, replicas in endpoints:
             try:
@@ -45,10 +47,14 @@ async def send_signature_rotation_requests(
                     address,
                     format_error(e),
                 )
+                failed_endpoints.extend(replicas)
                 continue
             approvals[address] = response
-
-    return process_oracles_approvals(approvals, oracles.validators_threshold)
+    try:
+        return process_oracles_approvals(approvals, oracles.validators_threshold)
+    except NotEnoughOracleApprovalsError as e:
+        e.failed_endpoints = failed_endpoints
+        raise
 
 
 # pylint: disable=duplicate-code