From 901cdcd7a094848be27e023317d7ebeba45bfffe Mon Sep 17 00:00:00 2001 From: Evgeny Gusarov Date: Sat, 18 Nov 2023 17:59:37 +0300 Subject: [PATCH 1/2] Add oracles cache --- src/common/abi/Multicall.json | 440 ++++++++++++++++++++++++++++++++++ src/common/contracts.py | 26 +- src/common/execution.py | 85 +++++-- src/common/typings.py | 10 +- src/config/networks.py | 13 + src/exits/tasks.py | 3 +- src/validators/tasks.py | 3 +- 7 files changed, 556 insertions(+), 24 deletions(-) create mode 100644 src/common/abi/Multicall.json diff --git a/src/common/abi/Multicall.json b/src/common/abi/Multicall.json new file mode 100644 index 00000000..bc6b3307 --- /dev/null +++ b/src/common/abi/Multicall.json @@ -0,0 +1,440 @@ +[ + { + "inputs": [ + { + "components": [ + { + "internalType": "address", + "name": "target", + "type": "address" + }, + { + "internalType": "bytes", + "name": "callData", + "type": "bytes" + } + ], + "internalType": "struct Multicall3.Call[]", + "name": "calls", + "type": "tuple[]" + } + ], + "name": "aggregate", + "outputs": [ + { + "internalType": "uint256", + "name": "blockNumber", + "type": "uint256" + }, + { + "internalType": "bytes[]", + "name": "returnData", + "type": "bytes[]" + } + ], + "stateMutability": "payable", + "type": "function" + }, + { + "inputs": [ + { + "components": [ + { + "internalType": "address", + "name": "target", + "type": "address" + }, + { + "internalType": "bool", + "name": "allowFailure", + "type": "bool" + }, + { + "internalType": "bytes", + "name": "callData", + "type": "bytes" + } + ], + "internalType": "struct Multicall3.Call3[]", + "name": "calls", + "type": "tuple[]" + } + ], + "name": "aggregate3", + "outputs": [ + { + "components": [ + { + "internalType": "bool", + "name": "success", + "type": "bool" + }, + { + "internalType": "bytes", + "name": "returnData", + "type": "bytes" + } + ], + "internalType": "struct Multicall3.Result[]", + "name": "returnData", + "type": "tuple[]" + } + ], + "stateMutability": "payable", + "type": "function" + }, + { + "inputs": [ + { + "components": [ + { + "internalType": "address", + "name": "target", + "type": "address" + }, + { + "internalType": "bool", + "name": "allowFailure", + "type": "bool" + }, + { + "internalType": "uint256", + "name": "value", + "type": "uint256" + }, + { + "internalType": "bytes", + "name": "callData", + "type": "bytes" + } + ], + "internalType": "struct Multicall3.Call3Value[]", + "name": "calls", + "type": "tuple[]" + } + ], + "name": "aggregate3Value", + "outputs": [ + { + "components": [ + { + "internalType": "bool", + "name": "success", + "type": "bool" + }, + { + "internalType": "bytes", + "name": "returnData", + "type": "bytes" + } + ], + "internalType": "struct Multicall3.Result[]", + "name": "returnData", + "type": "tuple[]" + } + ], + "stateMutability": "payable", + "type": "function" + }, + { + "inputs": [ + { + "components": [ + { + "internalType": "address", + "name": "target", + "type": "address" + }, + { + "internalType": "bytes", + "name": "callData", + "type": "bytes" + } + ], + "internalType": "struct Multicall3.Call[]", + "name": "calls", + "type": "tuple[]" + } + ], + "name": "blockAndAggregate", + "outputs": [ + { + "internalType": "uint256", + "name": "blockNumber", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "blockHash", + "type": "bytes32" + }, + { + "components": [ + { + "internalType": "bool", + "name": "success", + "type": "bool" + }, + { + "internalType": "bytes", + "name": "returnData", + "type": "bytes" + } + ], + "internalType": "struct Multicall3.Result[]", + "name": "returnData", + "type": "tuple[]" + } + ], + "stateMutability": "payable", + "type": "function" + }, + { + "inputs": [], + "name": "getBasefee", + "outputs": [ + { + "internalType": "uint256", + "name": "basefee", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "blockNumber", + "type": "uint256" + } + ], + "name": "getBlockHash", + "outputs": [ + { + "internalType": "bytes32", + "name": "blockHash", + "type": "bytes32" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getBlockNumber", + "outputs": [ + { + "internalType": "uint256", + "name": "blockNumber", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getChainId", + "outputs": [ + { + "internalType": "uint256", + "name": "chainid", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getCurrentBlockCoinbase", + "outputs": [ + { + "internalType": "address", + "name": "coinbase", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getCurrentBlockDifficulty", + "outputs": [ + { + "internalType": "uint256", + "name": "difficulty", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getCurrentBlockGasLimit", + "outputs": [ + { + "internalType": "uint256", + "name": "gaslimit", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getCurrentBlockTimestamp", + "outputs": [ + { + "internalType": "uint256", + "name": "timestamp", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "addr", + "type": "address" + } + ], + "name": "getEthBalance", + "outputs": [ + { + "internalType": "uint256", + "name": "balance", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getLastBlockHash", + "outputs": [ + { + "internalType": "bytes32", + "name": "blockHash", + "type": "bytes32" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "bool", + "name": "requireSuccess", + "type": "bool" + }, + { + "components": [ + { + "internalType": "address", + "name": "target", + "type": "address" + }, + { + "internalType": "bytes", + "name": "callData", + "type": "bytes" + } + ], + "internalType": "struct Multicall3.Call[]", + "name": "calls", + "type": "tuple[]" + } + ], + "name": "tryAggregate", + "outputs": [ + { + "components": [ + { + "internalType": "bool", + "name": "success", + "type": "bool" + }, + { + "internalType": "bytes", + "name": "returnData", + "type": "bytes" + } + ], + "internalType": "struct Multicall3.Result[]", + "name": "returnData", + "type": "tuple[]" + } + ], + "stateMutability": "payable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "bool", + "name": "requireSuccess", + "type": "bool" + }, + { + "components": [ + { + "internalType": "address", + "name": "target", + "type": "address" + }, + { + "internalType": "bytes", + "name": "callData", + "type": "bytes" + } + ], + "internalType": "struct Multicall3.Call[]", + "name": "calls", + "type": "tuple[]" + } + ], + "name": "tryBlockAndAggregate", + "outputs": [ + { + "internalType": "uint256", + "name": "blockNumber", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "blockHash", + "type": "bytes32" + }, + { + "components": [ + { + "internalType": "bool", + "name": "success", + "type": "bool" + }, + { + "internalType": "bytes", + "name": "returnData", + "type": "bytes" + } + ], + "internalType": "struct Multicall3.Result[]", + "name": "returnData", + "type": "tuple[]" + } + ], + "stateMutability": "payable", + "type": "function" + } +] diff --git a/src/common/contracts.py b/src/common/contracts.py index f927a791..37771e7f 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -1,13 +1,14 @@ import json import os from functools import cached_property +from typing import cast from eth_typing import HexStr from sw_utils.typings import Bytes32 from web3 import Web3 from web3.contract import AsyncContract from web3.contract.contract import ContractEvent -from web3.types import BlockNumber, ChecksumAddress, EventData +from web3.types import BlockIdentifier, BlockNumber, ChecksumAddress, EventData from src.common.clients import execution_client from src.common.typings import RewardVoteInfo @@ -141,12 +142,14 @@ class KeeperContract(ContractWrapper): abi_path = 'abi/IKeeper.json' settings_key = 'KEEPER_CONTRACT_ADDRESS' - async def get_config_updated_event(self) -> EventData | None: + async def get_config_updated_event( + self, from_block: BlockNumber | None = None, to_block: BlockNumber | None = None + ) -> EventData | None: """Fetches the last oracles config updated event.""" return await self._get_last_event( self.events.ConfigUpdated, - from_block=settings.network_config.KEEPER_GENESIS_BLOCK, - to_block=await execution_client.eth.get_block_number(), + from_block=from_block or settings.network_config.KEEPER_GENESIS_BLOCK, + to_block=to_block or await execution_client.eth.get_block_number(), ) async def get_last_rewards_update(self) -> RewardVoteInfo | None: @@ -190,8 +193,23 @@ async def can_harvest(self, vault_address: ChecksumAddress) -> bool: return await self.contract.functions.canHarvest(vault_address).call() +class MulticallContract(ContractWrapper): + abi_path = 'abi/Multicall.json' + settings_key = 'MULTICALL_CONTRACT_ADDRESS' + + async def aggregate( + self, + data: list[tuple[ChecksumAddress, bool, HexStr]], + block_number: BlockNumber | None = None, + ) -> list: + return await self.contract.functions.aggregate3(data).call( + block_identifier=cast(BlockIdentifier, block_number) + ) + + vault_contract = VaultContract() validators_registry_contract = ValidatorsRegistryContract() keeper_contract = KeeperContract() v2_pool_contract = V2PoolContract() v2_pool_escrow_contract = V2PoolEscrowContract() +multicall_contract = MulticallContract() diff --git a/src/common/execution.py b/src/common/execution.py index 9da26baf..59b8d81f 100644 --- a/src/common/execution.py +++ b/src/common/execution.py @@ -1,14 +1,16 @@ import logging import statistics +from typing import cast +from eth_typing import BlockNumber from web3 import Web3 from web3.exceptions import MethodUnavailable from web3.types import BlockIdentifier, Wei from src.common.clients import execution_client, ipfs_fetch_client -from src.common.contracts import keeper_contract +from src.common.contracts import keeper_contract, multicall_contract from src.common.metrics import metrics -from src.common.typings import Oracles +from src.common.typings import Oracles, OraclesCache from src.common.wallet import hot_wallet from src.config.settings import settings @@ -41,25 +43,74 @@ async def check_hot_wallet_balance() -> None: ) +_oracles_cache: OraclesCache | None = None + + +async def update_oracles_cache() -> None: + """ + Fetches latest oracle config from IPFS. Uses cache if possible. + """ + global _oracles_cache # pylint: disable=global-statement + + # Find the latest block for which oracle config is cached + if _oracles_cache: + from_block = BlockNumber(_oracles_cache.checkpoint_block + 1) + else: + from_block = settings.network_config.KEEPER_GENESIS_BLOCK + + to_block = await execution_client.eth.get_block_number() + + if from_block > to_block: + return + + logger.debug('update_oracles_cache: get logs from_block %s, to_block %s', from_block, to_block) + event = await keeper_contract.get_config_updated_event(from_block=from_block, to_block=to_block) + if event: + ipfs_hash = event['args']['configIpfsHash'] + config = cast(dict, await ipfs_fetch_client.fetch_json(ipfs_hash)) + else: + config = _oracles_cache.config # type: ignore + + rewards_threshold_call = keeper_contract.encode_abi(fn_name='rewardsMinOracles', args=[]) + validators_threshold_call = keeper_contract.encode_abi(fn_name='validatorsMinOracles', args=[]) + multicall_response = await multicall_contract.aggregate( + [ + (keeper_contract.address, False, rewards_threshold_call), + (keeper_contract.address, False, validators_threshold_call), + ], + block_number=to_block, + ) + rewards_threshold = Web3.to_int(multicall_response[0][1]) + validators_threshold = Web3.to_int(multicall_response[1][1]) + + if _oracles_cache: + _oracles_cache.config = config + _oracles_cache.validators_threshold = validators_threshold + _oracles_cache.rewards_threshold = rewards_threshold + _oracles_cache.checkpoint_block = to_block + else: + _oracles_cache = OraclesCache( + config=config, + validators_threshold=validators_threshold, + rewards_threshold=rewards_threshold, + checkpoint_block=to_block, + ) + + async def get_oracles() -> Oracles: - """Fetches oracles config.""" - event = await keeper_contract.get_config_updated_event() - if not event: - raise ValueError('Failed to fetch IPFS hash of oracles config') - - # fetch IPFS record - ipfs_hash = event['args']['configIpfsHash'] - config: dict = await ipfs_fetch_client.fetch_json(ipfs_hash) # type: ignore - rewards_threshold = await keeper_contract.get_rewards_min_oracles() - validators_threshold = await keeper_contract.get_validators_min_oracles() + if not _oracles_cache: + await update_oracles_cache() + + oracles_cache = cast(OraclesCache, _oracles_cache) + + config = oracles_cache.config + rewards_threshold = oracles_cache.rewards_threshold + validators_threshold = oracles_cache.validators_threshold + endpoints = [] public_keys = [] for oracle in config['oracles']: - if endpoint := oracle.get('endpoint'): - replicas = [endpoint] - else: - replicas = oracle['endpoints'] - endpoints.append(replicas) + endpoints.append(oracle['endpoints']) public_keys.append(oracle['public_key']) if not 1 <= rewards_threshold <= len(config['oracles']): diff --git a/src/common/typings.py b/src/common/typings.py index d86329dd..6ab16912 100644 --- a/src/common/typings.py +++ b/src/common/typings.py @@ -2,7 +2,7 @@ from functools import cached_property from eth_keys.datatypes import PublicKey -from eth_typing import ChecksumAddress, HexStr +from eth_typing import BlockNumber, ChecksumAddress, HexStr from web3 import Web3 from web3.types import Wei @@ -29,6 +29,14 @@ def addresses(self) -> list[ChecksumAddress]: return res +@dataclass +class OraclesCache: + checkpoint_block: BlockNumber + config: dict + validators_threshold: int + rewards_threshold: int + + @dataclass class RewardVoteInfo: rewards_root: bytes diff --git a/src/config/networks.py b/src/config/networks.py index 8f56e049..fda9b0af 100644 --- a/src/config/networks.py +++ b/src/config/networks.py @@ -37,6 +37,7 @@ class NetworkConfig: HOT_WALLET_MIN_BALANCE: Wei SHAPELLA_FORK_VERSION: bytes SHAPELLA_EPOCH: int + MULTICALL_CONTRACT_ADDRESS: ChecksumAddress @property def SHAPELLA_FORK(self) -> ConsensusFork: @@ -89,6 +90,9 @@ def IS_SUPPORT_V2_MIGRATION(self) -> bool: HOT_WALLET_MIN_BALANCE=Web3.to_wei('0.03', 'ether'), SHAPELLA_FORK_VERSION=Web3.to_bytes(hexstr=HexStr('0x03000000')), SHAPELLA_EPOCH=194048, + MULTICALL_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0xcA11bde05977b3631167028862bE2a173976CA11' + ), ), HOLESKY: NetworkConfig( SYMBOL='HolETH', @@ -117,6 +121,9 @@ def IS_SUPPORT_V2_MIGRATION(self) -> bool: HOT_WALLET_MIN_BALANCE=Web3.to_wei('0.03', 'ether'), SHAPELLA_FORK_VERSION=Web3.to_bytes(hexstr=HexStr('0x01017000')), SHAPELLA_EPOCH=256, + MULTICALL_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0xcA11bde05977b3631167028862bE2a173976CA11' + ), ), GOERLI: NetworkConfig( SYMBOL='GoerliETH', @@ -151,6 +158,9 @@ def IS_SUPPORT_V2_MIGRATION(self) -> bool: HOT_WALLET_MIN_BALANCE=Web3.to_wei('0.03', 'ether'), SHAPELLA_FORK_VERSION=Web3.to_bytes(hexstr=HexStr('0x03001020')), SHAPELLA_EPOCH=162304, + MULTICALL_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0xcA11bde05977b3631167028862bE2a173976CA11' + ), ), GNOSIS: NetworkConfig( SYMBOL='xDAI', @@ -180,5 +190,8 @@ def IS_SUPPORT_V2_MIGRATION(self) -> bool: HOT_WALLET_MIN_BALANCE=Web3.to_wei('0.03', 'ether'), SHAPELLA_FORK_VERSION=Web3.to_bytes(hexstr=HexStr('0x0')), SHAPELLA_EPOCH=0, # todo + MULTICALL_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0xcA11bde05977b3631167028862bE2a173976CA11' + ), ), } diff --git a/src/exits/tasks.py b/src/exits/tasks.py index d0e33ea6..300c8f28 100644 --- a/src/exits/tasks.py +++ b/src/exits/tasks.py @@ -8,7 +8,7 @@ from src.common.contracts import keeper_contract from src.common.exceptions import NotEnoughOracleApprovalsError -from src.common.execution import get_oracles +from src.common.execution import get_oracles, update_oracles_cache from src.common.metrics import metrics from src.common.typings import Oracles from src.common.utils import get_current_timestamp, is_block_finalized @@ -34,6 +34,7 @@ async def update_exit_signatures( keystores: Keystores, remote_signer_config: RemoteSignerConfiguration | None, ) -> None: + await update_oracles_cache() oracles = await get_oracles() update_block = await _fetch_last_update_block() if update_block and not await is_block_finalized(update_block): diff --git a/src/validators/tasks.py b/src/validators/tasks.py index 6aa94903..79007e97 100644 --- a/src/validators/tasks.py +++ b/src/validators/tasks.py @@ -9,7 +9,7 @@ from src.common.clients import ipfs_fetch_client from src.common.contracts import v2_pool_escrow_contract, validators_registry_contract from src.common.exceptions import NotEnoughOracleApprovalsError -from src.common.execution import check_gas_price, get_oracles +from src.common.execution import check_gas_price, get_oracles, update_oracles_cache from src.common.metrics import metrics from src.common.typings import Oracles from src.common.utils import MGNO_RATE, WAD, get_current_timestamp @@ -72,6 +72,7 @@ async def register_validators( return # get latest oracles + await update_oracles_cache() oracles = await get_oracles() validators_count = min(oracles.validators_approval_batch_limit, validators_count) From 8631d082ca453ce54349da2c806637d65ac8e954 Mon Sep 17 00:00:00 2001 From: Evgeny Gusarov Date: Thu, 23 Nov 2023 14:13:26 +0300 Subject: [PATCH 2/2] Review fix --- src/common/execution.py | 21 +++++++-------------- src/exits/tasks.py | 3 +-- src/validators/tasks.py | 3 +-- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/src/common/execution.py b/src/common/execution.py index 59b8d81f..a4c89188 100644 --- a/src/common/execution.py +++ b/src/common/execution.py @@ -83,23 +83,16 @@ async def update_oracles_cache() -> None: rewards_threshold = Web3.to_int(multicall_response[0][1]) validators_threshold = Web3.to_int(multicall_response[1][1]) - if _oracles_cache: - _oracles_cache.config = config - _oracles_cache.validators_threshold = validators_threshold - _oracles_cache.rewards_threshold = rewards_threshold - _oracles_cache.checkpoint_block = to_block - else: - _oracles_cache = OraclesCache( - config=config, - validators_threshold=validators_threshold, - rewards_threshold=rewards_threshold, - checkpoint_block=to_block, - ) + _oracles_cache = OraclesCache( + config=config, + validators_threshold=validators_threshold, + rewards_threshold=rewards_threshold, + checkpoint_block=to_block, + ) async def get_oracles() -> Oracles: - if not _oracles_cache: - await update_oracles_cache() + await update_oracles_cache() oracles_cache = cast(OraclesCache, _oracles_cache) diff --git a/src/exits/tasks.py b/src/exits/tasks.py index 300c8f28..d0e33ea6 100644 --- a/src/exits/tasks.py +++ b/src/exits/tasks.py @@ -8,7 +8,7 @@ from src.common.contracts import keeper_contract from src.common.exceptions import NotEnoughOracleApprovalsError -from src.common.execution import get_oracles, update_oracles_cache +from src.common.execution import get_oracles from src.common.metrics import metrics from src.common.typings import Oracles from src.common.utils import get_current_timestamp, is_block_finalized @@ -34,7 +34,6 @@ async def update_exit_signatures( keystores: Keystores, remote_signer_config: RemoteSignerConfiguration | None, ) -> None: - await update_oracles_cache() oracles = await get_oracles() update_block = await _fetch_last_update_block() if update_block and not await is_block_finalized(update_block): diff --git a/src/validators/tasks.py b/src/validators/tasks.py index 79007e97..6aa94903 100644 --- a/src/validators/tasks.py +++ b/src/validators/tasks.py @@ -9,7 +9,7 @@ from src.common.clients import ipfs_fetch_client from src.common.contracts import v2_pool_escrow_contract, validators_registry_contract from src.common.exceptions import NotEnoughOracleApprovalsError -from src.common.execution import check_gas_price, get_oracles, update_oracles_cache +from src.common.execution import check_gas_price, get_oracles from src.common.metrics import metrics from src.common.typings import Oracles from src.common.utils import MGNO_RATE, WAD, get_current_timestamp @@ -72,7 +72,6 @@ async def register_validators( return # get latest oracles - await update_oracles_cache() oracles = await get_oracles() validators_count = min(oracles.validators_approval_batch_limit, validators_count)