diff --git a/.env.sample b/.env.sample index 30c8c80..880ea87 100644 --- a/.env.sample +++ b/.env.sample @@ -28,3 +28,21 @@ ETHERSCAN_API_KEY= # Query ID for the aggregate query on Dune AGGREGATE_QUERY_ID= + +# Node for each chain we want to run a sync job on +NODE_URL_MAINNET= +NODE_URL_GNOSIS= +NODE_URL_ARBITRUM= + +# The network which we run a sync job on. +# Current options are: {"mainnet", "xdai", "arbitrum-one"} +NETWORK= + +# The prefix of the dune table where we sync the various mainnet price feeds +PRICE_FEED_TARGET_TABLE + +# The prefix of the dune table where we sync the monthly raw batch data +BATCH_DATA_TARGET_TABLE = "batch_data" + +# Dune api timeout parameter, where we recommend to set it to 600 +DUNE_API_REQUEST_TIMEOUT = 600 diff --git a/src/fetch/orderbook.py b/src/fetch/orderbook.py index 13e9162..5e929cc 100644 --- a/src/fetch/orderbook.py +++ b/src/fetch/orderbook.py @@ -19,6 +19,7 @@ log = set_log(__name__) MAX_PROCESSING_DELAY = 10 +BUCKET_SIZE = {"mainnet": 10000, "xdai": 30000, "arbitrum-one": 1000000} class OrderbookEnv(Enum): @@ -229,9 +230,10 @@ def get_batch_data(cls, block_range: BlockRange) -> DataFrame: so as to ensure the batch data query runs fast enough. At the end, it concatenates everything into one data frame """ + load_dotenv() start = block_range.block_from end = block_range.block_to - bucket_size = 20000 + bucket_size = BUCKET_SIZE[os.environ.get("NETWORK", "mainnet")] res = [] while start < end: size = min(end - start, bucket_size) diff --git a/src/main.py b/src/main.py index b04a92e..7276ec1 100644 --- a/src/main.py +++ b/src/main.py @@ -23,6 +23,7 @@ ) from src.sync.order_rewards import sync_order_rewards, sync_batch_rewards from src.sync.batch_data import sync_batch_data +from src.sync.common import node_suffix log = set_log(__name__) @@ -66,15 +67,9 @@ def main() -> None: orderbook = OrderbookFetcher() network = os.environ.get("NETWORK", "mainnet") log.info(f"Network is set to: {network}") - if network == "mainnet": - node_suffix = "MAINNET" - else: - if network == "xdai": - node_suffix = "GNOSIS" - else: - if network == "arbitrum-one": - node_suffix = "ARBITRUM" - web3 = Web3(Web3.HTTPProvider(os.environ.get("NODE_URL" + "_" + node_suffix))) + web3 = Web3( + Web3.HTTPProvider(os.environ.get("NODE_URL" + "_" + node_suffix(network))) + ) if args.sync_table == SyncTable.APP_DATA: table = os.environ["APP_DATA_TARGET_TABLE"] diff --git a/src/sync/batch_data.py b/src/sync/batch_data.py index c0e6963..385c565 100644 --- a/src/sync/batch_data.py +++ b/src/sync/batch_data.py @@ -1,11 +1,12 @@ """Main Entry point for batch data sync""" import os +from dotenv import load_dotenv from dune_client.client import DuneClient import web3 from src.fetch.orderbook import OrderbookFetcher from src.logger import set_log from src.sync.config import BatchDataSyncConfig -from src.sync.common import compute_block_and_month_range +from src.sync.common import compute_block_and_month_range, node_suffix from src.models.block_range import BlockRange @@ -20,15 +21,9 @@ async def sync_batch_data( dry_run: bool, ) -> None: """Batch data Sync Logic""" + load_dotenv() network = os.environ["NETWORK"] - - if network == "mainnet": - network_name = "ethereum" - else: - if network == "xdai": - network_name = "gnosis" - else: - network_name = "arbitrum" + network_name = node_suffix(network).lower() block_range_list, months_list = compute_block_and_month_range(node) for i in range(len(block_range_list)): diff --git a/src/sync/common.py b/src/sync/common.py index 94da1db..1a17485 100644 --- a/src/sync/common.py +++ b/src/sync/common.py @@ -21,6 +21,18 @@ def last_sync_block(aws: AWSClient, table: SyncTable, genesis_block: int = 0) -> return block_from +def node_suffix(network: str) -> str: + if network == "mainnet": + return "MAINNET" + else: + if network == "xdai": + return "GNOSIS" + else: + if network == "arbitrum-one": + return "ARBITRUM" + return "" + + def find_block_with_timestamp(node, time_stamp) -> int: """ This implements binary search and returns the smallest block number