diff --git a/README.md b/README.md index 4be9c342c..1f127dfdd 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Links: [Homepage](https://diadata.org/) | [Medium Blog](https://medium.com/dia-i DIA is an open-source ecosystem for financial data. The aim of DIA is to make transparent, accurate and verifiable data accessible for Web3, by bringing together data providers, users and DAO community members. -![architecure](https://user-images.githubusercontent.com/103409771/231812128-bb1d2fd0-5946-418b-a82c-641bf586a3d5.png) +![architecture](https://user-images.githubusercontent.com/103409771/231812128-bb1d2fd0-5946-418b-a82c-641bf586a3d5.png) **Data sources:** every second, thousands of trade level data points from multiple on-chain and off-chain sources (CEXs, DEXs, DeFi, NFT, Metaverse, etc) are aggregated into the DIA Platform. diff --git a/cmd/assetCollectionService/go.mod b/cmd/assetCollectionService/go.mod index e16d149dc..66a3dc233 100644 --- a/cmd/assetCollectionService/go.mod +++ b/cmd/assetCollectionService/go.mod @@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/assetCollectionService go 1.20 require ( - github.com/diadata-org/diadata v1.4.533 + github.com/diadata-org/diadata v1.4.539 github.com/sirupsen/logrus v1.9.3 ) diff --git a/cmd/assetCollectionService/main.go b/cmd/assetCollectionService/main.go index 670e54e10..85507de62 100644 --- a/cmd/assetCollectionService/main.go +++ b/cmd/assetCollectionService/main.go @@ -149,7 +149,7 @@ func NewAssetScraper(exchange string, secret string, relDB *models.RelDB) source case dia.OsmosisExchange: return source.NewOsmosisScraper(exchanges[dia.OsmosisExchange]) case dia.VelodromeExchange: - return source.NewVelodromeAssetSource(exchanges[dia.VelodromeExchange]) + return source.NewVelodromeAssetSource(exchanges[dia.VelodromeExchange], relDB) case dia.TraderJoeExchangeV2_1: return source.NewTraderJoeAssetSource(exchanges[dia.TraderJoeExchangeV2_1], relDB) case dia.TraderJoeExchangeV2_1Arbitrum: @@ -170,6 +170,10 @@ func NewAssetScraper(exchange string, secret string, relDB *models.RelDB) source return source.NewJSONReader(exchange, secret) case dia.AyinExchange: return source.NewAyinAssetSource(exchanges[dia.AyinExchange], relDB) + case dia.AerodromeSlipstreamExchange: + return source.NewVelodromeAssetSource(exchanges[dia.AerodromeSlipstreamExchange], relDB) + case dia.AerodromeV1Exchange: + return source.NewVelodromeAssetSource(exchanges[dia.AerodromeV1Exchange], relDB) default: return nil } diff --git a/cmd/exchange-scrapers/collector/go.mod b/cmd/exchange-scrapers/collector/go.mod index 0647803aa..8e2142311 100644 --- a/cmd/exchange-scrapers/collector/go.mod +++ b/cmd/exchange-scrapers/collector/go.mod @@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/exchange-scrapers/collector go 1.19 require ( - github.com/diadata-org/diadata v1.4.529 + github.com/diadata-org/diadata v1.4.541 github.com/segmentio/kafka-go v0.4.35 github.com/sirupsen/logrus v1.9.3 ) diff --git a/cmd/liquidityScraper/go.mod b/cmd/liquidityScraper/go.mod index a575abf5e..c61109e6d 100644 --- a/cmd/liquidityScraper/go.mod +++ b/cmd/liquidityScraper/go.mod @@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/liquidityScraper go 1.20 require ( - github.com/diadata-org/diadata v1.4.522 + github.com/diadata-org/diadata v1.4.539 github.com/sirupsen/logrus v1.9.3 ) diff --git a/cmd/services/tradesBlockService/go.mod b/cmd/services/tradesBlockService/go.mod index 64f81d2d1..c936fdc25 100644 --- a/cmd/services/tradesBlockService/go.mod +++ b/cmd/services/tradesBlockService/go.mod @@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/services/tradesBlockService go 1.19 require ( - github.com/diadata-org/diadata v1.4.527 + github.com/diadata-org/diadata v1.4.540 github.com/segmentio/kafka-go v0.4.35 github.com/sirupsen/logrus v1.8.1 ) diff --git a/config/BitMart.json b/config/BitMart.json index 513c8a948..e1b1e524e 100644 --- a/config/BitMart.json +++ b/config/BitMart.json @@ -12,6 +12,12 @@ "Ignore": false, "Symbol": "ADA" }, + { + "Exchange": "BitMart", + "ForeignName": "AERO_USDT", + "Ignore": false, + "Symbol": "AERO" + }, { "Exchange": "BitMart", "ForeignName": "ALPH_USDT", @@ -72,6 +78,12 @@ "Ignore": false, "Symbol": "BCH" }, + { + "Exchange": "BitMart", + "ForeignName": "BENJI_USDT", + "Ignore": false, + "Symbol": "BENJI" + }, { "Exchange": "BitMart", "ForeignName": "BEPRO_USDT", @@ -86,7 +98,7 @@ }, { "Exchange": "BitMart", - "ForeignName": "BRETT_BTC", + "ForeignName": "BRETT_USDT", "Ignore": false, "Symbol": "BRETT" }, @@ -114,18 +126,6 @@ "Ignore": false, "Symbol": "BTC" }, - { - "Exchange": "BitMart", - "ForeignName": "BUSD_USDT", - "Ignore": false, - "Symbol": "BUSD" - }, - { - "Exchange": "BitMart", - "ForeignName": "BUSD_USDC", - "Ignore": false, - "Symbol": "BUSD" - }, { "Exchange": "BitMart", "ForeignName": "CAKE_USDT", @@ -350,9 +350,9 @@ }, { "Exchange": "BitMart", - "ForeignName": "MATIC_USDT", + "ForeignName": "POL_USDT", "Ignore": false, - "Symbol": "MATIC" + "Symbol": "POL" }, { "Exchange": "BitMart", @@ -366,6 +366,12 @@ "Ignore": false, "Symbol": "METIS" }, + { + "Exchange": "BitMart", + "ForeignName": "MIGGLES_USDT", + "Ignore": false, + "Symbol": "MIGGLES" + }, { "Exchange": "BitMart", "ForeignName": "MKR_BTC", @@ -510,6 +516,12 @@ "Ignore": false, "Symbol": "TON" }, + { + "Exchange": "BitMart", + "ForeignName": "TOSHI_USDT", + "Ignore": false, + "Symbol": "TOSHI" + }, { "Exchange": "BitMart", "ForeignName": "TRUMP_USDT", diff --git a/config/Bitget.json b/config/Bitget.json index 5f0ecd292..60b2e0a4e 100644 --- a/config/Bitget.json +++ b/config/Bitget.json @@ -18,6 +18,12 @@ "Exchange": "Bitget", "Ignore": false }, + { + "Symbol": "AERO", + "ForeignName": "AEROUSDT", + "Exchange": "Bitget", + "Ignore": false + }, { "Symbol": "ARB", "ForeignName": "ARBUSDT", @@ -78,6 +84,12 @@ "Exchange": "Bitget", "Ignore": false }, + { + "Symbol": "BRETT", + "ForeignName": "BRETTUSDT", + "Exchange": "Bitget", + "Ignore": false + }, { "Symbol": "BTC", "ForeignName": "BTCEUR", @@ -210,6 +222,12 @@ "Exchange": "Bitget", "Ignore": false }, + { + "Symbol": "MIGGLES", + "ForeignName": "MIGGLESUSDT", + "Exchange": "Bitget", + "Ignore": false + }, { "Symbol": "MOVR", "ForeignName": "MOVRUSDT", diff --git a/config/Bitmax.json b/config/Bitmax.json index d9d6adc24..6e13c62f3 100644 --- a/config/Bitmax.json +++ b/config/Bitmax.json @@ -6,6 +6,30 @@ "Exchange": "Bitmax", "Ignore": false }, + { + "Symbol": "BRETT", + "ForeignName": "BRETT/USDT", + "Exchange": "Bitmax", + "Ignore": false + }, + { + "Symbol": "DEGEN", + "ForeignName": "DEGEN/USDT", + "Exchange": "Bitmax", + "Ignore": false + }, + { + "Symbol": "MIGGLES", + "ForeignName": "MIGGLES/USDT", + "Exchange": "Bitmax", + "Ignore": false + }, + { + "Symbol": "TOSHI", + "ForeignName": "TOSHI/USDT", + "Exchange": "Bitmax", + "Ignore": false + }, { "Symbol": "ETHFI", "ForeignName": "ETHFI/USDT", diff --git a/config/ByBit.json b/config/ByBit.json index 79ff30f0d..69478ac08 100644 --- a/config/ByBit.json +++ b/config/ByBit.json @@ -126,6 +126,12 @@ "Exchange": "ByBit", "Ignore": false }, + { + "Symbol": "BRETT", + "ForeignName": "BRETTUSDC", + "Exchange": "ByBit", + "Ignore": false + }, { "Symbol": "BSV", "ForeignName": "BSVUSDT", @@ -222,6 +228,12 @@ "Exchange": "ByBit", "Ignore": false }, + { + "Symbol": "DEGEN", + "ForeignName": "DEGENUSDT", + "Exchange": "ByBit", + "Ignore": false + }, { "Symbol": "DOGE", "ForeignName": "DOGEUSDT", diff --git a/config/CoinBase.json b/config/CoinBase.json index e6aff4680..5b182aa39 100644 --- a/config/CoinBase.json +++ b/config/CoinBase.json @@ -6,6 +6,12 @@ "Ignore": false, "Symbol": "AAVE" }, + { + "Exchange": "CoinBase", + "ForeignName": "AERO-USD", + "Ignore": false, + "Symbol": "AERO" + }, { "Exchange": "CoinBase", "ForeignName": "AKT-USD", diff --git a/config/Crypto.com.json b/config/Crypto.com.json index 2302eb00d..303e9cb60 100644 --- a/config/Crypto.com.json +++ b/config/Crypto.com.json @@ -42,6 +42,12 @@ "Exchange": "Crypto.com", "Ignore": false }, + { + "Symbol": "AERO", + "ForeignName": "AERO_USDT", + "Exchange": "Crypto.com", + "Ignore": false + }, { "Symbol": "ADA", "ForeignName": "ADA_USDT", diff --git a/config/GateIO.json b/config/GateIO.json index 6955691db..2c0cc62f6 100644 --- a/config/GateIO.json +++ b/config/GateIO.json @@ -54,6 +54,12 @@ "Ignore": false, "Symbol": "AE" }, + { + "Exchange": "GateIO", + "ForeignName": "AERO_USDT", + "Ignore": false, + "Symbol": "AERO" + }, { "Exchange": "GateIO", "ForeignName": "AKRO_USDT", @@ -222,6 +228,12 @@ "Ignore": false, "Symbol": "BCX" }, + { + "Exchange": "GateIO", + "ForeignName": "BENJI_USDT", + "Ignore": false, + "Symbol": "BENJI" + }, { "Exchange": "GateIO", "ForeignName": "BEPRO_USDT", @@ -600,6 +612,12 @@ "Ignore": false, "Symbol": "DDD" }, + { + "Exchange": "GateIO", + "ForeignName": "DEGEN_USDT", + "Ignore": false, + "Symbol": "DEGEN" + }, { "Exchange": "GateIO", "ForeignName": "DFYN_USDT", @@ -1530,6 +1548,12 @@ "Ignore": false, "Symbol": "METIS" }, + { + "Exchange": "GateIO", + "ForeignName": "MIGGLES_USDT", + "Ignore": false, + "Symbol": "MIGGLES" + }, { "Exchange": "GateIO", "ForeignName": "IOTA_BTC", @@ -2430,6 +2454,12 @@ "Ignore": false, "Symbol": "TON" }, + { + "Exchange": "GateIO", + "ForeignName": "TOSHI_USDT", + "Ignore": false, + "Symbol": "TOSHI" + }, { "Exchange": "GateIO", "ForeignName": "TRX_ETH", diff --git a/config/Huobi.json b/config/Huobi.json index 759381713..ea8f6be3e 100644 --- a/config/Huobi.json +++ b/config/Huobi.json @@ -174,6 +174,12 @@ "Ignore": false, "Symbol": "DCR" }, + { + "Exchange": "Huobi", + "ForeignName": "DEGENUSDT", + "Ignore": false, + "Symbol": "DEGEN" + }, { "Exchange": "Huobi", "ForeignName": "DGBBTC", diff --git a/config/KuCoin.json b/config/KuCoin.json index 294a45ceb..31a32bcc0 100644 --- a/config/KuCoin.json +++ b/config/KuCoin.json @@ -60,6 +60,12 @@ "Exchange": "KuCoin", "Ignore": false }, + { + "Symbol": "AERO", + "ForeignName": "AERO-USDT", + "Exchange": "KuCoin", + "Ignore": false + }, { "Symbol": "AKRO", "ForeignName": "AKRO-BTC", @@ -516,6 +522,12 @@ "Exchange": "KuCoin", "Ignore": false }, + { + "Symbol": "DEGEN", + "ForeignName": "DEGEN-USDT", + "Exchange": "KuCoin", + "Ignore": false + }, { "Symbol": "DENT", "ForeignName": "DENT-ETH", diff --git a/config/MEXC.json b/config/MEXC.json index e3237f387..c0453e4ae 100644 --- a/config/MEXC.json +++ b/config/MEXC.json @@ -168,6 +168,12 @@ "Exchange": "MEXC", "Ignore": false }, + { + "Symbol": "DEGEN", + "ForeignName": "DEGENUSDT", + "Exchange": "MEXC", + "Ignore": false + }, { "Symbol": "DAI", "ForeignName": "DAIUSDT", @@ -438,6 +444,12 @@ "Exchange": "MEXC", "Ignore": false }, + { + "Symbol": "MIGGLES", + "ForeignName": "MIGGLESUSDT", + "Exchange": "MEXC", + "Ignore": false + }, { "Symbol": "METIS", "ForeignName": "METISUSDT", diff --git a/config/OKEx.json b/config/OKEx.json index 91445d83b..afe9b3c00 100644 --- a/config/OKEx.json +++ b/config/OKEx.json @@ -774,6 +774,12 @@ "Ignore": false, "Symbol": "DCR" }, + { + "Exchange": "OKEx", + "ForeignName": "DEGEN-USDT", + "Ignore": false, + "Symbol": "DEGEN" + }, { "Exchange": "OKEx", "ForeignName": "DENT-BTC", diff --git a/config/gitcoinverified/BitMart.json b/config/gitcoinverified/BitMart.json index 8b5f09ff4..a2d07d818 100644 --- a/config/gitcoinverified/BitMart.json +++ b/config/gitcoinverified/BitMart.json @@ -12,6 +12,12 @@ "Blockchain": "Cardano", "Address": "0x0000000000000000000000000000000000000000" }, + { + "Symbol": "AERO", + "Exchange": "BitMart", + "Blockchain": "Base", + "Address": "0x940181a94A35A4569E4529A3CDfB74e38FD98631" + }, { "Symbol": "ALPH", "Exchange": "BitMart", @@ -72,6 +78,12 @@ "Blockchain": "BitcoinCash", "Address": "0x0000000000000000000000000000000000000000" }, + { + "Symbol": "BENJI", + "Exchange": "BitMart", + "Blockchain": "Base", + "Address": "0xBC45647eA894030a4E9801Ec03479739FA2485F0" + }, { "Symbol": "BEPRO", "Exchange": "BitMart", @@ -318,6 +330,12 @@ "Blockchain": "Metis", "Address": "0xDeadDeAddeAddEAddeadDEaDDEAdDeaDDeAD0000" }, + { + "Symbol": "MIGGLES", + "Exchange": "BitMart", + "Blockchain": "Base", + "Address": "0xB1a03EdA10342529bBF8EB700a06C60441fEf25d" + }, { "Symbol": "MOG", "Exchange": "BitMart", @@ -456,6 +474,12 @@ "Blockchain": "Ton", "Address": "0x0000000000000000000000000000000000000000" }, + { + "Symbol": "TOSHI", + "Exchange": "BitMart", + "Blockchain": "Base", + "Address": "0xAC1Bd2486aAf3B5C0fc3Fd868558b082a531B2B4" + }, { "Symbol": "TRUMP", "Exchange": "BitMart", diff --git a/config/gitcoinverified/Bitget.json b/config/gitcoinverified/Bitget.json index f49ad919a..aef8a6231 100644 --- a/config/gitcoinverified/Bitget.json +++ b/config/gitcoinverified/Bitget.json @@ -12,6 +12,12 @@ "Blockchain": "Cardano", "Address": "0x0000000000000000000000000000000000000000" }, + { + "Symbol": "AERO", + "Exchange": "Bitget", + "Blockchain": "Base", + "Address": "0x940181a94A35A4569E4529A3CDfB74e38FD98631" + }, { "Symbol": "ALPH", "Exchange": "Bitget", @@ -54,6 +60,12 @@ "Blockchain": "BinanceSmartChain", "Address": "0x0000000000000000000000000000000000000000" }, + { + "Symbol": "BRETT", + "Exchange": "Bitget", + "Blockchain": "Base", + "Address": "0x532f27101965dd16442E59d40670FaF5eBB142E4" + }, { "Symbol": "BTC", "Exchange": "Bitget", @@ -174,6 +186,12 @@ "Blockchain": "Lukso", "Address": "0x0000000000000000000000000000000000000000" }, + { + "Symbol": "MIGGLES", + "Exchange": "Bitget", + "Blockchain": "Base", + "Address": "0xB1a03EdA10342529bBF8EB700a06C60441fEf25d" + }, { "Symbol": "MOVR", "Exchange": "Bitget", diff --git a/config/gitcoinverified/Bitmax.json b/config/gitcoinverified/Bitmax.json index 14ac5b9ac..d99ecb503 100644 --- a/config/gitcoinverified/Bitmax.json +++ b/config/gitcoinverified/Bitmax.json @@ -72,6 +72,12 @@ "Blockchain": "BinanceSmartChain", "Address": "0x0000000000000000000000000000000000000000" }, + { + "Symbol": "BRETT", + "Exchange": "Bitmax", + "Blockchain": "Base", + "Address": "0x532f27101965dd16442E59d40670FaF5eBB142E4" + }, { "Symbol": "BSV", "Exchange": "Bitmax", @@ -132,6 +138,12 @@ "Blockchain": "Dash", "Address": "0x0000000000000000000000000000000000000000" }, + { + "Symbol": "DEGEN", + "Exchange": "Bitmax", + "Blockchain": "Base", + "Address": "0x4ed4E862860beD51a9570b96d89aF5E1B0Efefed" + }, { "Symbol": "DIA", "Exchange": "Bitmax", @@ -282,6 +294,12 @@ "Blockchain": "Polygon", "Address": "0x0000000000000000000000000000000000001010" }, + { + "Symbol": "MIGGLES", + "Exchange": "Bitmax", + "Blockchain": "Base", + "Address": "0xB1a03EdA10342529bBF8EB700a06C60441fEf25d" + }, { "Symbol": "NEO", "Exchange": "Bitmax", @@ -378,6 +396,12 @@ "Blockchain": "Osmosis", "Address": "ibc/D79E7D83AB399BFFF93433E54FAA480C191248FC556924A2A8351AE2638B3877" }, + { + "Symbol": "TOSHI", + "Exchange": "Bitmax", + "Blockchain": "Base", + "Address": "0xAC1Bd2486aAf3B5C0fc3Fd868558b082a531B2B4" + }, { "Symbol": "TRX", "Exchange": "Bitmax", diff --git a/config/gitcoinverified/ByBit.json b/config/gitcoinverified/ByBit.json index a139611a9..c23a571b1 100644 --- a/config/gitcoinverified/ByBit.json +++ b/config/gitcoinverified/ByBit.json @@ -60,6 +60,12 @@ "Blockchain": "Base", "Address": "0x532f27101965dd16442E59d40670FaF5eBB142E4" }, + { + "Symbol": "DEGEN", + "Exchange": "ByBit", + "Blockchain": "Base", + "Address": "0x4ed4E862860beD51a9570b96d89aF5E1B0Efefed" + }, { "Symbol": "WLD", "Exchange": "ByBit", diff --git a/config/gitcoinverified/CoinBase.json b/config/gitcoinverified/CoinBase.json index 793f808bd..8ee0bb1ff 100644 --- a/config/gitcoinverified/CoinBase.json +++ b/config/gitcoinverified/CoinBase.json @@ -288,6 +288,12 @@ "Blockchain": "BinanceSmartChain", "Address": "0xe9e7CEA3DedcA5984780Bafc599bD69ADd087D56" }, + { + "Symbol": "AERO", + "Exchange": "CoinBase", + "Blockchain": "Base", + "Address": "0x940181a94A35A4569E4529A3CDfB74e38FD98631" + }, { "Symbol": "PNG", "Exchange": "CoinBase", diff --git a/config/gitcoinverified/Crypto.com.json b/config/gitcoinverified/Crypto.com.json index c44bfe89e..35abe7a1b 100644 --- a/config/gitcoinverified/Crypto.com.json +++ b/config/gitcoinverified/Crypto.com.json @@ -72,6 +72,12 @@ "Blockchain": "Cardano", "Address": "0x0000000000000000000000000000000000000000" }, + { + "Symbol": "AERO", + "Exchange": "Crypto.com", + "Blockchain": "Base", + "Address": "0x940181a94A35A4569E4529A3CDfB74e38FD98631" + }, { "Symbol": "BCH", "Exchange": "Crypto.com", diff --git a/config/gitcoinverified/GateIO.json b/config/gitcoinverified/GateIO.json index 91fb4926b..d26cd97df 100755 --- a/config/gitcoinverified/GateIO.json +++ b/config/gitcoinverified/GateIO.json @@ -6,6 +6,12 @@ "Blockchain": "Acala", "Address": "Token:ACA" }, + { + "Symbol": "AERO", + "Exchange": "GateIO", + "Blockchain": "Base", + "Address": "0x940181a94A35A4569E4529A3CDfB74e38FD98631" + }, { "Symbol": "ANGLE", "Exchange": "GateIO", @@ -54,6 +60,12 @@ "Blockchain": "AlephZero", "Address": "0x0000000000000000000000000000000000000000" }, + { + "Symbol": "BENJI", + "Exchange": "GateIO", + "Blockchain": "Base", + "Address": "0xBC45647eA894030a4E9801Ec03479739FA2485F0" + }, { "Symbol": "BEPRO", "Exchange": "GateIO", @@ -384,6 +396,12 @@ "Blockchain": "Ethereum", "Address": "0x9F5F3CFD7a32700C93F971637407ff17b91c7342" }, + { + "Symbol": "DEGEN", + "Exchange": "GateIO", + "Blockchain": "Base", + "Address": "0x4ed4E862860beD51a9570b96d89aF5E1B0Efefed" + }, { "Symbol": "DGD", "Exchange": "GateIO", @@ -666,6 +684,12 @@ "Blockchain": "Metis", "Address": "0xDeadDeAddeAddEAddeadDEaDDEAdDeaDDeAD0000" }, + { + "Symbol": "MIGGLES", + "Exchange": "GateIO", + "Blockchain": "Base", + "Address": "0xB1a03EdA10342529bBF8EB700a06C60441fEf25d" + }, { "Symbol": "MCO", "Exchange": "GateIO", @@ -1014,6 +1038,12 @@ "Blockchain": "Ton", "Address": "0x0000000000000000000000000000000000000000" }, + { + "Symbol": "TOSHI", + "Exchange": "GateIO", + "Blockchain": "Base", + "Address": "0xAC1Bd2486aAf3B5C0fc3Fd868558b082a531B2B4" + }, { "Symbol": "UNI", "Exchange": "GateIO", diff --git a/config/gitcoinverified/Huobi.json b/config/gitcoinverified/Huobi.json index 9b583d98e..ab3287bf3 100644 --- a/config/gitcoinverified/Huobi.json +++ b/config/gitcoinverified/Huobi.json @@ -150,6 +150,12 @@ "Blockchain": "Dash", "Address": "0x0000000000000000000000000000000000000000" }, + { + "Symbol": "DEGEN", + "Exchange": "Huobi", + "Blockchain": "Base", + "Address": "0x4ed4E862860beD51a9570b96d89aF5E1B0Efefed" + }, { "Symbol": "DOGE", "Exchange": "Huobi", diff --git a/config/gitcoinverified/KuCoin.json b/config/gitcoinverified/KuCoin.json index 9eddcbd55..5cceb8346 100755 --- a/config/gitcoinverified/KuCoin.json +++ b/config/gitcoinverified/KuCoin.json @@ -18,6 +18,12 @@ "Blockchain": "Ethereum", "Address": "0x91Af0fBB28ABA7E31403Cb457106Ce79397FD4E6" }, + { + "Symbol": "AERO", + "Exchange": "KuCoin", + "Blockchain": "Base", + "Address": "0x940181a94A35A4569E4529A3CDfB74e38FD98631" + }, { "Symbol": "AKRO", "Exchange": "KuCoin", @@ -228,6 +234,12 @@ "Blockchain": "Ethereum", "Address": "0xaBbBB6447B68ffD6141DA77C18c7B5876eD6c5ab" }, + { + "Symbol": "DEGEN", + "Exchange": "KuCoin", + "Blockchain": "Base", + "Address": "0x4ed4E862860beD51a9570b96d89aF5E1B0Efefed" + }, { "Symbol": "DEGO", "Exchange": "KuCoin", diff --git a/config/gitcoinverified/MEXC.json b/config/gitcoinverified/MEXC.json index cdfbd89fa..275bd8e3e 100644 --- a/config/gitcoinverified/MEXC.json +++ b/config/gitcoinverified/MEXC.json @@ -168,6 +168,12 @@ "Blockchain": "Ethereum", "Address": "0x6B175474E89094C44Da98b954EedeAC495271d0F" }, + { + "Symbol": "DEGEN", + "Exchange": "MEXC", + "Blockchain": "Base", + "Address": "0x4ed4E862860beD51a9570b96d89aF5E1B0Efefed" + }, { "Symbol": "DFYN", "Exchange": "MEXC", @@ -408,6 +414,12 @@ "Blockchain": "Metis", "Address": "0xDeadDeAddeAddEAddeadDEaDDEAdDeaDDeAD0000" }, + { + "Symbol": "MIGGLES", + "Exchange": "MEXC", + "Blockchain": "Base", + "Address": "0xB1a03EdA10342529bBF8EB700a06C60441fEf25d" + }, { "Symbol": "MOG", "Exchange": "MEXC", diff --git a/config/gitcoinverified/OKEx.json b/config/gitcoinverified/OKEx.json index e894f4ab2..830501fe2 100755 --- a/config/gitcoinverified/OKEx.json +++ b/config/gitcoinverified/OKEx.json @@ -96,6 +96,12 @@ "Blockchain": "Ethereum", "Address": "0xBe428c3867F05deA2A89Fc76a102b544eaC7f772" }, + { + "Symbol": "DEGEN", + "Exchange": "OKEx", + "Blockchain": "Base", + "Address": "0x4ed4E862860beD51a9570b96d89aF5E1B0Efefed" + }, { "Symbol": "DIA", "Exchange": "OKEx", diff --git a/config/liquidity-scrapers/uniswapv2/UniswapBase.json b/config/liquidity-scrapers/uniswapv2/UniswapBase.json new file mode 100644 index 000000000..6e4f438d1 --- /dev/null +++ b/config/liquidity-scrapers/uniswapv2/UniswapBase.json @@ -0,0 +1,12 @@ +{ + "Pools": [ + { + "Address": "0x16905890A1D02b6F824387419319Bf4188B961b0", + "ForeignName": "WETH-BENJI" + }, + { + "Address": "0x17A3Ad8c74c4947005aFEDa9965305ae2EB2518a", + "ForeignName": "WETH-MIGGLES" + } + ] +} \ No newline at end of file diff --git a/config/uniswap/reverse_tokens/Uniswap-BaseQuotetoken.json b/config/uniswap/reverse_tokens/Uniswap-BaseQuotetoken.json new file mode 100644 index 000000000..6eeaf35e9 --- /dev/null +++ b/config/uniswap/reverse_tokens/Uniswap-BaseQuotetoken.json @@ -0,0 +1,8 @@ +{ + "Tokens": [ + { + "Address": "0x4200000000000000000000000000000000000006", + "Symbol": "WETH" + } + ] +} \ No newline at end of file diff --git a/config/uniswap/subscribe_pools/Uniswap-Base.json b/config/uniswap/subscribe_pools/Uniswap-Base.json new file mode 100644 index 000000000..6e4f438d1 --- /dev/null +++ b/config/uniswap/subscribe_pools/Uniswap-Base.json @@ -0,0 +1,12 @@ +{ + "Pools": [ + { + "Address": "0x16905890A1D02b6F824387419319Bf4188B961b0", + "ForeignName": "WETH-BENJI" + }, + { + "Address": "0x17A3Ad8c74c4947005aFEDa9965305ae2EB2518a", + "ForeignName": "WETH-MIGGLES" + } + ] +} \ No newline at end of file diff --git a/config/uniswapv3/fullPools/AerodromeSlipstreamFullPools.json b/config/uniswapv3/fullPools/AerodromeSlipstreamFullPools.json new file mode 100644 index 000000000..694ced41d --- /dev/null +++ b/config/uniswapv3/fullPools/AerodromeSlipstreamFullPools.json @@ -0,0 +1,12 @@ +{ + "Tokens": [ + { + "Address": "0xd0b53D9277642d899DF5C87A3966A349A798F224", + "ForeignName": "USDC-WETH" + }, + { + "Address": "0xb2cc224c1c9feE385f8ad6a55b4d94E92359DC59", + "ForeignName": "USDC-WETH" + } + ] +} \ No newline at end of file diff --git a/config/uniswapv3/reverse_tokens/AerodromeSlipstreamQuotetoken.json b/config/uniswapv3/reverse_tokens/AerodromeSlipstreamQuotetoken.json new file mode 100644 index 000000000..6eeaf35e9 --- /dev/null +++ b/config/uniswapv3/reverse_tokens/AerodromeSlipstreamQuotetoken.json @@ -0,0 +1,8 @@ +{ + "Tokens": [ + { + "Address": "0x4200000000000000000000000000000000000006", + "Symbol": "WETH" + } + ] +} \ No newline at end of file diff --git a/config/uniswapv3/reverse_tokens/UniswapV3-BaseQuotetoken.json b/config/uniswapv3/reverse_tokens/UniswapV3-BaseQuotetoken.json new file mode 100644 index 000000000..6eeaf35e9 --- /dev/null +++ b/config/uniswapv3/reverse_tokens/UniswapV3-BaseQuotetoken.json @@ -0,0 +1,8 @@ +{ + "Tokens": [ + { + "Address": "0x4200000000000000000000000000000000000006", + "Symbol": "WETH" + } + ] +} \ No newline at end of file diff --git a/config/uniswapv3/subscribe_pools/AerodromeSlipstream.json b/config/uniswapv3/subscribe_pools/AerodromeSlipstream.json new file mode 100644 index 000000000..5f71dab69 --- /dev/null +++ b/config/uniswapv3/subscribe_pools/AerodromeSlipstream.json @@ -0,0 +1,37 @@ +{ + "Pools": [ + { + "Address": "0xd0b53D9277642d899DF5C87A3966A349A798F224", + "ForeignName": "USDC-WETH" + }, + { + "Address": "0xb2cc224c1c9feE385f8ad6a55b4d94E92359DC59", + "ForeignName": "USDC-WETH" + }, + { + "Address": "0x6446021F4E396dA3df4235C62537431372195D38", + "ForeignName": "OETHb-WETH" + }, + { + "Address": "0xcf88B8bf7CCCe2D836878E538197eB20fC673BCE", + "ForeignName": "USDz-WETH" + }, + { + "Address": "0xdE5Ff829fEF54d1BdEc957D9538A306f0EAD1368", + "ForeignName": "USDz-USDC" + }, + { + "Address": "0x4e829F8A5213c42535AB84AA40BD4aDCCE9cBa02", + "ForeignName": "BRETT-WETH" + }, + { + "Address": "0x82321f3BEB69f503380D6B233857d5C43562e2D0", + "ForeignName": "AERO-WETH" + }, + { + "Address": "0xaFB62448929664Bfccb0aAe22f232520e765bA88", + "ForeignName": "DEGEN-WETH" + } + + ] +} \ No newline at end of file diff --git a/config/uniswapv3/subscribe_pools/UniswapV3-Base.json b/config/uniswapv3/subscribe_pools/UniswapV3-Base.json new file mode 100644 index 000000000..c21f5884e --- /dev/null +++ b/config/uniswapv3/subscribe_pools/UniswapV3-Base.json @@ -0,0 +1,21 @@ +{ + "Pools": [ + { + "Address": "0x4b0Aaf3EBb163dd45F663b38b6d93f6093EBC2d3", + "ForeignName": "WETH-TOSHI" + }, + { + "Address": "0xadAd4ce0C68F50A19CF5063E0B91d701Daab1df1", + "ForeignName": "WETH-MIGGLES" + }, + { + "Address": "0x3d5D143381916280ff91407FeBEB52f2b60f33Cf", + "ForeignName": "WETH-AERO" + }, + { + "Address": "0xc9034c3E7F58003E6ae0C8438e7c8f4598d5ACAA", + "ForeignName": "WETH-DEGEN" + } + + ] +} \ No newline at end of file diff --git a/config/velodrome/fullPools/AerodromeV1FullPools.json b/config/velodrome/fullPools/AerodromeV1FullPools.json new file mode 100644 index 000000000..a1ba3963a --- /dev/null +++ b/config/velodrome/fullPools/AerodromeV1FullPools.json @@ -0,0 +1,16 @@ +{ + "Tokens": [ + { + "Address": "0x6cDcb1C4A4D1C3C6d054b27AC5B77e89eAFb971d", + "ForeignName": "USDC-AERO" + }, + { + "Address": "0x7818B53F423457151407103321d8F961773815D1", + "ForeignName": "USDz-sUSDz" + }, + { + "Address": "0x6d0b9C9E92a3De30081563c3657B5258b3fFa38B", + "ForeignName": "USDz-USDC" + } + ] +} \ No newline at end of file diff --git a/config/velodrome/reverse_tokens/AerodromeV1Quotetoken.json b/config/velodrome/reverse_tokens/AerodromeV1Quotetoken.json new file mode 100644 index 000000000..6eeaf35e9 --- /dev/null +++ b/config/velodrome/reverse_tokens/AerodromeV1Quotetoken.json @@ -0,0 +1,8 @@ +{ + "Tokens": [ + { + "Address": "0x4200000000000000000000000000000000000006", + "Symbol": "WETH" + } + ] +} \ No newline at end of file diff --git a/config/velodrome/subscribe_pools/AerodromeV1.json b/config/velodrome/subscribe_pools/AerodromeV1.json new file mode 100644 index 000000000..22e441689 --- /dev/null +++ b/config/velodrome/subscribe_pools/AerodromeV1.json @@ -0,0 +1,28 @@ +{ + "Pools": [ + { + "Address": "0x2C4909355b0C036840819484c3A882A95659aBf3", + "ForeignName": "WETH-DEGEN" + }, + { + "Address": "0x7f670f78B17dEC44d5Ef68a48740b6f8849cc2e6", + "ForeignName": "WETH-AERO" + }, + { + "Address": "0x6cDcb1C4A4D1C3C6d054b27AC5B77e89eAFb971d", + "ForeignName": "USDC-AERO" + }, + { + "Address": "0x2ce63497999F520CC2afaaadbCFC37Afd9deF4b0", + "ForeignName": "USDz-WETH" + }, + { + "Address": "0x7818B53F423457151407103321d8F961773815D1", + "ForeignName": "USDz-sUSDz" + }, + { + "Address": "0x6d0b9C9E92a3De30081563c3657B5258b3fFa38B", + "ForeignName": "USDz-USDC" + } + ] +} \ No newline at end of file diff --git a/internal/pkg/tradesBlockService/tradesBlockService.go b/internal/pkg/tradesBlockService/tradesBlockService.go index 41390f22b..76b460794 100644 --- a/internal/pkg/tradesBlockService/tradesBlockService.go +++ b/internal/pkg/tradesBlockService/tradesBlockService.go @@ -683,6 +683,18 @@ func buildBridge(t dia.Trade) dia.Asset { } } } + if basetoken.Blockchain == dia.BASE && (t.Source == dia.UniswapExchangeV3Base || + t.Source == dia.UniswapExchangeBase || + t.Source == dia.AerodromeSlipstreamExchange || + t.Source == dia.AerodromeV1Exchange) { + if basetoken.Address == "0x4200000000000000000000000000000000000006" { + basetoken = dia.Asset{ + Symbol: "WETH", + Address: "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + Blockchain: dia.ETHEREUM, + } + } + } return basetoken } diff --git a/pkg/dia/Config.go b/pkg/dia/Config.go index c4b949fdc..dff97dad9 100644 --- a/pkg/dia/Config.go +++ b/pkg/dia/Config.go @@ -122,6 +122,8 @@ const ( ThenaV3Exchange = "ThenaV3" AyinExchange = "Ayin" BitgetExchange = "Bitget" + AerodromeSlipstreamExchange = "AerodromeSlipstream" + AerodromeV1Exchange = "AerodromeV1" // FinageForex = "FinageForex" ) diff --git a/pkg/dia/Messages.go b/pkg/dia/Messages.go index 40251978d..983d79da1 100644 --- a/pkg/dia/Messages.go +++ b/pkg/dia/Messages.go @@ -52,6 +52,7 @@ const ( LINEA = "Linea" OPTIMISM = "Optimism" ALEPHIUM = "Alephium" + BASE = "Base" ) var CRYPTO_ZERO_UNIX_TIME = time.Unix(1230768000, 0) diff --git a/pkg/dia/scraper/exchange-scrapers/APIScraper.go b/pkg/dia/scraper/exchange-scrapers/APIScraper.go index b4ab26f90..73ac52141 100644 --- a/pkg/dia/scraper/exchange-scrapers/APIScraper.go +++ b/pkg/dia/scraper/exchange-scrapers/APIScraper.go @@ -141,6 +141,8 @@ func NewAPIScraper(exchange string, scrape bool, key string, secret string, relD return NewBancorScraper(Exchanges[dia.BancorExchange], scrape) case dia.UniswapExchange: return NewUniswapScraper(Exchanges[dia.UniswapExchange], scrape, relDB) + case dia.UniswapExchangeBase: + return NewUniswapScraper(Exchanges[dia.UniswapExchangeBase], scrape, relDB) case dia.PanCakeSwap: return NewUniswapScraper(Exchanges[dia.PanCakeSwap], scrape, relDB) case dia.PanCakeSwapExchangeV3: @@ -189,6 +191,8 @@ func NewAPIScraper(exchange string, scrape bool, key string, secret string, relD return NewBKEXScraper(Exchanges[dia.BKEXExchange], dia.BKEX2Exchange, scrape, relDB) case dia.UniswapExchangeV3: return NewUniswapV3Scraper(Exchanges[dia.UniswapExchangeV3], scrape, relDB) + case dia.UniswapExchangeV3Base: + return NewUniswapV3Scraper(Exchanges[dia.UniswapExchangeV3Base], scrape, relDB) case dia.DfynNetwork: return NewUniswapScraper(Exchanges[dia.DfynNetwork], scrape, relDB) case dia.UbeswapExchange: @@ -269,6 +273,8 @@ func NewAPIScraper(exchange string, scrape bool, key string, secret string, relD return NewZenlinkScraper(Exchanges[dia.ZenlinkswapExchangeBifrostPolkadot], scrape) case dia.VelodromeExchange: return NewVelodromeScraper(Exchanges[dia.VelodromeExchange], scrape, relDB) + case dia.AerodromeV1Exchange: + return NewVelodromeScraper(Exchanges[dia.AerodromeV1Exchange], scrape, relDB) case dia.MaverickExchange: return NewMaverickScraper(Exchanges[dia.MaverickExchange], scrape, relDB) case dia.PearlfiExchangeTestnet: @@ -283,6 +289,10 @@ func NewAPIScraper(exchange string, scrape bool, key string, secret string, relD // return NewFinageForexScraper(Exchanges[dia.FinageForex], scrape, relDB, key, secret) case dia.BitgetExchange: return NewBitgetScraper(Exchanges[dia.BitgetExchange], scrape, relDB) + case dia.AerodromeSlipstreamExchange: + return NewUniswapV3Scraper(Exchanges[dia.AerodromeSlipstreamExchange], scrape, relDB) + // case dia.FinageForex: + // return NewFinageForexScraper(Exchanges[dia.FinageForex], scrape, relDB, key, secret) case dia.MultiChain: return NewBridgeSwapScraper(Exchanges[dia.MultiChain], scrape, relDB) diff --git a/pkg/dia/scraper/exchange-scrapers/UniswapV2Scraper.go b/pkg/dia/scraper/exchange-scrapers/UniswapV2Scraper.go index d20ffa5fc..bdba9dc97 100644 --- a/pkg/dia/scraper/exchange-scrapers/UniswapV2Scraper.go +++ b/pkg/dia/scraper/exchange-scrapers/UniswapV2Scraper.go @@ -38,6 +38,9 @@ const ( restDialEth = "" wsDialEth = "" + restDialBase = "" + wsDialBase = "" + restDialBSC = "" wsDialBSC = "" @@ -174,6 +177,8 @@ func NewUniswapScraper(exchange dia.Exchange, scrape bool, relDB *models.RelDB) switch exchange.Name { case dia.UniswapExchange: s = makeUniswapScraper(exchange, listenByAddress, fetchPoolsFromDB, restDialEth, wsDialEth, uniswapWaitMilliseconds) + case dia.UniswapExchangeBase: + s = makeUniswapScraper(exchange, listenByAddress, fetchPoolsFromDB, restDialBase, wsDialBase, uniswapWaitMilliseconds) case dia.SushiSwapExchange: s = makeUniswapScraper(exchange, listenByAddress, fetchPoolsFromDB, restDialEth, wsDialEth, sushiswapWaitMilliseconds) case dia.SushiSwapExchangePolygon: diff --git a/pkg/dia/scraper/exchange-scrapers/UniswapV3Scraper.go b/pkg/dia/scraper/exchange-scrapers/UniswapV3Scraper.go index 445934e3f..941bf29ee 100644 --- a/pkg/dia/scraper/exchange-scrapers/UniswapV3Scraper.go +++ b/pkg/dia/scraper/exchange-scrapers/UniswapV3Scraper.go @@ -86,6 +86,8 @@ func NewUniswapV3Scraper(exchange dia.Exchange, scrape bool, relDB *models.RelDB s = makeUniswapV3Scraper(exchange, listenByAddress, "", "", "200", uint64(22757913)) case dia.UniswapExchangeV3Arbitrum: s = makeUniswapV3Scraper(exchange, listenByAddress, "", "", "200", uint64(165)) + case dia.UniswapExchangeV3Base: + s = makeUniswapV3Scraper(exchange, listenByAddress, "", "", "200", uint64(1371680)) case dia.PanCakeSwapExchangeV3: s = makeUniswapV3Scraper(exchange, listenByAddress, "", "", "200", uint64(26956207)) case dia.CamelotExchangeV3: @@ -101,6 +103,8 @@ func NewUniswapV3Scraper(exchange dia.Exchange, scrape bool, relDB *models.RelDB s = makeUniswapV3Scraper(exchange, listenByAddress, "", "", "200", uint64(90593047)) case dia.NileV2Exchange: s = makeUniswapV3Scraper(exchange, listenByAddress, "", "", "2000", uint64(1768866)) + case dia.AerodromeSlipstreamExchange: + s = makeUniswapV3Scraper(exchange, listenByAddress, "", "", "2000", uint64(0)) } s.relDB = relDB diff --git a/pkg/dia/scraper/exchange-scrapers/VelodromeScraper.go b/pkg/dia/scraper/exchange-scrapers/VelodromeScraper.go index 04900f88c..b74e27e4f 100644 --- a/pkg/dia/scraper/exchange-scrapers/VelodromeScraper.go +++ b/pkg/dia/scraper/exchange-scrapers/VelodromeScraper.go @@ -21,6 +21,8 @@ import ( const ( velodromeRestDial = "" velodromeWsDial = "" + baseRestDial = "" + baseWsDial = "" ) type VelodromeSwap struct { @@ -40,10 +42,14 @@ type VelodromeScraper struct { relDB *models.RelDB // error handling; to read error or closed, first acquire read lock // only cleanup method should hold write lock - errorLock sync.RWMutex - error error - closed bool - pools []dia.Pool + errorLock sync.RWMutex + error error + closed bool + pools []dia.Pool + listenByAddress bool + reverseQuotetokens *[]string + reverseBasetokens *[]string + fullPools *[]string // used to keep track of trading pairs that we subscribed to pairScrapers map[string]*VelodromePairScraper exchangeName string @@ -52,11 +58,17 @@ type VelodromeScraper struct { func NewVelodromeScraper(exchange dia.Exchange, scrape bool, relDB *models.RelDB) *VelodromeScraper { log.Info("NewVelodromeScraper: ", exchange.Name) - var s *VelodromeScraper + var ( + s *VelodromeScraper + listenByAddress bool + err error + ) switch exchange.Name { case dia.VelodromeExchange: s = makeVelodromeScraper(exchange, velodromeRestDial, velodromeWsDial, relDB) + case dia.AerodromeV1Exchange: + s = makeVelodromeScraper(exchange, baseRestDial, baseWsDial, relDB) } // Only include pools with (minimum) liquidity bigger than given env var. @@ -73,6 +85,30 @@ func NewVelodromeScraper(exchange dia.Exchange, scrape bool, relDB *models.RelDB log.Warnf("parse liquidity threshold: %v. Set to default %v", err, liquidityThresholdUSD) } + listenByAddress, err = strconv.ParseBool(utils.Getenv("LISTEN_BY_ADDRESS", "")) + if err != nil { + log.Fatal("parse LISTEN_BY_ADDRESS: ", err) + } + s.listenByAddress = listenByAddress + + s.reverseBasetokens, err = getReverseTokensFromConfig("velodrome/reverse_tokens/" + s.exchangeName + "Basetoken") + if err != nil { + log.Error("error getting basetokens for which pairs should be reversed: ", err) + } + log.Infof("reverse the following basetokens on %s: %v", s.exchangeName, s.reverseBasetokens) + + s.reverseQuotetokens, err = getReverseTokensFromConfig("velodrome/reverse_tokens/" + s.exchangeName + "Quotetoken") + if err != nil { + log.Error("error getting quotetokens for which pairs should be reversed: ", err) + } + log.Infof("reverse the following quotetokens on %s: %v", s.exchangeName, s.reverseQuotetokens) + + s.fullPools, err = getReverseTokensFromConfig("velodrome/fullPools/" + s.exchangeName + "FullPools") + if err != nil { + log.Error("error getting fullPools for which pairs should be reversed: ", err) + } + log.Infof("Take into account both directions of a trade on the following pools: %v", s.fullPools) + err = s.loadPools(liquidityThreshold, liquidityThresholdUSD) if err != nil { log.Fatal("load pools: ", err) @@ -153,6 +189,30 @@ func (s *VelodromeScraper) WatchSwaps(pool dia.Pool) { VerifiedPair: true, } + switch { + case utils.Contains(s.reverseBasetokens, token1.Address): + // If we need quotation of a base token, reverse pair + tSwapped, err := dia.SwapTrade(*t) + if err == nil { + t = &tSwapped + } + case utils.Contains(s.reverseQuotetokens, token0.Address): + // If we need quotation of a base token, reverse pair + tSwapped, err := dia.SwapTrade(*t) + if err == nil { + t = &tSwapped + } + } + + if utils.Contains(s.fullPools, pool.Address) { + tSwapped, err := dia.SwapTrade(*t) + if err == nil { + if tSwapped.Price > 0 { + s.chanTrades <- &tSwapped + } + } + } + if price > 0 { log.Infof("Got trade at time %v - symbol: %s, pair: %s, price: %v, volume:%v", t.Time, t.Symbol, t.Pair, t.Price, t.Volume) s.chanTrades <- t @@ -292,33 +352,51 @@ func (ps *VelodromePairScraper) Pair() dia.ExchangePair { func (s *VelodromeScraper) loadPools(liquiThreshold float64, liquidityThresholdUSD float64) (err error) { var pools []dia.Pool - // Load all pools above liqui threshold. - pools, err = s.relDB.GetAllPoolsExchange(s.exchangeName, liquiThreshold) - if err != nil { - return - } + if s.listenByAddress { - log.Info("Found ", len(pools), " pools.") - log.Info("make pool map...") - lowerBoundCount := 0 - for _, pool := range pools { - if len(pool.Assetvolumes) != 2 { - log.Warn("not enough assets in pool with address: ", pool.Address) - continue + // Only load pool info for addresses from json file. + poolAddresses, errAddr := getAddressesFromConfig("velodrome/subscribe_pools/" + s.exchangeName) + if errAddr != nil { + log.Error("fetch pool addresses from config file: ", errAddr) + } + for _, address := range poolAddresses { + pool, errPool := s.relDB.GetPoolByAddress(Exchanges[s.exchangeName].BlockChain.Name, address.Hex()) + if errPool != nil { + log.Fatalf("Get pool with address %s: %v", address.Hex(), errPool) + } + s.pools = append(s.pools, pool) } - liquidity, lowerBound := pool.GetPoolLiquidityUSD() - // Discard pool if complete USD liquidity is below threshold. - if !lowerBound && liquidity < liquidityThresholdUSD { - continue + } else { + + // Load all pools above liqui threshold. + pools, err = s.relDB.GetAllPoolsExchange(s.exchangeName, liquiThreshold) + if err != nil { + return } - if lowerBound { - lowerBoundCount++ + + log.Info("Found ", len(pools), " pools.") + log.Info("make pool map...") + lowerBoundCount := 0 + for _, pool := range pools { + if len(pool.Assetvolumes) != 2 { + log.Warn("not enough assets in pool with address: ", pool.Address) + continue + } + + liquidity, lowerBound := pool.GetPoolLiquidityUSD() + // Discard pool if complete USD liquidity is below threshold. + if !lowerBound && liquidity < liquidityThresholdUSD { + continue + } + if lowerBound { + lowerBoundCount++ + } + s.pools = append(s.pools, pool) } - s.pools = append(s.pools, pool) + log.Infof("found %v subscribable pools.", len(s.pools)) + log.Infof("%v pools with lowerBound=true.", lowerBoundCount) } - log.Infof("found %v subscribable pools.", len(s.pools)) - log.Infof("%v pools with lowerBound=true.", lowerBoundCount) return } diff --git a/pkg/dia/scraper/liquidity-scrapers/ScraperInterface.go b/pkg/dia/scraper/liquidity-scrapers/ScraperInterface.go index 100297a7d..3d00b241d 100644 --- a/pkg/dia/scraper/liquidity-scrapers/ScraperInterface.go +++ b/pkg/dia/scraper/liquidity-scrapers/ScraperInterface.go @@ -33,6 +33,8 @@ func NewLiquidityScraper(source string, relDB *models.RelDB, datastore *models.D switch source { case dia.UniswapExchange: return NewUniswapScraper(exchanges[dia.UniswapExchange], relDB, datastore) + case dia.UniswapExchangeBase: + return NewUniswapScraper(exchanges[dia.UniswapExchangeBase], relDB, datastore) case dia.SushiSwapExchange: return NewUniswapScraper(exchanges[dia.SushiSwapExchange], relDB, datastore) case dia.SushiSwapExchangePolygon: @@ -139,6 +141,8 @@ func NewLiquidityScraper(source string, relDB *models.RelDB, datastore *models.D return NewUniswapV3Scraper(exchanges[dia.UniswapExchangeV3Polygon], relDB, datastore) case dia.UniswapExchangeV3Arbitrum: return NewUniswapV3Scraper(exchanges[dia.UniswapExchangeV3Arbitrum], relDB, datastore) + case dia.UniswapExchangeV3Base: + return NewUniswapV3Scraper(exchanges[dia.UniswapExchangeV3Base], relDB, datastore) case dia.BancorExchange: return NewBancorPoolScraper(exchanges[dia.BancorExchange], datastore) @@ -150,6 +154,10 @@ func NewLiquidityScraper(source string, relDB *models.RelDB, datastore *models.D return NewPlatypusScraper(exchanges[dia.PlatypusExchange], datastore) case dia.VelodromeExchange: return NewVelodromePoolScraper(exchanges[dia.VelodromeExchange], relDB, datastore) + case dia.AerodromeV1Exchange: + return NewVelodromePoolScraper(exchanges[dia.AerodromeV1Exchange], relDB, datastore) + case dia.AerodromeSlipstreamExchange: + return NewVelodromePoolScraper(exchanges[dia.AerodromeSlipstreamExchange], relDB, datastore) case dia.MaverickExchange: return NewMaverickScraper(exchanges[dia.MaverickExchange], relDB, datastore) case dia.PearlfiExchangeTestnet: diff --git a/pkg/dia/scraper/liquidity-scrapers/UniswapV2.go b/pkg/dia/scraper/liquidity-scrapers/UniswapV2.go index 8f5163058..592322252 100644 --- a/pkg/dia/scraper/liquidity-scrapers/UniswapV2.go +++ b/pkg/dia/scraper/liquidity-scrapers/UniswapV2.go @@ -31,6 +31,7 @@ type UniswapPair struct { const ( restDialEthereum = "" + restDialBase = "" restDialBSC = "" restDialPolygon = "" restDialCelo = "" @@ -87,6 +88,8 @@ func NewUniswapScraper(exchange dia.Exchange, relDB *models.RelDB, datastore *mo switch exchange.Name { case dia.UniswapExchange: us = makeUniswapPoolScraper(exchange, pathToPools, restDialEthereum, relDB, datastore, uniswapWaitMilliseconds) + case dia.UniswapExchangeBase: + us = makeUniswapPoolScraper(exchange, pathToPools, restDialBase, relDB, datastore, uniswapWaitMilliseconds) case dia.SushiSwapExchange: us = makeUniswapPoolScraper(exchange, pathToPools, restDialEthereum, relDB, datastore, sushiswapWaitMilliseconds) case dia.SushiSwapExchangePolygon: diff --git a/pkg/dia/scraper/liquidity-scrapers/UniswapV3.go b/pkg/dia/scraper/liquidity-scrapers/UniswapV3.go index 782634d37..36e1d3590 100644 --- a/pkg/dia/scraper/liquidity-scrapers/UniswapV3.go +++ b/pkg/dia/scraper/liquidity-scrapers/UniswapV3.go @@ -45,6 +45,8 @@ func NewUniswapV3Scraper(exchange dia.Exchange, relDB *models.RelDB, datastore * switch exchange.Name { case dia.UniswapExchangeV3: uls = makeUniswapV3Scraper(exchange, "", "", relDB, datastore, "200", uint64(12369621)) + case dia.UniswapExchangeV3Base: + uls = makeUniswapV3Scraper(exchange, "", "", relDB, datastore, "200", uint64(1371680)) case dia.UniswapExchangeV3Polygon: uls = makeUniswapV3Scraper(exchange, "", "", relDB, datastore, "200", uint64(22757913)) case dia.UniswapExchangeV3Arbitrum: diff --git a/pkg/dia/scraper/liquidity-scrapers/Velodrome.go b/pkg/dia/scraper/liquidity-scrapers/Velodrome.go index b8b838d98..c29c4e427 100644 --- a/pkg/dia/scraper/liquidity-scrapers/Velodrome.go +++ b/pkg/dia/scraper/liquidity-scrapers/Velodrome.go @@ -31,14 +31,18 @@ type VelodromePoolScraper struct { var ( velodromeWaitMilliseconds = "500" - velodromeRestDial = "" + restDialOptimism = "" ) func NewVelodromePoolScraper(exchange dia.Exchange, relDB *models.RelDB, datastore *models.DB) (us *VelodromePoolScraper) { switch exchange.Name { case dia.VelodromeExchange: - us = makeVelodromePoolScraper(exchange, relDB, datastore, velodromeRestDial, velodromeWaitMilliseconds) + us = makeVelodromePoolScraper(exchange, relDB, datastore, restDialOptimism, velodromeWaitMilliseconds) + case dia.AerodromeV1Exchange: + us = makeVelodromePoolScraper(exchange, relDB, datastore, restDialBase, velodromeWaitMilliseconds) + case dia.AerodromeSlipstreamExchange: + us = makeVelodromePoolScraper(exchange, relDB, datastore, restDialBase, velodromeWaitMilliseconds) } go func() { @@ -111,19 +115,20 @@ func (us *VelodromePoolScraper) GetPoolByID(num int64) (dia.Pool, error) { contract, err := velodrome.NewPoolFactoryCaller(common.HexToAddress(us.exchange.Contract), us.RestClient) if err != nil { - log.Error(err) + log.Error("NewPoolFactoryCaller: ", err) return dia.Pool{}, err } pairAddress, err := contract.AllPools(&bind.CallOpts{}, big.NewInt(num)) if err != nil { - log.Error(err) + log.Error("AllPools: ", err) return dia.Pool{}, err } + log.Info("pool: ", pairAddress.Hex()) pool, err := us.GetPoolByAddress(pairAddress) if err != nil { - log.Error(err) + log.Error("GetPoolByAddress: ", err) return dia.Pool{}, err } @@ -165,7 +170,20 @@ func (us *VelodromePoolScraper) GetPoolByAddress(pairAddress common.Address) (po } } - // Getting liquidity + if us.exchange.Name == dia.AerodromeSlipstreamExchange { + us.GetLiquidityUniV3Type(token0, token1, pairAddress, &pool) + } else { + us.GetLiquidityUniV2Type(token0, token1, pairContract, &pool) + } + + pool.Address = pairAddress.Hex() + pool.Blockchain = dia.BlockChain{Name: us.blockchain} + pool.Exchange = dia.Exchange{Name: us.exchange.Name} + + return pool, nil +} + +func (us *VelodromePoolScraper) GetLiquidityUniV2Type(token0 dia.Asset, token1 dia.Asset, pairContract *velodrome.IPoolCaller, pool *dia.Pool) { liquidity, err := pairContract.GetReserves(&bind.CallOpts{}) if err != nil { log.Error("get reserves: ", err) @@ -191,14 +209,30 @@ func (us *VelodromePoolScraper) GetPoolByAddress(pairAddress common.Address) (po // Determine USD liquidity if pool.SufficientNativeBalance(GLOBAL_NATIVE_LIQUIDITY_THRESHOLD) { - us.datastore.GetPoolLiquiditiesUSD(&pool, priceCache) + us.datastore.GetPoolLiquiditiesUSD(pool, priceCache) } +} - pool.Address = pairAddress.Hex() - pool.Blockchain = dia.BlockChain{Name: us.blockchain} - pool.Exchange = dia.Exchange{Name: us.exchange.Name} +func (us *VelodromePoolScraper) GetLiquidityUniV3Type(token0 dia.Asset, token1 dia.Asset, pairAddress common.Address, pool *dia.Pool) { + balance0Big, err := ethhelper.GetBalanceOf(common.HexToAddress(token0.Address), pairAddress, us.RestClient) + if err != nil { + log.Error("GetBalanceOf: ", err) + } + balance1Big, err := ethhelper.GetBalanceOf(common.HexToAddress(token1.Address), pairAddress, us.RestClient) + if err != nil { + log.Error("GetBalanceOf: ", err) + } + log.Infof("balance0 -- balance1 -- pool: %v -- %v -- %s ", balance0Big, balance1Big, pairAddress) + balance0, _ := new(big.Float).Quo(big.NewFloat(0).SetInt(balance0Big), new(big.Float).SetFloat64(math.Pow10(int(token0.Decimals)))).Float64() + balance1, _ := new(big.Float).Quo(big.NewFloat(0).SetInt(balance1Big), new(big.Float).SetFloat64(math.Pow10(int(token1.Decimals)))).Float64() - return pool, nil + pool.Assetvolumes = append(pool.Assetvolumes, dia.AssetVolume{Asset: token0, Volume: balance0, Index: uint8(0)}) + pool.Assetvolumes = append(pool.Assetvolumes, dia.AssetVolume{Asset: token1, Volume: balance1, Index: uint8(1)}) + + // Determine USD liquidity + if balance0 > GLOBAL_NATIVE_LIQUIDITY_THRESHOLD && balance1 > GLOBAL_NATIVE_LIQUIDITY_THRESHOLD { + us.datastore.GetPoolLiquiditiesUSD(pool, priceCache) + } } // GetDecimals returns the decimals of the token with address @tokenAddress diff --git a/pkg/dia/service/assetservice/source/velodrome.go b/pkg/dia/service/assetservice/source/velodrome.go index 3e808bdb0..4ca4df53f 100644 --- a/pkg/dia/service/assetservice/source/velodrome.go +++ b/pkg/dia/service/assetservice/source/velodrome.go @@ -7,6 +7,7 @@ import ( "time" "github.com/diadata-org/diadata/pkg/dia/scraper/exchange-scrapers/velodrome" + models "github.com/diadata-org/diadata/pkg/model" "github.com/diadata-org/diadata/pkg/dia" "github.com/diadata-org/diadata/pkg/utils" @@ -24,22 +25,29 @@ type VelodromePair struct { const ( velodromeWaitMilliseconds = "500" + restDialOptimism = "" ) var velodromeExchangeFactoryContractAddress string type VelodromeAssetSource struct { RestClient *ethclient.Client + relDB *models.RelDB assetChannel chan dia.Asset doneChannel chan bool + exchange dia.Exchange blockchain string waitTime int } -func NewVelodromeAssetSource(exchange dia.Exchange) (uas *VelodromeAssetSource) { +func NewVelodromeAssetSource(exchange dia.Exchange, relDB *models.RelDB) (uas *VelodromeAssetSource) { switch exchange.Name { case dia.VelodromeExchange: - uas = makeVelodromeAssetSource(exchange, exchange.RestAPI, velodromeWaitMilliseconds) + uas = makeVelodromeAssetSource(exchange, restDialOptimism, relDB, velodromeWaitMilliseconds) + case dia.AerodromeSlipstreamExchange: + uas = makeVelodromeAssetSource(exchange, restDialBase, relDB, velodromeWaitMilliseconds) + case dia.AerodromeV1Exchange: + uas = makeVelodromeAssetSource(exchange, restDialBase, relDB, velodromeWaitMilliseconds) } velodromeExchangeFactoryContractAddress = exchange.Contract @@ -52,7 +60,7 @@ func NewVelodromeAssetSource(exchange dia.Exchange) (uas *VelodromeAssetSource) } // makeVelodromeAssetSource returns an asset source as used in NewVelodromeAssetSource. -func makeVelodromeAssetSource(exchange dia.Exchange, restDial string, waitMilliseconds string) *VelodromeAssetSource { +func makeVelodromeAssetSource(exchange dia.Exchange, restDial string, relDB *models.RelDB, waitMilliseconds string) *VelodromeAssetSource { var ( restClient *ethclient.Client err error @@ -75,8 +83,10 @@ func makeVelodromeAssetSource(exchange dia.Exchange, restDial string, waitMillis } uas = &VelodromeAssetSource{ RestClient: restClient, + relDB: relDB, assetChannel: assetChannel, doneChannel: doneChannel, + exchange: exchange, blockchain: exchange.BlockChain.Name, waitTime: waitTime, } @@ -95,7 +105,7 @@ func (uas *VelodromeAssetSource) getNumPairs() (int, error) { var contract *velodrome.PoolFactoryCaller contract, err := velodrome.NewPoolFactoryCaller(common.HexToAddress(velodromeExchangeFactoryContractAddress), uas.RestClient) if err != nil { - log.Error(err) + log.Error("getNumPairs: ", err) } numPairs, err := contract.AllPoolsLength(&bind.CallOpts{}) @@ -111,11 +121,18 @@ func (uas *VelodromeAssetSource) fetchAssets() { log.Info("Found ", numPairs, " pairs") checkMap := make(map[string]struct{}) - for i := 0; i < numPairs; i++ { + _, index, err := uas.relDB.GetScraperIndex(uas.exchange.Name, dia.SCRAPER_TYPE_ASSETCOLLECTOR) + if err != nil { + log.Error("GetScraperIndex: ", err) + } + log.Info("Start at index ", index) + + for index < int64(numPairs) { + log.Info("index: ", index) time.Sleep(time.Duration(uas.waitTime) * time.Millisecond) - pair, err := uas.GetPairByID(int64(numPairs - 1 - i)) + pair, err := uas.GetPairByID(int64(numPairs) - 1 - index) if err != nil { - log.Errorln("Error getting pair with ID ", numPairs-1-i) + log.Errorln("Error getting pair with ID ", numPairs-1-int(index)) } asset0 := pair.Token0 asset1 := pair.Token1 @@ -134,6 +151,7 @@ func (uas *VelodromeAssetSource) fetchAssets() { uas.assetChannel <- asset1 } } + index++ } uas.doneChannel <- true } diff --git a/pkg/utils/strings.go b/pkg/utils/strings.go index 52fa1ae16..6b635c361 100644 --- a/pkg/utils/strings.go +++ b/pkg/utils/strings.go @@ -18,6 +18,9 @@ func UniqueStrings(s []string) []string { // Contains takes a slice of strings and a string and checks if it is contained in the slice. func Contains(s *[]string, str string) bool { + if s == nil { + return false + } for _, a := range *s { if a == str { return true