diff --git a/cmd/assetCollectionService/go.mod b/cmd/assetCollectionService/go.mod index 60223aaa1..e16d149dc 100644 --- a/cmd/assetCollectionService/go.mod +++ b/cmd/assetCollectionService/go.mod @@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/assetCollectionService go 1.20 require ( - github.com/diadata-org/diadata v1.4.528 + github.com/diadata-org/diadata v1.4.533 github.com/sirupsen/logrus v1.9.3 ) diff --git a/cmd/assetCollectionService/main.go b/cmd/assetCollectionService/main.go index 7a136915a..670e54e10 100644 --- a/cmd/assetCollectionService/main.go +++ b/cmd/assetCollectionService/main.go @@ -46,6 +46,8 @@ func NewAssetScraper(exchange string, secret string, relDB *models.RelDB) source switch exchange { case dia.UniswapExchange: return source.NewUniswapAssetSource(exchanges[dia.UniswapExchange], relDB) + case dia.UniswapExchangeBase: + return source.NewUniswapAssetSource(exchanges[dia.UniswapExchangeBase], relDB) case dia.PanCakeSwap: return source.NewUniswapAssetSource(exchanges[dia.PanCakeSwap], relDB) case dia.PanCakeSwapExchangeV3: @@ -116,6 +118,8 @@ func NewAssetScraper(exchange string, secret string, relDB *models.RelDB) source return source.NewUniswapAssetSource(exchanges[dia.NileV1Exchange], relDB) case dia.UniswapExchangeV3: return source.NewUniswapV3AssetSource(exchanges[dia.UniswapExchangeV3], relDB) + case dia.UniswapExchangeV3Base: + return source.NewUniswapV3AssetSource(exchanges[dia.UniswapExchangeV3Base], relDB) case dia.UniswapExchangeV3Polygon: return source.NewUniswapV3AssetSource(exchanges[dia.UniswapExchangeV3Polygon], relDB) case dia.UniswapExchangeV3Arbitrum: diff --git a/go.mod b/go.mod index 198a06aff..a4d580680 100644 --- a/go.mod +++ b/go.mod @@ -229,6 +229,7 @@ require ( github.com/hdevalence/ed25519consensus v0.1.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect + github.com/machinebox/graphql v0.2.2 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/prometheus/tsdb v0.10.0 // indirect github.com/rjeczalik/notify v0.9.2 // indirect diff --git a/go.sum b/go.sum index cf2a63988..178a8c287 100644 --- a/go.sum +++ b/go.sum @@ -1092,6 +1092,8 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo= +github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAEolaQybkjWA= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= diff --git a/pkg/dia/Config.go b/pkg/dia/Config.go index 5ea761247..c4b949fdc 100644 --- a/pkg/dia/Config.go +++ b/pkg/dia/Config.go @@ -57,9 +57,11 @@ const ( FilterKing = "MAIR120" BancorExchange = "Bancor" UniswapExchange = "Uniswap" + UniswapExchangeBase = "Uniswap-Base" UniswapExchangeV3 = "UniswapV3" UniswapExchangeV3Polygon = "UniswapV3-polygon" UniswapExchangeV3Arbitrum = "UniswapV3-Arbitrum" + UniswapExchangeV3Base = "UniswapV3-Base" LoopringExchange = "Loopring" CamelotExchange = "Camelot" CamelotExchangeV3 = "CamelotV3" diff --git a/pkg/dia/service/assetservice/source/uniswap.go b/pkg/dia/service/assetservice/source/uniswap.go index 57d99d792..8bc42b4fb 100644 --- a/pkg/dia/service/assetservice/source/uniswap.go +++ b/pkg/dia/service/assetservice/source/uniswap.go @@ -25,6 +25,7 @@ type UniswapPair struct { const ( restDial = "" + restDialBase = "" restDialBSC = "" restDialPolygon = "" restDialCelo = "" @@ -44,6 +45,7 @@ const ( restDialLinea = "" uniswapWaitMilliseconds = "25" + baseWaitMilliseconds = "200" sushiswapWaitMilliseconds = "100" pancakeswapWaitMilliseconds = "100" dfynWaitMilliseconds = "100" @@ -138,6 +140,8 @@ func NewUniswapAssetSource(exchange dia.Exchange, relDB *models.RelDB) (uas *Uni uas = makeUniswapAssetSource(exchange, restDialLinea, relDB, wanchainWaitMilliseconds) case dia.ThenaExchange: uas = makeUniswapAssetSource(exchange, restDialBSC, relDB, sushiswapWaitMilliseconds) + case dia.UniswapExchangeBase: + uas = makeUniswapAssetSource(exchange, restDialBase, relDB, baseWaitMilliseconds) } exchangeFactoryContractAddress = exchange.Contract diff --git a/pkg/dia/service/assetservice/source/uniswapv3.go b/pkg/dia/service/assetservice/source/uniswapv3.go index 2c385b5c0..6aebad55f 100644 --- a/pkg/dia/service/assetservice/source/uniswapv3.go +++ b/pkg/dia/service/assetservice/source/uniswapv3.go @@ -1,6 +1,7 @@ package source import ( + "context" "math/big" "strconv" "strings" @@ -18,6 +19,8 @@ import ( "github.com/ethereum/go-ethereum/ethclient" ) +var numBlocksQuery = uint64(1000) + type UniswapV3AssetSource struct { RestClient *ethclient.Client WsClient *ethclient.Client @@ -45,6 +48,8 @@ func NewUniswapV3AssetSource(exchange dia.Exchange, relDB *models.RelDB) *Uniswa uas = makeUniswapV3AssetSource(exchange, "", "", relDB, "200", uint64(22757913)) case dia.UniswapExchangeV3Arbitrum: uas = makeUniswapV3AssetSource(exchange, "", "", relDB, "200", uint64(165)) + case dia.UniswapExchangeV3Base: + uas = makeUniswapV3AssetSource(exchange, "", "", relDB, "200", uint64(1371680)) case dia.PanCakeSwapExchangeV3: uas = makeUniswapV3AssetSource(exchange, "", "", relDB, "200", uint64(26956207)) case dia.RamsesV2Exchange: @@ -124,42 +129,55 @@ func (uas *UniswapV3AssetSource) fetchAssets() { log.Error(err) } - poolCreated, err := contract.FilterPoolCreated( - &bind.FilterOpts{Start: uas.startBlock}, - []common.Address{}, - []common.Address{}, - []*big.Int{}, - ) + currentBlockNumber, err := uas.RestClient.BlockNumber(context.Background()) if err != nil { - log.Error("filter pool created: ", err) + log.Error("GetBlockNumber: ", err) } - for poolCreated.Next() { - time.Sleep(time.Duration(uas.waitTime) * time.Millisecond) - poolsCount++ - log.Info("pools count: ", poolsCount) - blocknumber = int64(poolCreated.Event.Raw.BlockNumber) - // Don't repeat sending already sent assets - if _, ok := checkMap[poolCreated.Event.Token0.Hex()]; !ok { - checkMap[poolCreated.Event.Token0.Hex()] = struct{}{} - asset, err := uas.GetAssetFromAddress(poolCreated.Event.Token0) - if err != nil { - log.Warnf("cannot fetch asset from address %s: %v", poolCreated.Event.Token0.Hex(), err) - } - uas.assetChannel <- asset + + endblock := utils.Min(uint64(uas.startBlock)+numBlocksQuery, currentBlockNumber) + log.Infof("startblock -- endblock: %v -- %v", uas.startBlock, endblock) + + for uas.startBlock <= currentBlockNumber { + poolCreated, err := contract.FilterPoolCreated( + &bind.FilterOpts{Start: uas.startBlock, End: &endblock}, + []common.Address{}, + []common.Address{}, + []*big.Int{}, + ) + if err != nil { + log.Error("filter pool created: ", err) } - if _, ok := checkMap[poolCreated.Event.Token1.Hex()]; !ok { - checkMap[poolCreated.Event.Token1.Hex()] = struct{}{} - asset, err := uas.GetAssetFromAddress(poolCreated.Event.Token1) - if err != nil { - log.Warnf("cannot fetch asset from address %s: %v", poolCreated.Event.Token1.Hex(), err) + for poolCreated.Next() { + time.Sleep(time.Duration(uas.waitTime) * time.Millisecond) + poolsCount++ + log.Info("pools count: ", poolsCount) + blocknumber = int64(poolCreated.Event.Raw.BlockNumber) + // Don't repeat sending already sent assets + if _, ok := checkMap[poolCreated.Event.Token0.Hex()]; !ok { + checkMap[poolCreated.Event.Token0.Hex()] = struct{}{} + asset, err := uas.GetAssetFromAddress(poolCreated.Event.Token0) + if err != nil { + log.Warnf("cannot fetch asset from address %s: %v", poolCreated.Event.Token0.Hex(), err) + } + uas.assetChannel <- asset + } + if _, ok := checkMap[poolCreated.Event.Token1.Hex()]; !ok { + checkMap[poolCreated.Event.Token1.Hex()] = struct{}{} + asset, err := uas.GetAssetFromAddress(poolCreated.Event.Token1) + if err != nil { + log.Warnf("cannot fetch asset from address %s: %v", poolCreated.Event.Token1.Hex(), err) + } + uas.assetChannel <- asset } - uas.assetChannel <- asset } + err = uas.relDB.SetScraperIndex(uas.exchange.Name, dia.SCRAPER_TYPE_ASSETCOLLECTOR, dia.INDEX_TYPE_BLOCKNUMBER, blocknumber) + if err != nil { + log.Error("SetScraperIndex: ", err) + } + endblock += numBlocksQuery + uas.startBlock += numBlocksQuery } - err = uas.relDB.SetScraperIndex(uas.exchange.Name, dia.SCRAPER_TYPE_ASSETCOLLECTOR, dia.INDEX_TYPE_BLOCKNUMBER, blocknumber) - if err != nil { - log.Error("SetScraperIndex: ", err) - } + uas.doneChannel <- true } diff --git a/pkg/utils/numbers.go b/pkg/utils/numbers.go index d9515ed59..3a7b67096 100644 --- a/pkg/utils/numbers.go +++ b/pkg/utils/numbers.go @@ -14,3 +14,10 @@ func StringToFloat64(value string, decimals int64) (float64, bool) { result := num.Div(mul) return result.Float64() } + +func Min(n1 uint64, n2 uint64) uint64 { + if n1 <= n2 { + return n1 + } + return n2 +}