Merge branch 'master' of github.com:diadata-org/diadata

cmd/assetCollectionService/go.mod

@@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/assetCollectionService
 go 1.20
 
 require (
-	github.com/diadata-org/diadata v1.4.528
+	github.com/diadata-org/diadata v1.4.533
 	github.com/sirupsen/logrus v1.9.3
 )
 

cmd/assetCollectionService/main.go

@@ -46,6 +46,8 @@ func NewAssetScraper(exchange string, secret string, relDB *models.RelDB) source
 	switch exchange {
 	case dia.UniswapExchange:
 		return source.NewUniswapAssetSource(exchanges[dia.UniswapExchange], relDB)
+	case dia.UniswapExchangeBase:
+		return source.NewUniswapAssetSource(exchanges[dia.UniswapExchangeBase], relDB)
 	case dia.PanCakeSwap:
 		return source.NewUniswapAssetSource(exchanges[dia.PanCakeSwap], relDB)
 	case dia.PanCakeSwapExchangeV3:
@@ -116,6 +118,8 @@ func NewAssetScraper(exchange string, secret string, relDB *models.RelDB) source
 		return source.NewUniswapAssetSource(exchanges[dia.NileV1Exchange], relDB)
 	case dia.UniswapExchangeV3:
 		return source.NewUniswapV3AssetSource(exchanges[dia.UniswapExchangeV3], relDB)
+	case dia.UniswapExchangeV3Base:
+		return source.NewUniswapV3AssetSource(exchanges[dia.UniswapExchangeV3Base], relDB)
 	case dia.UniswapExchangeV3Polygon:
 		return source.NewUniswapV3AssetSource(exchanges[dia.UniswapExchangeV3Polygon], relDB)
 	case dia.UniswapExchangeV3Arbitrum:

go.mod

@@ -229,6 +229,7 @@ require (
 	github.com/hdevalence/ed25519consensus v0.1.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
+	github.com/machinebox/graphql v0.2.2
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/prometheus/tsdb v0.10.0 // indirect
 	github.com/rjeczalik/notify v0.9.2 // indirect

go.sum

@@ -1092,6 +1092,8 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z
 github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4=
 github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
+github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo=
+github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAEolaQybkjWA=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=

pkg/dia/Config.go

@@ -57,9 +57,11 @@ const (
 	FilterKing                         = "MAIR120"
 	BancorExchange                     = "Bancor"
 	UniswapExchange                    = "Uniswap"
+	UniswapExchangeBase                = "Uniswap-Base"
 	UniswapExchangeV3                  = "UniswapV3"
 	UniswapExchangeV3Polygon           = "UniswapV3-polygon"
 	UniswapExchangeV3Arbitrum          = "UniswapV3-Arbitrum"
+	UniswapExchangeV3Base              = "UniswapV3-Base"
 	LoopringExchange                   = "Loopring"
 	CamelotExchange                    = "Camelot"
 	CamelotExchangeV3                  = "CamelotV3"

pkg/dia/service/assetservice/source/uniswap.go

@@ -25,6 +25,7 @@ type UniswapPair struct {
 
 const (
 	restDial              = ""
+	restDialBase          = ""
 	restDialBSC           = ""
 	restDialPolygon       = ""
 	restDialCelo          = ""
@@ -44,6 +45,7 @@ const (
 	restDialLinea         = ""
 
 	uniswapWaitMilliseconds     = "25"
+	baseWaitMilliseconds        = "200"
 	sushiswapWaitMilliseconds   = "100"
 	pancakeswapWaitMilliseconds = "100"
 	dfynWaitMilliseconds        = "100"
@@ -138,6 +140,8 @@ func NewUniswapAssetSource(exchange dia.Exchange, relDB *models.RelDB) (uas *Uni
 		uas = makeUniswapAssetSource(exchange, restDialLinea, relDB, wanchainWaitMilliseconds)
 	case dia.ThenaExchange:
 		uas = makeUniswapAssetSource(exchange, restDialBSC, relDB, sushiswapWaitMilliseconds)
+	case dia.UniswapExchangeBase:
+		uas = makeUniswapAssetSource(exchange, restDialBase, relDB, baseWaitMilliseconds)
 	}
 
 	exchangeFactoryContractAddress = exchange.Contract

pkg/dia/service/assetservice/source/uniswapv3.go

@@ -1,6 +1,7 @@
 package source
 
 import (
+	"context"
 	"math/big"
 	"strconv"
 	"strings"
@@ -18,6 +19,8 @@ import (
 	"github.com/ethereum/go-ethereum/ethclient"
 )
 
+var numBlocksQuery = uint64(1000)
+
 type UniswapV3AssetSource struct {
 	RestClient *ethclient.Client
 	WsClient   *ethclient.Client
@@ -45,6 +48,8 @@ func NewUniswapV3AssetSource(exchange dia.Exchange, relDB *models.RelDB) *Uniswa
 		uas = makeUniswapV3AssetSource(exchange, "", "", relDB, "200", uint64(22757913))
 	case dia.UniswapExchangeV3Arbitrum:
 		uas = makeUniswapV3AssetSource(exchange, "", "", relDB, "200", uint64(165))
+	case dia.UniswapExchangeV3Base:
+		uas = makeUniswapV3AssetSource(exchange, "", "", relDB, "200", uint64(1371680))
 	case dia.PanCakeSwapExchangeV3:
 		uas = makeUniswapV3AssetSource(exchange, "", "", relDB, "200", uint64(26956207))
 	case dia.RamsesV2Exchange:
@@ -124,42 +129,55 @@ func (uas *UniswapV3AssetSource) fetchAssets() {
 		log.Error(err)
 	}
 
-	poolCreated, err := contract.FilterPoolCreated(
-		&bind.FilterOpts{Start: uas.startBlock},
-		[]common.Address{},
-		[]common.Address{},
-		[]*big.Int{},
-	)
+	currentBlockNumber, err := uas.RestClient.BlockNumber(context.Background())
 	if err != nil {
-		log.Error("filter pool created: ", err)
+		log.Error("GetBlockNumber: ", err)
 	}
-	for poolCreated.Next() {
-		time.Sleep(time.Duration(uas.waitTime) * time.Millisecond)
-		poolsCount++
-		log.Info("pools count: ", poolsCount)
-		blocknumber = int64(poolCreated.Event.Raw.BlockNumber)
-		// Don't repeat sending already sent assets
-		if _, ok := checkMap[poolCreated.Event.Token0.Hex()]; !ok {
-			checkMap[poolCreated.Event.Token0.Hex()] = struct{}{}
-			asset, err := uas.GetAssetFromAddress(poolCreated.Event.Token0)
-			if err != nil {
-				log.Warnf("cannot fetch asset from address %s: %v", poolCreated.Event.Token0.Hex(), err)
-			}
-			uas.assetChannel <- asset
+
+	endblock := utils.Min(uint64(uas.startBlock)+numBlocksQuery, currentBlockNumber)
+	log.Infof("startblock -- endblock: %v -- %v", uas.startBlock, endblock)
+
+	for uas.startBlock <= currentBlockNumber {
+		poolCreated, err := contract.FilterPoolCreated(
+			&bind.FilterOpts{Start: uas.startBlock, End: &endblock},
+			[]common.Address{},
+			[]common.Address{},
+			[]*big.Int{},
+		)
+		if err != nil {
+			log.Error("filter pool created: ", err)
 		}
-		if _, ok := checkMap[poolCreated.Event.Token1.Hex()]; !ok {
-			checkMap[poolCreated.Event.Token1.Hex()] = struct{}{}
-			asset, err := uas.GetAssetFromAddress(poolCreated.Event.Token1)
-			if err != nil {
-				log.Warnf("cannot fetch asset from address %s: %v", poolCreated.Event.Token1.Hex(), err)
+		for poolCreated.Next() {
+			time.Sleep(time.Duration(uas.waitTime) * time.Millisecond)
+			poolsCount++
+			log.Info("pools count: ", poolsCount)
+			blocknumber = int64(poolCreated.Event.Raw.BlockNumber)
+			// Don't repeat sending already sent assets
+			if _, ok := checkMap[poolCreated.Event.Token0.Hex()]; !ok {
+				checkMap[poolCreated.Event.Token0.Hex()] = struct{}{}
+				asset, err := uas.GetAssetFromAddress(poolCreated.Event.Token0)
+				if err != nil {
+					log.Warnf("cannot fetch asset from address %s: %v", poolCreated.Event.Token0.Hex(), err)
+				}
+				uas.assetChannel <- asset
+			}
+			if _, ok := checkMap[poolCreated.Event.Token1.Hex()]; !ok {
+				checkMap[poolCreated.Event.Token1.Hex()] = struct{}{}
+				asset, err := uas.GetAssetFromAddress(poolCreated.Event.Token1)
+				if err != nil {
+					log.Warnf("cannot fetch asset from address %s: %v", poolCreated.Event.Token1.Hex(), err)
+				}
+				uas.assetChannel <- asset
 			}
-			uas.assetChannel <- asset
 		}
+		err = uas.relDB.SetScraperIndex(uas.exchange.Name, dia.SCRAPER_TYPE_ASSETCOLLECTOR, dia.INDEX_TYPE_BLOCKNUMBER, blocknumber)
+		if err != nil {
+			log.Error("SetScraperIndex: ", err)
+		}
+		endblock += numBlocksQuery
+		uas.startBlock += numBlocksQuery
 	}
-	err = uas.relDB.SetScraperIndex(uas.exchange.Name, dia.SCRAPER_TYPE_ASSETCOLLECTOR, dia.INDEX_TYPE_BLOCKNUMBER, blocknumber)
-	if err != nil {
-		log.Error("SetScraperIndex: ", err)
-	}
+
 	uas.doneChannel <- true
 }
 

pkg/utils/numbers.go

@@ -14,3 +14,10 @@ func StringToFloat64(value string, decimals int64) (float64, bool) {
 	result := num.Div(mul)
 	return result.Float64()
 }
+
+func Min(n1 uint64, n2 uint64) uint64 {
+	if n1 <= n2 {
+		return n1
+	}
+	return n2
+}