feat: implement retry calls for etherscan and bscscan when rate limited

go.mod

@@ -17,3 +17,5 @@ require (
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sys v0.0.0-20190412213103-97732733099d
 )
+
+replace github.com/nanmu42/etherscan-api v1.2.0 => github.com/will7200/etherscan-api v1.2.1-0.20210516225556-39daf32d6557

go.sum

@@ -23,8 +23,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/nanmu42/etherscan-api v1.2.0 h1:ZUK8Ed4JZT83FC3Ze2889t6w8X3D5RZjsN/PTJv8Psc=
-github.com/nanmu42/etherscan-api v1.2.0/go.mod h1:JNY1YEQ0cL4Ytlnb3Hf3tjk7rIgprDbludV4xVESLcg=
 github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw=
 github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
@@ -42,6 +40,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/will7200/etherscan-api v1.2.1-0.20210516225556-39daf32d6557 h1:KcMyg9HIxRTFnBK6IH/big7kHvYlmH1pNmovxcqCQK8=
+github.com/will7200/etherscan-api v1.2.1-0.20210516225556-39daf32d6557/go.mod h1:JNY1YEQ0cL4Ytlnb3Hf3tjk7rIgprDbludV4xVESLcg=
 go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A=

internal/providers/bscscan/holding.go

@@ -4,15 +4,18 @@ import (
 	"errors"
 	"math"
 	"math/big"
+	"net/http"
 	"sync"
 	"time"
 
+	"github.com/HereMobilityDevelopers/mediary"
 	"github.com/mitchellh/mapstructure"
 	"github.com/nanmu42/etherscan-api"
 	"go.uber.org/zap"
 
 	"github.com/will7200/go-crypto-sync/internal/common"
 	"github.com/will7200/go-crypto-sync/internal/providers"
+	ietherscan "github.com/will7200/go-crypto-sync/internal/providers/etherscan"
 )
 
 func init() {
@@ -65,26 +68,26 @@ type Provider struct {
 var _ providers.Account = &Provider{}
 var _ providers.Provider = &Provider{}
 
-func (p *Provider) Name() string {
-	return "bscscan"
-}
-
 func (p *Provider) Once() {
-	client := etherscan.NewCustomized(etherscan.Customization{
-		Timeout:       15 * time.Second,
-		Key:           p.data.ApiKey,
-		BaseURL:       p.data.BaseURL,
-		Verbose:       p.data.Debug,
-		BeforeRequest: nil,
-		AfterRequest:  nil,
+	httpClientBase := &http.Client{
+		Timeout: 15 * time.Second,
+	}
+	httpClient := mediary.Init().WithPreconfiguredClient(httpClientBase)
+
+	if p.data.Debug {
+		httpClient = httpClient.AddInterceptors(common.DumpRequestResponseWrappedLogger(p.logger))
+	}
+	httpClient = httpClient.AddInterceptors(ietherscan.AddTurtleForRateLimiter(p.logger, 3))
+	p.client = etherscan.NewCustomized(etherscan.Customization{
+		Key:     p.data.ApiKey,
+		BaseURL: p.data.BaseURL,
+		Client:  httpClient.Build(),
+		Verbose: p.data.Debug,
 	})
-	p.client = client
-	//client.BeforeRequest = func(module, action string, param map[string]interface{}) error {
-	//	// ...
-	//}
-	//client.AfterRequest = func(module, action string, param map[string]interface{}, outcome interface{}, requestErr error) {
-	//	// ...
-	//}
+}
+
+func (p *Provider) Name() string {
+	return "bscscan"
 }
 
 func (p *Provider) GetHoldings() (providers.Holdings, error) {

internal/providers/etherscan/holding.go

@@ -1,13 +1,20 @@
 package etherscan
 
 import (
+	"bytes"
 	"errors"
+	"fmt"
+	"io/ioutil"
 	"math"
 	"math/big"
+	"net/http"
 	"sync"
+	"time"
 
+	"github.com/HereMobilityDevelopers/mediary"
 	"github.com/mitchellh/mapstructure"
 	"github.com/nanmu42/etherscan-api"
+	"github.com/will7200/go-crypto-sync/internal/common"
 	"go.uber.org/zap"
 
 	"github.com/will7200/go-crypto-sync/internal/providers"
@@ -62,22 +69,53 @@ type Provider struct {
 var _ providers.Account = &Provider{}
 var _ providers.Provider = &Provider{}
 
+func AddTurtleForRateLimiter(logger *zap.SugaredLogger, retryAmount int) func(req *http.Request, handler mediary.Handler) (*http.Response, error) {
+	maxLimitMessage := "Max rate limit reached"
+	return func(req *http.Request, handler mediary.Handler) (*http.Response, error) {
+		r, err := handler(req)
+		for ra := retryAmount; ra > 0 && err == nil; ra-- {
+			if r.StatusCode == http.StatusOK {
+				bodyBytes, _ := ioutil.ReadAll(r.Body)
+				r.Body.Close() //  must close
+				r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
+				if bytes.Contains(bodyBytes, []byte(maxLimitMessage)) {
+					logger.Warnf("Rate limited for %s. Sleeping for one second", req.Host)
+					time.Sleep(1 * time.Second)
+					r, err = handler(req)
+				} else {
+					break
+				}
+			} else {
+				break
+			}
+		}
+		return r, err
+	}
+}
+
+func (p *Provider) Once() {
+	httpClientBase := &http.Client{
+		Timeout: 15 * time.Second,
+	}
+	httpClient := mediary.Init().WithPreconfiguredClient(httpClientBase)
+
+	if p.data.Debug {
+		httpClient = httpClient.AddInterceptors(common.DumpRequestResponseWrappedLogger(p.logger))
+	}
+	httpClient = httpClient.AddInterceptors(AddTurtleForRateLimiter(p.logger, 3))
+	p.client = etherscan.NewCustomized(etherscan.Customization{
+		Key:     p.data.ApiKey,
+		BaseURL: fmt.Sprintf(`https://%s.etherscan.io/api?`, p.data.Network.SubDomain()),
+		Client:  httpClient.Build(),
+		//Verbose: p.data.Debug,
+	})
+}
+
 func (p *Provider) Name() string {
 	return "etherscan"
 }
 
 func (p *Provider) GetHoldings() (providers.Holdings, error) {
-	p.once.Do(func() {
-		client := etherscan.New(p.data.Network, p.data.ApiKey)
-		client.Verbose = p.data.Debug
-		p.client = client
-		//client.BeforeRequest = func(module, action string, param map[string]interface{}) error {
-		//	// ...
-		//}
-		//client.AfterRequest = func(module, action string, param map[string]interface{}, outcome interface{}, requestErr error) {
-		//	// ...
-		//}
-	})
 	client := p.client
 	h := make([]providers.Holding, 0, 25)
 	for _, account := range p.data.Accounts {
@@ -130,5 +168,6 @@ func (p *Provider) Open(config providers.Config, params ...interface{}) (provide
 	} else {
 		return nil, errors.New("invalid parameters")
 	}
+	p.Once()
 	return p, nil
 }