Merge pull request #73 from mailgun/thrawn/develop

Added setter.IsNil and retry package

README.md

@@ -286,6 +286,20 @@ argFoo = flag.String("foo", "", "foo via cli arg")
 setter.SetOverride(&config.Foo, *argFoo, os.Env("FOO"))
 ```
 
+## Check for Nil interface
+```go
+func NewImplementation() MyInterface {
+    // Type and Value are not nil
+    var p *MyImplementation = nil
+    return p
+}
+
+thing := NewImplementation()
+assert.False(t, thing == nil)
+assert.True(t, setter.IsNil(thing))
+assert.False(t, setter.IsNil(&MyImplementation{}))
+```
+
 ## GetEnv
 import "github.com/mailgun/holster/v3/config"
 Get a value from an environment variable or return the provided default
@@ -490,4 +504,63 @@ func TestUntilConnect(t *testing.T) {
     // Wait until we can connect, then continue with the test
     testutil.UntilConnect(t, 10, time.Millisecond*100, ln.Addr().String())
 }
-```
+```
+
+### Retry Until
+Retries a function until the function returns error = nil or until the context is deadline is exceeded
+```go
+ctx, cancel := context.WithTimeout(context.Background(), time.Second*20)
+defer cancel()
+err := retry.Until(ctx, retry.Interval(time.Millisecond*10), func(ctx context.Context, att int) error {
+    res, err := http.Get("http://example.com/get")
+    if err != nil {
+        return err
+    }
+    if res.StatusCode != http.StatusOK {
+        return errors.New("expected status 200")
+    }
+    // Do something with the body
+    return nil
+})
+if err != nil {
+    panic(err)
+}
+```
+
+Backoff functions provided
+
+* `retry.Attempts(10, time.Millisecond*10)` retries up to `10` attempts
+* `retry.Interval(time.Millisecond*10)` retries at an interval indefinitely or until context is cancelled
+* `retry.ExponentialBackoff{ Min: time.Millisecond, Max: time.Millisecond * 100, Factor: 2}` retries
+ at an exponential backoff interval. Can accept an optional `Attempts` which will limit the number of attempts
+
+
+### Retry Async
+Runs a function asynchronously and retries it until it succeeds, or the context is 
+cancelled or `Stop()` is called. This is useful in distributed programming where
+you know a remote thing will eventually succeed, but you need to keep trying until
+the remote thing succeeds, or we are told to shutdown.
+
+```go
+ctx := context.Background()
+async := retry.NewRetryAsync()
+
+backOff := &retry.ExponentialBackoff{
+    Min:      time.Millisecond,
+    Max:      time.Millisecond * 100,
+    Factor:   2,
+    Attempts: 10,
+}
+
+id := createNewEC2("my-new-server")
+
+async.Async(id, ctx, backOff, func(ctx context.Context, i int) error {
+    // Waits for a new EC2 instance to be created then updates the config and exits
+    if err := updateInstance(id, mySettings); err != nil {
+        return err
+    }
+    return nil
+})
+// Wait for all the asyncs to complete
+async.Wait()
+```

collections/expire_cache_test.go

@@ -0,0 +1,33 @@
+package collections_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/mailgun/holster/v3/collections"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewExpireCache(t *testing.T) {
+	ec := collections.NewExpireCache(time.Millisecond * 100)
+
+	ec.Add("one", "one")
+	time.Sleep(time.Millisecond * 100)
+
+	var runs int
+	ec.Each(1, func(key interface{}, value interface{}) error {
+		assert.Equal(t, key, "one")
+		assert.Equal(t, value, "one")
+		runs++
+		return nil
+	})
+	assert.Equal(t, runs, 1)
+
+	// Should NOT be in the cache
+	time.Sleep(time.Millisecond * 100)
+	ec.Each(1, func(key interface{}, value interface{}) error {
+		runs++
+		return nil
+	})
+	assert.Equal(t, runs, 1)
+}

errors/go113.go

@@ -0,0 +1,38 @@
+// +build go1.13
+
+package errors
+
+import (
+	stderrors "errors"
+)
+
+// Is reports whether any error in err's chain matches target.
+//
+// The chain consists of err itself followed by the sequence of errors obtained by
+// repeatedly calling Unwrap.
+//
+// An error is considered to match a target if it is equal to that target or if
+// it implements a method Is(error) bool such that Is(target) returns true.
+func Is(err, target error) bool { return stderrors.Is(err, target) }
+
+// As finds the first error in err's chain that matches target, and if so, sets
+// target to that error value and returns true.
+//
+// The chain consists of err itself followed by the sequence of errors obtained by
+// repeatedly calling Unwrap.
+//
+// An error matches target if the error's concrete value is assignable to the value
+// pointed to by target, or if the error has a method As(interface{}) bool such that
+// As(target) returns true. In the latter case, the As method is responsible for
+// setting target.
+//
+// As will panic if target is not a non-nil pointer to either a type that implements
+// error, or to any interface type. As returns false if err is nil.
+func As(err error, target interface{}) bool { return stderrors.As(err, target) }
+
+// Unwrap returns the result of calling the Unwrap method on err, if err's
+// type contains an Unwrap method returning error.
+// Otherwise, Unwrap returns nil.
+func Unwrap(err error) error {
+	return stderrors.Unwrap(err)
+}

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.10.0 // indirect
 	github.com/jonboulle/clockwork v0.1.0 // indirect
-	github.com/pkg/errors v0.8.1
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.1.0 // indirect
 	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
 	github.com/sirupsen/logrus v1.4.2

go.sum

@@ -89,6 +89,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=

retry/backoff.go

@@ -0,0 +1,103 @@
+package retry
+
+import (
+	"math"
+	"sync/atomic"
+	"time"
+)
+
+type BackOff interface {
+	New() BackOff
+	Next() (time.Duration, bool)
+	NumRetries() int
+	Reset()
+}
+
+func Interval(t time.Duration) *ConstBackOff {
+	return &ConstBackOff{Interval: t}
+}
+
+// Retry indefinitely sleeping for `interval` between each retry
+type ConstBackOff struct {
+	Interval time.Duration
+	retries  int64
+}
+
+func (b *ConstBackOff) NumRetries() int { return int(atomic.LoadInt64(&b.retries)) }
+func (b *ConstBackOff) Reset()          {}
+func (b *ConstBackOff) Next() (time.Duration, bool) {
+	atomic.AddInt64(&b.retries, 1)
+	return b.Interval, true
+}
+func (b *ConstBackOff) New() BackOff {
+	return &ConstBackOff{
+		retries:  atomic.LoadInt64(&b.retries),
+		Interval: b.Interval,
+	}
+}
+
+func Attempts(a int, t time.Duration) *AttemptsBackOff {
+	return &AttemptsBackOff{Interval: t, Attempts: int64(a)}
+}
+
+// Retry for `attempts` number of retries sleeping for `interval` between each retry
+type AttemptsBackOff struct {
+	Interval time.Duration
+	Attempts int64
+	retries  int64
+}
+
+func (b *AttemptsBackOff) NumRetries() int { return int(atomic.LoadInt64(&b.retries)) }
+func (b *AttemptsBackOff) Reset()          { atomic.StoreInt64(&b.retries, 0) }
+func (b *AttemptsBackOff) Next() (time.Duration, bool) {
+	retries := atomic.AddInt64(&b.retries, 1)
+	if retries < b.Attempts {
+		return b.Interval, true
+	}
+	return b.Interval, false
+}
+func (b *AttemptsBackOff) New() BackOff {
+	return &AttemptsBackOff{
+		retries:  atomic.LoadInt64(&b.retries),
+		Interval: b.Interval,
+		Attempts: b.Attempts,
+	}
+}
+
+type ExponentialBackOff struct {
+	Min, Max time.Duration
+	Factor   float64
+	Attempts int64
+	retries  int64
+}
+
+func (b *ExponentialBackOff) NumRetries() int { return int(atomic.LoadInt64(&b.retries)) }
+func (b *ExponentialBackOff) Reset()          { atomic.StoreInt64(&b.retries, 0) }
+func (b *ExponentialBackOff) Next() (time.Duration, bool) {
+	retries := atomic.AddInt64(&b.retries, 1)
+	interval := b.nextInterval(retries)
+	if b.Attempts != 0 && retries > b.Attempts {
+		return interval, false
+	}
+	return interval, true
+}
+func (b *ExponentialBackOff) New() BackOff {
+	return &ExponentialBackOff{
+		retries:  atomic.LoadInt64(&b.retries),
+		Attempts: b.Attempts,
+		Factor:   b.Factor,
+		Min:      b.Min,
+		Max:      b.Max,
+	}
+}
+
+func (b *ExponentialBackOff) nextInterval(retries int64) time.Duration {
+	d := time.Duration(float64(b.Min) * math.Pow(b.Factor, float64(retries)))
+	if d > b.Max {
+		return b.Max
+	}
+	if d < b.Min {
+		return b.Min
+	}
+	return d
+}