diff --git a/ChangeLog b/ChangeLog index c2fecb40..2d2d681b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +Version 11.6.1 +-------------- + * Randomly offset retentionTime and expiryTime for each job so the + goroutines don't all wake up at the same time. This is a bit of a hack, + a proper fix will be raised soon. + Version 11.6.0 -------------- * Add rate-limiter to Redis client so we don't fail slowly diff --git a/VERSION b/VERSION index 146d5de7..1a31377d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -11.6.0 +11.6.1 diff --git a/mettle/api/api.go b/mettle/api/api.go index 9731b125..cf75673d 100644 --- a/mettle/api/api.go +++ b/mettle/api/api.go @@ -4,6 +4,7 @@ package api import ( "context" "fmt" + "math/rand" "net" "strings" "sync" @@ -563,7 +564,7 @@ func (s *server) process(msg *pubsub.Message) { // deleteJob waits for a period then removes the given job from memory. func (s *server) deleteJob(hash string, j *job) { - time.Sleep(retentionTime) + time.Sleep(retentionTime + time.Duration(rand.Int63n(int64(retentionTime)))) s.mutex.Lock() defer s.mutex.Unlock() // Check the action hasn't been replaced since deleteJob was called @@ -576,11 +577,11 @@ func (s *server) deleteJob(hash string, j *job) { // expireJob expires an action that hasn't progressed. func (s *server) expireJob(hash string) { - time.Sleep(expiryTime) + time.Sleep(expiryTime + time.Duration(rand.Int63n(int64(expiryTime)))) if s.maybeExpireJob(hash, false) { return } - time.Sleep(expiryTime) + time.Sleep(expiryTime + time.Duration(rand.Int63n(int64(expiryTime)))) s.maybeExpireJob(hash, true) }