Randomly offset retention and expiration time. (thought-machine#270)

* Randomly offset retention and expiration time.

ChangeLog

@@ -1,3 +1,9 @@
+Version 11.6.1
+--------------
+    * Randomly offset retentionTime and expiryTime for each job so the
+      goroutines don't all wake up at the same time. This is a bit of a hack,
+      a proper fix will be raised soon.
+
 Version 11.6.0
 --------------
     * Add rate-limiter to Redis client so we don't fail slowly

VERSION

@@ -1 +1 @@
-11.6.0
+11.6.1

mettle/api/api.go

@@ -4,6 +4,7 @@ package api
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"net"
 	"strings"
 	"sync"
@@ -563,7 +564,7 @@ func (s *server) process(msg *pubsub.Message) {
 
 // deleteJob waits for a period then removes the given job from memory.
 func (s *server) deleteJob(hash string, j *job) {
-	time.Sleep(retentionTime)
+	time.Sleep(retentionTime + time.Duration(rand.Int63n(int64(retentionTime))))
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 	// Check the action hasn't been replaced since deleteJob was called
@@ -576,11 +577,11 @@ func (s *server) deleteJob(hash string, j *job) {
 
 // expireJob expires an action that hasn't progressed.
 func (s *server) expireJob(hash string) {
-	time.Sleep(expiryTime)
+	time.Sleep(expiryTime + time.Duration(rand.Int63n(int64(expiryTime))))
 	if s.maybeExpireJob(hash, false) {
 		return
 	}
-	time.Sleep(expiryTime)
+	time.Sleep(expiryTime + time.Duration(rand.Int63n(int64(expiryTime))))
 	s.maybeExpireJob(hash, true)
 }