From 2c9e9bd615f40817be2c3c8dd40f7fbdacf647a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Fri, 17 Jan 2025 09:41:26 +0100 Subject: [PATCH 01/18] feat(ARCO-291): Ordered send manager --- .../send_manager/ordered/send_manager.go | 357 ++++++++++++++++++ 1 file changed, 357 insertions(+) create mode 100644 internal/callbacker/send_manager/ordered/send_manager.go diff --git a/internal/callbacker/send_manager/ordered/send_manager.go b/internal/callbacker/send_manager/ordered/send_manager.go new file mode 100644 index 000000000..f99cf3ba0 --- /dev/null +++ b/internal/callbacker/send_manager/ordered/send_manager.go @@ -0,0 +1,357 @@ +package ordered + +import ( + "container/list" + "context" + "errors" + "log/slog" + "sync" + "time" + + "github.com/bitcoin-sv/arc/internal/callbacker" + //"github.com/bitcoin-sv/arc/internal/callbacker" + "github.com/bitcoin-sv/arc/internal/callbacker/store" +) + +var ( + ErrSendBatchedCallbacks = errors.New("failed to send batched callback") +) + +type SendManager struct { + url string + + // dependencies + sender callbacker.SenderI + store store.CallbackerStore + logger *slog.Logger + + expiration time.Duration + + // internal state + entriesWg sync.WaitGroup + cancelAll context.CancelFunc + ctx context.Context + + singleSendSleep time.Duration + batchSendInterval time.Duration + delayDuration time.Duration + + bufferSize int + callbackList *list.List +} + +const ( + entriesBufferSize = 10000 + batchSize = 50 + defaultBatchSendInterval = 5 * time.Second +) + +func WithBufferSize(size int) func(*SendManager) { + return func(m *SendManager) { + m.bufferSize = size + } +} + +func RunNewSendManager(url string, sender callbacker.SenderI, store store.CallbackerStore, logger *slog.Logger, sendingConfig *callbacker.SendConfig, opts ...func(*SendManager)) *SendManager { + batchSendInterval := defaultBatchSendInterval + if sendingConfig.BatchSendInterval != 0 { + batchSendInterval = sendingConfig.BatchSendInterval + } + + m := &SendManager{ + url: url, + sender: sender, + store: store, + logger: logger, + + singleSendSleep: sendingConfig.PauseAfterSingleModeSuccessfulSend, + batchSendInterval: batchSendInterval, + delayDuration: sendingConfig.DelayDuration, + expiration: sendingConfig.Expiration, + + callbackList: list.New(), + bufferSize: entriesBufferSize, + } + + for _, opt := range opts { + opt(m) + } + + ctx, cancelAll := context.WithCancel(context.Background()) + m.cancelAll = cancelAll + m.ctx = ctx + + m.StartProcessCallbackQueue() + return m +} + +func (m *SendManager) Enqueue(entry callbacker.CallbackEntry) { + if m.callbackList.Len() >= m.bufferSize { + m.storeToDB(entry) + return + } + + m.callbackList.PushBack(entry) +} + +func (m *SendManager) dequeueAll() []callbacker.CallbackEntry { + callbacks := make([]callbacker.CallbackEntry, 0, m.callbackList.Len()) + + var next *list.Element + for front := m.callbackList.Front(); front != nil; front = next { + next = front.Next() + entry, ok := front.Value.(callbacker.CallbackEntry) + if !ok { + continue + } + callbacks = append(callbacks, entry) + } + + return callbacks +} + +func (m *SendManager) sortByTimestamp() { + current := m.callbackList.Front() + if m.callbackList.Front() == nil { + return + } + for current != nil { + index := current.Next() + for index != nil { + currentTime := current.Value.(*callbacker.CallbackEntry).Data.Timestamp + indexTime := index.Value.(*callbacker.CallbackEntry).Data.Timestamp + if currentTime.Before(indexTime) { + temp := current.Value + current.Value = index.Value + index.Value = temp + } + index = index.Next() + } + current = current.Next() + } +} + +func (m *SendManager) capacityLeft() int { + return m.bufferSize - m.callbackList.Len() +} + +func (m *SendManager) StartProcessCallbackQueue() { + queueTicker := time.NewTicker(m.singleSendSleep) + batchSendTicker := time.NewTicker(m.batchSendInterval) + sortTicker := time.NewTicker(10 * time.Second) + backFillQueueTicker := time.NewTicker(10 * time.Second) + + m.entriesWg.Add(1) + go func() { + var err error + defer func() { + // read all from callback queue and store in database + data := make([]*store.CallbackData, m.callbackList.Len()) + + for i, entry := range m.dequeueAll() { + data[i] = toStoreDto(m.url, entry) + } + + err = m.store.SetMany(context.Background(), data) + if err != nil { + m.logger.Error("callback queue enqueue failed", slog.String("err", err.Error())) + } + m.entriesWg.Done() + }() + + var callbackElements []*list.Element + + mainLoop: + for { + select { + case <-m.ctx.Done(): + return + case <-sortTicker.C: + m.sortByTimestamp() + + case <-backFillQueueTicker.C: + capacityLeft := m.capacityLeft() + if capacityLeft == 0 { + continue + } + + callbacks, err := m.store.PopMany(m.ctx, m.url, capacityLeft) + if err != nil { + m.logger.Error("Failed to load callbacks", slog.String("err", err.Error())) + continue + } + + for _, callback := range callbacks { + m.Enqueue(toEntry(callback)) + } + case <-batchSendTicker.C: + if len(callbackElements) == 0 { + continue + } + + callbackBatch := make([]callbacker.CallbackEntry, len(callbackElements)) + for i, element := range callbackElements { + cb, ok := element.Value.(callbacker.CallbackEntry) + if !ok { + continue + } + callbackBatch[i] = cb + } + + success, retry := m.sendBatch(callbackBatch) + if !retry || success { + for _, callbackElement := range callbackElements { + m.callbackList.Remove(callbackElement) + } + + callbackElements = callbackElements[:0] + continue + } + + m.logger.Error("failed to send batched callbacks") + case <-queueTicker.C: + front := m.callbackList.Front() + if front == nil { + continue + } + + callbackEntry, ok := front.Value.(callbacker.CallbackEntry) + if !ok { + continue + } + + // If item is expired - dequeue without storing + if time.Until(callbackEntry.Data.Timestamp) > m.expiration { + m.logger.Warn("callback expired", slog.Time("timestamp", callbackEntry.Data.Timestamp), slog.String("hash", callbackEntry.Data.TxID), slog.String("status", callbackEntry.Data.TxStatus)) + m.callbackList.Remove(front) + continue + } + + for callbackEntry.AllowBatch { + callbackElements = append(callbackElements, front) + + callbackElement := front.Next() + if callbackElement != nil && len(callbackElements) < batchSize { + callbackEntry, ok = callbackElement.Value.(callbacker.CallbackEntry) + if !ok { + continue + } + + continue + } + + err = m.sendElementBatch(callbackElements) + if err != nil { + m.logger.Error("failed to send batched callbacks", slog.String("err", err.Error())) + } else { + callbackElements = callbackElements[:0] + } + continue mainLoop + } + + // if entry is not a batched entry, but there are items in the batch, send them first to keep the order + if len(callbackElements) > 0 { + err = m.sendElementBatch(callbackElements) + if err != nil { + m.logger.Error("failed to send batched callbacks", slog.String("err", err.Error())) + } else { + callbackElements = callbackElements[:0] + } + continue mainLoop + } + + success, retry := m.sender.Send(m.url, callbackEntry.Token, callbackEntry.Data) + if !retry || success { + m.callbackList.Remove(front) + continue + } + m.logger.Error("failed to send single callback", slog.String("url", m.url)) + } + } + }() +} + +func (m *SendManager) sendElementBatch(callbackElements []*list.Element) error { + var callbackElement *list.Element + callbackBatch := make([]callbacker.CallbackEntry, 0, len(callbackElements)) + for _, element := range callbackElements { + callback, ok := element.Value.(callbacker.CallbackEntry) + if !ok { + continue + } + callbackBatch = append(callbackBatch, callback) + } + success, retry := m.sendBatch(callbackBatch) + if !retry || success { + for _, callbackElement = range callbackElements { + m.callbackList.Remove(callbackElement) + } + + return nil + } + + return ErrSendBatchedCallbacks +} + +func (m *SendManager) sendBatch(batch []callbacker.CallbackEntry) (success, retry bool) { + token := batch[0].Token + callbacks := make([]*callbacker.Callback, len(batch)) + for i, e := range batch { + callbacks[i] = e.Data + } + + return m.sender.SendBatch(m.url, token, callbacks) +} + +func (m *SendManager) storeToDB(entry callbacker.CallbackEntry) { + callbackData := toStoreDto(m.url, entry) + err := m.store.Set(context.Background(), callbackData) + if err != nil { + m.logger.Error("Failed to set callback data", slog.String("hash", callbackData.TxID), slog.String("status", callbackData.TxStatus), slog.String("err", err.Error())) + } +} + +func toStoreDto(url string, entry callbacker.CallbackEntry) *store.CallbackData { + return &store.CallbackData{ + URL: url, + Token: entry.Token, + Timestamp: entry.Data.Timestamp, + + CompetingTxs: entry.Data.CompetingTxs, + TxID: entry.Data.TxID, + TxStatus: entry.Data.TxStatus, + ExtraInfo: entry.Data.ExtraInfo, + MerklePath: entry.Data.MerklePath, + + BlockHash: entry.Data.BlockHash, + BlockHeight: entry.Data.BlockHeight, + + AllowBatch: entry.AllowBatch, + } +} + +func toEntry(callbackData *store.CallbackData) callbacker.CallbackEntry { + return callbacker.CallbackEntry{ + Token: callbackData.Token, + Data: &callbacker.Callback{ + Timestamp: callbackData.Timestamp, + CompetingTxs: callbackData.CompetingTxs, + TxID: callbackData.TxID, + TxStatus: callbackData.TxStatus, + ExtraInfo: callbackData.ExtraInfo, + MerklePath: callbackData.MerklePath, + BlockHash: callbackData.BlockHash, + BlockHeight: callbackData.BlockHeight, + }, + AllowBatch: callbackData.AllowBatch, + } +} + +// GracefulStop On service termination, any unsent callbacks are persisted in the store, ensuring no loss of data during shutdown. +func (m *SendManager) GracefulStop() { + if m.cancelAll != nil { + m.cancelAll() + } + + m.entriesWg.Wait() +} From e8a973cbdd8be28387574834aebd30a979d868e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Fri, 17 Jan 2025 09:42:31 +0100 Subject: [PATCH 02/18] feat(ARCO-291): Remove batched callbacks temporarily --- .../send_manager/ordered/send_manager.go | 62 ------------------- 1 file changed, 62 deletions(-) diff --git a/internal/callbacker/send_manager/ordered/send_manager.go b/internal/callbacker/send_manager/ordered/send_manager.go index f99cf3ba0..0c8a2d6d4 100644 --- a/internal/callbacker/send_manager/ordered/send_manager.go +++ b/internal/callbacker/send_manager/ordered/send_manager.go @@ -3,20 +3,14 @@ package ordered import ( "container/list" "context" - "errors" "log/slog" "sync" "time" "github.com/bitcoin-sv/arc/internal/callbacker" - //"github.com/bitcoin-sv/arc/internal/callbacker" "github.com/bitcoin-sv/arc/internal/callbacker/store" ) -var ( - ErrSendBatchedCallbacks = errors.New("failed to send batched callback") -) - type SendManager struct { url string @@ -161,7 +155,6 @@ func (m *SendManager) StartProcessCallbackQueue() { var callbackElements []*list.Element - mainLoop: for { select { case <-m.ctx.Done(): @@ -227,39 +220,6 @@ func (m *SendManager) StartProcessCallbackQueue() { continue } - for callbackEntry.AllowBatch { - callbackElements = append(callbackElements, front) - - callbackElement := front.Next() - if callbackElement != nil && len(callbackElements) < batchSize { - callbackEntry, ok = callbackElement.Value.(callbacker.CallbackEntry) - if !ok { - continue - } - - continue - } - - err = m.sendElementBatch(callbackElements) - if err != nil { - m.logger.Error("failed to send batched callbacks", slog.String("err", err.Error())) - } else { - callbackElements = callbackElements[:0] - } - continue mainLoop - } - - // if entry is not a batched entry, but there are items in the batch, send them first to keep the order - if len(callbackElements) > 0 { - err = m.sendElementBatch(callbackElements) - if err != nil { - m.logger.Error("failed to send batched callbacks", slog.String("err", err.Error())) - } else { - callbackElements = callbackElements[:0] - } - continue mainLoop - } - success, retry := m.sender.Send(m.url, callbackEntry.Token, callbackEntry.Data) if !retry || success { m.callbackList.Remove(front) @@ -271,28 +231,6 @@ func (m *SendManager) StartProcessCallbackQueue() { }() } -func (m *SendManager) sendElementBatch(callbackElements []*list.Element) error { - var callbackElement *list.Element - callbackBatch := make([]callbacker.CallbackEntry, 0, len(callbackElements)) - for _, element := range callbackElements { - callback, ok := element.Value.(callbacker.CallbackEntry) - if !ok { - continue - } - callbackBatch = append(callbackBatch, callback) - } - success, retry := m.sendBatch(callbackBatch) - if !retry || success { - for _, callbackElement = range callbackElements { - m.callbackList.Remove(callbackElement) - } - - return nil - } - - return ErrSendBatchedCallbacks -} - func (m *SendManager) sendBatch(batch []callbacker.CallbackEntry) (success, retry bool) { token := batch[0].Token callbacks := make([]*callbacker.Callback, len(batch)) From 661de208d2c97536c448d533186c1dfefe939c23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Fri, 17 Jan 2025 11:09:52 +0100 Subject: [PATCH 03/18] feat(ARCO-291): Unit tests --- .../send_manager/ordered/send_manager.go | 259 +++++++++--------- .../send_manager/ordered/send_manager_test.go | 120 ++++++++ 2 files changed, 247 insertions(+), 132 deletions(-) create mode 100644 internal/callbacker/send_manager/ordered/send_manager_test.go diff --git a/internal/callbacker/send_manager/ordered/send_manager.go b/internal/callbacker/send_manager/ordered/send_manager.go index 0c8a2d6d4..bb8db5dc8 100644 --- a/internal/callbacker/send_manager/ordered/send_manager.go +++ b/internal/callbacker/send_manager/ordered/send_manager.go @@ -11,12 +11,18 @@ import ( "github.com/bitcoin-sv/arc/internal/callbacker/store" ) +type SendManagerStore interface { + Set(ctx context.Context, dto *store.CallbackData) error + SetMany(ctx context.Context, data []*store.CallbackData) error + PopMany(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) +} + type SendManager struct { url string // dependencies sender callbacker.SenderI - store store.CallbackerStore + store SendManagerStore logger *slog.Logger expiration time.Duration @@ -26,31 +32,52 @@ type SendManager struct { cancelAll context.CancelFunc ctx context.Context - singleSendSleep time.Duration - batchSendInterval time.Duration - delayDuration time.Duration + singleSendInterval time.Duration + //batchSendInterval time.Duration + //delayDuration time.Duration bufferSize int callbackList *list.List + + now func() time.Time } const ( - entriesBufferSize = 10000 - batchSize = 50 - defaultBatchSendInterval = 5 * time.Second + entriesBufferSize = 10000 + batchSendIntervalDefault = 5 * time.Second + singleSendIntervalDefault = 5 * time.Second + expirationDefault = 24 * time.Hour ) +func WithNow(nowFunc func() time.Time) func(*SendManager) { + return func(m *SendManager) { + m.now = nowFunc + } +} + func WithBufferSize(size int) func(*SendManager) { return func(m *SendManager) { m.bufferSize = size } } -func RunNewSendManager(url string, sender callbacker.SenderI, store store.CallbackerStore, logger *slog.Logger, sendingConfig *callbacker.SendConfig, opts ...func(*SendManager)) *SendManager { - batchSendInterval := defaultBatchSendInterval - if sendingConfig.BatchSendInterval != 0 { - batchSendInterval = sendingConfig.BatchSendInterval +func WithSingleSendInterval(d time.Duration) func(*SendManager) { + return func(m *SendManager) { + m.singleSendInterval = d + } +} + +func WithExpiration(d time.Duration) func(*SendManager) { + return func(m *SendManager) { + m.expiration = d } +} + +func New(url string, sender callbacker.SenderI, store SendManagerStore, logger *slog.Logger, opts ...func(*SendManager)) *SendManager { + //batchSendInterval := defaultBatchSendInterval + //if sendingConfig.BatchSendInterval != 0 { + // batchSendInterval = sendingConfig.BatchSendInterval + //} m := &SendManager{ url: url, @@ -58,10 +85,10 @@ func RunNewSendManager(url string, sender callbacker.SenderI, store store.Callba store: store, logger: logger, - singleSendSleep: sendingConfig.PauseAfterSingleModeSuccessfulSend, - batchSendInterval: batchSendInterval, - delayDuration: sendingConfig.DelayDuration, - expiration: sendingConfig.Expiration, + singleSendInterval: singleSendIntervalDefault, + //batchSendInterval: batchSendInterval, + //delayDuration: sendingConfig.DelayDuration, + expiration: expirationDefault, callbackList: list.New(), bufferSize: entriesBufferSize, @@ -75,7 +102,6 @@ func RunNewSendManager(url string, sender callbacker.SenderI, store store.Callba m.cancelAll = cancelAll m.ctx = ctx - m.StartProcessCallbackQueue() return m } @@ -88,52 +114,35 @@ func (m *SendManager) Enqueue(entry callbacker.CallbackEntry) { m.callbackList.PushBack(entry) } -func (m *SendManager) dequeueAll() []callbacker.CallbackEntry { - callbacks := make([]callbacker.CallbackEntry, 0, m.callbackList.Len()) - - var next *list.Element - for front := m.callbackList.Front(); front != nil; front = next { - next = front.Next() - entry, ok := front.Value.(callbacker.CallbackEntry) - if !ok { - continue - } - callbacks = append(callbacks, entry) - } - - return callbacks +//func (m *SendManager) sortByTimestamp() { +// current := m.callbackList.Front() +// if m.callbackList.Front() == nil { +// return +// } +// for current != nil { +// index := current.Next() +// for index != nil { +// currentTime := current.Value.(*callbacker.CallbackEntry).Data.Timestamp +// indexTime := index.Value.(*callbacker.CallbackEntry).Data.Timestamp +// if currentTime.Before(indexTime) { +// temp := current.Value +// current.Value = index.Value +// index.Value = temp +// } +// index = index.Next() +// } +// current = current.Next() +// } +//} + +func (m *SendManager) CallbacksQueued() int { + return m.callbackList.Len() } -func (m *SendManager) sortByTimestamp() { - current := m.callbackList.Front() - if m.callbackList.Front() == nil { - return - } - for current != nil { - index := current.Next() - for index != nil { - currentTime := current.Value.(*callbacker.CallbackEntry).Data.Timestamp - indexTime := index.Value.(*callbacker.CallbackEntry).Data.Timestamp - if currentTime.Before(indexTime) { - temp := current.Value - current.Value = index.Value - index.Value = temp - } - index = index.Next() - } - current = current.Next() - } -} - -func (m *SendManager) capacityLeft() int { - return m.bufferSize - m.callbackList.Len() -} - -func (m *SendManager) StartProcessCallbackQueue() { - queueTicker := time.NewTicker(m.singleSendSleep) - batchSendTicker := time.NewTicker(m.batchSendInterval) - sortTicker := time.NewTicker(10 * time.Second) - backFillQueueTicker := time.NewTicker(10 * time.Second) +func (m *SendManager) Start() { + queueTicker := time.NewTicker(m.singleSendInterval) + //sortTicker := time.NewTicker(10 * time.Second) + //backFillQueueTicker := time.NewTicker(10 * time.Second) m.entriesWg.Add(1) go func() { @@ -146,62 +155,39 @@ func (m *SendManager) StartProcessCallbackQueue() { data[i] = toStoreDto(m.url, entry) } - err = m.store.SetMany(context.Background(), data) - if err != nil { - m.logger.Error("callback queue enqueue failed", slog.String("err", err.Error())) + if len(data) > 0 { + err = m.store.SetMany(context.Background(), data) + if err != nil { + m.logger.Error("Failed to set remaining callbacks from queue", slog.String("err", err.Error())) + } } + m.entriesWg.Done() }() - var callbackElements []*list.Element - for { select { case <-m.ctx.Done(): return - case <-sortTicker.C: - m.sortByTimestamp() - - case <-backFillQueueTicker.C: - capacityLeft := m.capacityLeft() - if capacityLeft == 0 { - continue - } - - callbacks, err := m.store.PopMany(m.ctx, m.url, capacityLeft) - if err != nil { - m.logger.Error("Failed to load callbacks", slog.String("err", err.Error())) - continue - } - - for _, callback := range callbacks { - m.Enqueue(toEntry(callback)) - } - case <-batchSendTicker.C: - if len(callbackElements) == 0 { - continue - } - - callbackBatch := make([]callbacker.CallbackEntry, len(callbackElements)) - for i, element := range callbackElements { - cb, ok := element.Value.(callbacker.CallbackEntry) - if !ok { - continue - } - callbackBatch[i] = cb - } - - success, retry := m.sendBatch(callbackBatch) - if !retry || success { - for _, callbackElement := range callbackElements { - m.callbackList.Remove(callbackElement) - } - - callbackElements = callbackElements[:0] - continue - } + //case <-sortTicker.C: + // m.sortByTimestamp() + + //case <-backFillQueueTicker.C: + // capacityLeft := m.bufferSize - m.callbackList.Len() + // if capacityLeft == 0 { + // continue + // } + // + // callbacks, err := m.store.PopMany(m.ctx, m.url, capacityLeft) + // if err != nil { + // m.logger.Error("Failed to load callbacks", slog.String("err", err.Error())) + // continue + // } + // + // for _, callback := range callbacks { + // m.Enqueue(toEntry(callback)) + // } - m.logger.Error("failed to send batched callbacks") case <-queueTicker.C: front := m.callbackList.Front() if front == nil { @@ -214,7 +200,7 @@ func (m *SendManager) StartProcessCallbackQueue() { } // If item is expired - dequeue without storing - if time.Until(callbackEntry.Data.Timestamp) > m.expiration { + if m.now().Sub(callbackEntry.Data.Timestamp) > m.expiration { m.logger.Warn("callback expired", slog.Time("timestamp", callbackEntry.Data.Timestamp), slog.String("hash", callbackEntry.Data.TxID), slog.String("status", callbackEntry.Data.TxStatus)) m.callbackList.Remove(front) continue @@ -231,16 +217,6 @@ func (m *SendManager) StartProcessCallbackQueue() { }() } -func (m *SendManager) sendBatch(batch []callbacker.CallbackEntry) (success, retry bool) { - token := batch[0].Token - callbacks := make([]*callbacker.Callback, len(batch)) - for i, e := range batch { - callbacks[i] = e.Data - } - - return m.sender.SendBatch(m.url, token, callbacks) -} - func (m *SendManager) storeToDB(entry callbacker.CallbackEntry) { callbackData := toStoreDto(m.url, entry) err := m.store.Set(context.Background(), callbackData) @@ -268,21 +244,40 @@ func toStoreDto(url string, entry callbacker.CallbackEntry) *store.CallbackData } } -func toEntry(callbackData *store.CallbackData) callbacker.CallbackEntry { - return callbacker.CallbackEntry{ - Token: callbackData.Token, - Data: &callbacker.Callback{ - Timestamp: callbackData.Timestamp, - CompetingTxs: callbackData.CompetingTxs, - TxID: callbackData.TxID, - TxStatus: callbackData.TxStatus, - ExtraInfo: callbackData.ExtraInfo, - MerklePath: callbackData.MerklePath, - BlockHash: callbackData.BlockHash, - BlockHeight: callbackData.BlockHeight, - }, - AllowBatch: callbackData.AllowBatch, +// +//func toEntry(callbackData *store.CallbackData) callbacker.CallbackEntry { +// return callbacker.CallbackEntry{ +// Token: callbackData.Token, +// Data: &callbacker.Callback{ +// Timestamp: callbackData.Timestamp, +// CompetingTxs: callbackData.CompetingTxs, +// TxID: callbackData.TxID, +// TxStatus: callbackData.TxStatus, +// ExtraInfo: callbackData.ExtraInfo, +// MerklePath: callbackData.MerklePath, +// BlockHash: callbackData.BlockHash, +// BlockHeight: callbackData.BlockHeight, +// }, +// AllowBatch: callbackData.AllowBatch, +// } +//} + +func (m *SendManager) dequeueAll() []callbacker.CallbackEntry { + callbacks := make([]callbacker.CallbackEntry, 0, m.callbackList.Len()) + + var next *list.Element + for front := m.callbackList.Front(); front != nil; front = next { + next = front.Next() + entry, ok := front.Value.(callbacker.CallbackEntry) + if !ok { + continue + } + callbacks = append(callbacks, entry) + + m.callbackList.Remove(front) } + + return callbacks } // GracefulStop On service termination, any unsent callbacks are persisted in the store, ensuring no loss of data during shutdown. diff --git a/internal/callbacker/send_manager/ordered/send_manager_test.go b/internal/callbacker/send_manager/ordered/send_manager_test.go new file mode 100644 index 000000000..33c558201 --- /dev/null +++ b/internal/callbacker/send_manager/ordered/send_manager_test.go @@ -0,0 +1,120 @@ +package ordered_test + +import ( + "context" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/bitcoin-sv/arc/internal/callbacker" + "github.com/bitcoin-sv/arc/internal/callbacker/mocks" + "github.com/bitcoin-sv/arc/internal/callbacker/send_manager/ordered" + "github.com/bitcoin-sv/arc/internal/callbacker/store" +) + +func TestSendManagerStart(t *testing.T) { + tcs := []struct { + name string + callbacksEnqueued int + singleSendInterval time.Duration + callbackTimestamp time.Time + + expectedCallbacksEnqueued int + expectedSetManyCalls int + expectedSetCalls int + expectedSendCalls int + }{ + { + name: "enqueue 10 callbacks - 10ms interval", + callbacksEnqueued: 10, + singleSendInterval: 10 * time.Millisecond, + callbackTimestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC), + + expectedCallbacksEnqueued: 10, + expectedSetManyCalls: 0, + expectedSetCalls: 0, + expectedSendCalls: 10, + }, + { + name: "enqueue 10 callbacks - 100ms interval - store remaining at graceful stop", + callbacksEnqueued: 10, + singleSendInterval: 100 * time.Millisecond, + callbackTimestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC), + + expectedCallbacksEnqueued: 10, + expectedSetManyCalls: 1, + expectedSetCalls: 0, + expectedSendCalls: 1, + }, + { + name: "enqueue 10 callbacks - expired", + callbacksEnqueued: 10, + singleSendInterval: 10 * time.Millisecond, + callbackTimestamp: time.Date(2025, 1, 9, 12, 0, 0, 0, time.UTC), + + expectedCallbacksEnqueued: 10, + expectedSetManyCalls: 0, + expectedSetCalls: 0, + expectedSendCalls: 0, + }, + { + name: "enqueue 15 callbacks - buffer size reached", + callbacksEnqueued: 15, + singleSendInterval: 10 * time.Millisecond, + callbackTimestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC), + + expectedCallbacksEnqueued: 10, + expectedSetManyCalls: 0, + expectedSetCalls: 5, + expectedSendCalls: 10, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + // given + senderMock := &mocks.SenderIMock{ + SendFunc: func(_, _ string, _ *callbacker.Callback) (bool, bool) { return true, false }, + SendBatchFunc: func(_, _ string, _ []*callbacker.Callback) (bool, bool) { return true, false }, + } + + storeMock := &mocks.CallbackerStoreMock{ + SetManyFunc: func(_ context.Context, data []*store.CallbackData) error { + return nil + }, + SetFunc: func(_ context.Context, data *store.CallbackData) error { + return nil + }, + } + + sut := ordered.New("https://abc.com", senderMock, storeMock, slog.Default(), + ordered.WithBufferSize(10), + ordered.WithNow(func() time.Time { + return time.Date(2025, 1, 10, 12, 0, 0, 0, time.UTC) + }), + ordered.WithSingleSendInterval(tc.singleSendInterval), + ordered.WithExpiration(time.Hour), + ) + + // add callbacks before starting the manager to queue them + for range tc.callbacksEnqueued { + sut.Enqueue(callbacker.CallbackEntry{Data: &callbacker.Callback{ + Timestamp: tc.callbackTimestamp, + }}) + } + require.Equal(t, tc.expectedCallbacksEnqueued, sut.CallbacksQueued()) + + sut.Start() + + time.Sleep(150 * time.Millisecond) + sut.GracefulStop() + + require.Equal(t, 0, sut.CallbacksQueued()) + require.Equal(t, tc.expectedSetManyCalls, len(storeMock.SetManyCalls())) + require.Equal(t, tc.expectedSetCalls, len(storeMock.SetCalls())) + require.Equal(t, tc.expectedSendCalls, len(senderMock.SendCalls())) + }) + } +} From 70bb588b5fdbffa56990807ed06aaa53c05e2dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Fri, 17 Jan 2025 13:27:28 +0100 Subject: [PATCH 04/18] feat(ARCO-291): Backfill callbacks --- .../send_manager/ordered/mocks/sender_mock.go | 144 +++++++++++++ .../send_manager/ordered/mocks/store_mock.go | 189 ++++++++++++++++++ .../send_manager/ordered/ordered_mocks.go | 5 + .../send_manager/ordered/send_manager.go | 152 +++++++------- .../send_manager/ordered/send_manager_test.go | 42 +++- 5 files changed, 456 insertions(+), 76 deletions(-) create mode 100644 internal/callbacker/send_manager/ordered/mocks/sender_mock.go create mode 100644 internal/callbacker/send_manager/ordered/mocks/store_mock.go create mode 100644 internal/callbacker/send_manager/ordered/ordered_mocks.go diff --git a/internal/callbacker/send_manager/ordered/mocks/sender_mock.go b/internal/callbacker/send_manager/ordered/mocks/sender_mock.go new file mode 100644 index 000000000..b3401d262 --- /dev/null +++ b/internal/callbacker/send_manager/ordered/mocks/sender_mock.go @@ -0,0 +1,144 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mocks + +import ( + "github.com/bitcoin-sv/arc/internal/callbacker" + "github.com/bitcoin-sv/arc/internal/callbacker/send_manager/ordered" + "sync" +) + +// Ensure, that SenderMock does implement ordered.Sender. +// If this is not the case, regenerate this file with moq. +var _ ordered.Sender = &SenderMock{} + +// SenderMock is a mock implementation of ordered.Sender. +// +// func TestSomethingThatUsesSender(t *testing.T) { +// +// // make and configure a mocked ordered.Sender +// mockedSender := &SenderMock{ +// SendFunc: func(url string, token string, callback *callbacker.Callback) (bool, bool) { +// panic("mock out the Send method") +// }, +// SendBatchFunc: func(url string, token string, callbacks []*callbacker.Callback) (bool, bool) { +// panic("mock out the SendBatch method") +// }, +// } +// +// // use mockedSender in code that requires ordered.Sender +// // and then make assertions. +// +// } +type SenderMock struct { + // SendFunc mocks the Send method. + SendFunc func(url string, token string, callback *callbacker.Callback) (bool, bool) + + // SendBatchFunc mocks the SendBatch method. + SendBatchFunc func(url string, token string, callbacks []*callbacker.Callback) (bool, bool) + + // calls tracks calls to the methods. + calls struct { + // Send holds details about calls to the Send method. + Send []struct { + // URL is the url argument value. + URL string + // Token is the token argument value. + Token string + // Callback is the callback argument value. + Callback *callbacker.Callback + } + // SendBatch holds details about calls to the SendBatch method. + SendBatch []struct { + // URL is the url argument value. + URL string + // Token is the token argument value. + Token string + // Callbacks is the callbacks argument value. + Callbacks []*callbacker.Callback + } + } + lockSend sync.RWMutex + lockSendBatch sync.RWMutex +} + +// Send calls SendFunc. +func (mock *SenderMock) Send(url string, token string, callback *callbacker.Callback) (bool, bool) { + if mock.SendFunc == nil { + panic("SenderMock.SendFunc: method is nil but Sender.Send was just called") + } + callInfo := struct { + URL string + Token string + Callback *callbacker.Callback + }{ + URL: url, + Token: token, + Callback: callback, + } + mock.lockSend.Lock() + mock.calls.Send = append(mock.calls.Send, callInfo) + mock.lockSend.Unlock() + return mock.SendFunc(url, token, callback) +} + +// SendCalls gets all the calls that were made to Send. +// Check the length with: +// +// len(mockedSender.SendCalls()) +func (mock *SenderMock) SendCalls() []struct { + URL string + Token string + Callback *callbacker.Callback +} { + var calls []struct { + URL string + Token string + Callback *callbacker.Callback + } + mock.lockSend.RLock() + calls = mock.calls.Send + mock.lockSend.RUnlock() + return calls +} + +// SendBatch calls SendBatchFunc. +func (mock *SenderMock) SendBatch(url string, token string, callbacks []*callbacker.Callback) (bool, bool) { + if mock.SendBatchFunc == nil { + panic("SenderMock.SendBatchFunc: method is nil but Sender.SendBatch was just called") + } + callInfo := struct { + URL string + Token string + Callbacks []*callbacker.Callback + }{ + URL: url, + Token: token, + Callbacks: callbacks, + } + mock.lockSendBatch.Lock() + mock.calls.SendBatch = append(mock.calls.SendBatch, callInfo) + mock.lockSendBatch.Unlock() + return mock.SendBatchFunc(url, token, callbacks) +} + +// SendBatchCalls gets all the calls that were made to SendBatch. +// Check the length with: +// +// len(mockedSender.SendBatchCalls()) +func (mock *SenderMock) SendBatchCalls() []struct { + URL string + Token string + Callbacks []*callbacker.Callback +} { + var calls []struct { + URL string + Token string + Callbacks []*callbacker.Callback + } + mock.lockSendBatch.RLock() + calls = mock.calls.SendBatch + mock.lockSendBatch.RUnlock() + return calls +} diff --git a/internal/callbacker/send_manager/ordered/mocks/store_mock.go b/internal/callbacker/send_manager/ordered/mocks/store_mock.go new file mode 100644 index 000000000..d3695e3bb --- /dev/null +++ b/internal/callbacker/send_manager/ordered/mocks/store_mock.go @@ -0,0 +1,189 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mocks + +import ( + "context" + "github.com/bitcoin-sv/arc/internal/callbacker/send_manager/ordered" + "github.com/bitcoin-sv/arc/internal/callbacker/store" + "sync" +) + +// Ensure, that SendManagerStoreMock does implement ordered.SendManagerStore. +// If this is not the case, regenerate this file with moq. +var _ ordered.SendManagerStore = &SendManagerStoreMock{} + +// SendManagerStoreMock is a mock implementation of ordered.SendManagerStore. +// +// func TestSomethingThatUsesSendManagerStore(t *testing.T) { +// +// // make and configure a mocked ordered.SendManagerStore +// mockedSendManagerStore := &SendManagerStoreMock{ +// GetAndDeleteFunc: func(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) { +// panic("mock out the GetAndDelete method") +// }, +// SetFunc: func(ctx context.Context, dto *store.CallbackData) error { +// panic("mock out the Set method") +// }, +// SetManyFunc: func(ctx context.Context, data []*store.CallbackData) error { +// panic("mock out the SetMany method") +// }, +// } +// +// // use mockedSendManagerStore in code that requires ordered.SendManagerStore +// // and then make assertions. +// +// } +type SendManagerStoreMock struct { + // GetAndDeleteFunc mocks the GetAndDelete method. + GetAndDeleteFunc func(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) + + // SetFunc mocks the Set method. + SetFunc func(ctx context.Context, dto *store.CallbackData) error + + // SetManyFunc mocks the SetMany method. + SetManyFunc func(ctx context.Context, data []*store.CallbackData) error + + // calls tracks calls to the methods. + calls struct { + // GetAndDelete holds details about calls to the GetAndDelete method. + GetAndDelete []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // URL is the url argument value. + URL string + // Limit is the limit argument value. + Limit int + } + // Set holds details about calls to the Set method. + Set []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Dto is the dto argument value. + Dto *store.CallbackData + } + // SetMany holds details about calls to the SetMany method. + SetMany []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Data is the data argument value. + Data []*store.CallbackData + } + } + lockGetAndDelete sync.RWMutex + lockSet sync.RWMutex + lockSetMany sync.RWMutex +} + +// GetAndDelete calls GetAndDeleteFunc. +func (mock *SendManagerStoreMock) GetAndDelete(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) { + if mock.GetAndDeleteFunc == nil { + panic("SendManagerStoreMock.GetAndDeleteFunc: method is nil but SendManagerStore.GetAndDelete was just called") + } + callInfo := struct { + Ctx context.Context + URL string + Limit int + }{ + Ctx: ctx, + URL: url, + Limit: limit, + } + mock.lockGetAndDelete.Lock() + mock.calls.GetAndDelete = append(mock.calls.GetAndDelete, callInfo) + mock.lockGetAndDelete.Unlock() + return mock.GetAndDeleteFunc(ctx, url, limit) +} + +// GetAndDeleteCalls gets all the calls that were made to GetAndDelete. +// Check the length with: +// +// len(mockedSendManagerStore.GetAndDeleteCalls()) +func (mock *SendManagerStoreMock) GetAndDeleteCalls() []struct { + Ctx context.Context + URL string + Limit int +} { + var calls []struct { + Ctx context.Context + URL string + Limit int + } + mock.lockGetAndDelete.RLock() + calls = mock.calls.GetAndDelete + mock.lockGetAndDelete.RUnlock() + return calls +} + +// Set calls SetFunc. +func (mock *SendManagerStoreMock) Set(ctx context.Context, dto *store.CallbackData) error { + if mock.SetFunc == nil { + panic("SendManagerStoreMock.SetFunc: method is nil but SendManagerStore.Set was just called") + } + callInfo := struct { + Ctx context.Context + Dto *store.CallbackData + }{ + Ctx: ctx, + Dto: dto, + } + mock.lockSet.Lock() + mock.calls.Set = append(mock.calls.Set, callInfo) + mock.lockSet.Unlock() + return mock.SetFunc(ctx, dto) +} + +// SetCalls gets all the calls that were made to Set. +// Check the length with: +// +// len(mockedSendManagerStore.SetCalls()) +func (mock *SendManagerStoreMock) SetCalls() []struct { + Ctx context.Context + Dto *store.CallbackData +} { + var calls []struct { + Ctx context.Context + Dto *store.CallbackData + } + mock.lockSet.RLock() + calls = mock.calls.Set + mock.lockSet.RUnlock() + return calls +} + +// SetMany calls SetManyFunc. +func (mock *SendManagerStoreMock) SetMany(ctx context.Context, data []*store.CallbackData) error { + if mock.SetManyFunc == nil { + panic("SendManagerStoreMock.SetManyFunc: method is nil but SendManagerStore.SetMany was just called") + } + callInfo := struct { + Ctx context.Context + Data []*store.CallbackData + }{ + Ctx: ctx, + Data: data, + } + mock.lockSetMany.Lock() + mock.calls.SetMany = append(mock.calls.SetMany, callInfo) + mock.lockSetMany.Unlock() + return mock.SetManyFunc(ctx, data) +} + +// SetManyCalls gets all the calls that were made to SetMany. +// Check the length with: +// +// len(mockedSendManagerStore.SetManyCalls()) +func (mock *SendManagerStoreMock) SetManyCalls() []struct { + Ctx context.Context + Data []*store.CallbackData +} { + var calls []struct { + Ctx context.Context + Data []*store.CallbackData + } + mock.lockSetMany.RLock() + calls = mock.calls.SetMany + mock.lockSetMany.RUnlock() + return calls +} diff --git a/internal/callbacker/send_manager/ordered/ordered_mocks.go b/internal/callbacker/send_manager/ordered/ordered_mocks.go new file mode 100644 index 000000000..1900542c2 --- /dev/null +++ b/internal/callbacker/send_manager/ordered/ordered_mocks.go @@ -0,0 +1,5 @@ +package ordered + +//go:generate moq -pkg mocks -out ./mocks/store_mock.go . SendManagerStore + +//go:generate moq -pkg mocks -out ./mocks/sender_mock.go . Sender diff --git a/internal/callbacker/send_manager/ordered/send_manager.go b/internal/callbacker/send_manager/ordered/send_manager.go index bb8db5dc8..38089617f 100644 --- a/internal/callbacker/send_manager/ordered/send_manager.go +++ b/internal/callbacker/send_manager/ordered/send_manager.go @@ -14,14 +14,19 @@ import ( type SendManagerStore interface { Set(ctx context.Context, dto *store.CallbackData) error SetMany(ctx context.Context, data []*store.CallbackData) error - PopMany(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) + GetAndDelete(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) +} + +type Sender interface { + Send(url, token string, callback *callbacker.Callback) (success, retry bool) + SendBatch(url, token string, callbacks []*callbacker.Callback) (success, retry bool) } type SendManager struct { url string // dependencies - sender callbacker.SenderI + sender Sender store SendManagerStore logger *slog.Logger @@ -35,6 +40,7 @@ type SendManager struct { singleSendInterval time.Duration //batchSendInterval time.Duration //delayDuration time.Duration + backfillQueueInterval time.Duration bufferSize int callbackList *list.List @@ -43,10 +49,11 @@ type SendManager struct { } const ( - entriesBufferSize = 10000 - batchSendIntervalDefault = 5 * time.Second - singleSendIntervalDefault = 5 * time.Second - expirationDefault = 24 * time.Hour + entriesBufferSize = 10000 + batchSendIntervalDefault = 5 * time.Second + singleSendIntervalDefault = 5 * time.Second + backfillQueueIntervalDefault = 5 * time.Second + expirationDefault = 24 * time.Hour ) func WithNow(nowFunc func() time.Time) func(*SendManager) { @@ -67,6 +74,12 @@ func WithSingleSendInterval(d time.Duration) func(*SendManager) { } } +func WithBackfillQueueInterval(d time.Duration) func(*SendManager) { + return func(m *SendManager) { + m.backfillQueueInterval = d + } +} + func WithExpiration(d time.Duration) func(*SendManager) { return func(m *SendManager) { m.expiration = d @@ -74,11 +87,6 @@ func WithExpiration(d time.Duration) func(*SendManager) { } func New(url string, sender callbacker.SenderI, store SendManagerStore, logger *slog.Logger, opts ...func(*SendManager)) *SendManager { - //batchSendInterval := defaultBatchSendInterval - //if sendingConfig.BatchSendInterval != 0 { - // batchSendInterval = sendingConfig.BatchSendInterval - //} - m := &SendManager{ url: url, sender: sender, @@ -88,7 +96,8 @@ func New(url string, sender callbacker.SenderI, store SendManagerStore, logger * singleSendInterval: singleSendIntervalDefault, //batchSendInterval: batchSendInterval, //delayDuration: sendingConfig.DelayDuration, - expiration: expirationDefault, + expiration: expirationDefault, + backfillQueueInterval: backfillQueueIntervalDefault, callbackList: list.New(), bufferSize: entriesBufferSize, @@ -142,7 +151,7 @@ func (m *SendManager) CallbacksQueued() int { func (m *SendManager) Start() { queueTicker := time.NewTicker(m.singleSendInterval) //sortTicker := time.NewTicker(10 * time.Second) - //backFillQueueTicker := time.NewTicker(10 * time.Second) + backfillQueueTicker := time.NewTicker(m.backfillQueueInterval) m.entriesWg.Add(1) go func() { @@ -172,51 +181,59 @@ func (m *SendManager) Start() { //case <-sortTicker.C: // m.sortByTimestamp() - //case <-backFillQueueTicker.C: - // capacityLeft := m.bufferSize - m.callbackList.Len() - // if capacityLeft == 0 { - // continue - // } - // - // callbacks, err := m.store.PopMany(m.ctx, m.url, capacityLeft) - // if err != nil { - // m.logger.Error("Failed to load callbacks", slog.String("err", err.Error())) - // continue - // } - // - // for _, callback := range callbacks { - // m.Enqueue(toEntry(callback)) - // } + case <-backfillQueueTicker.C: + m.backfillQueue() case <-queueTicker.C: - front := m.callbackList.Front() - if front == nil { - continue - } - - callbackEntry, ok := front.Value.(callbacker.CallbackEntry) - if !ok { - continue - } - - // If item is expired - dequeue without storing - if m.now().Sub(callbackEntry.Data.Timestamp) > m.expiration { - m.logger.Warn("callback expired", slog.Time("timestamp", callbackEntry.Data.Timestamp), slog.String("hash", callbackEntry.Data.TxID), slog.String("status", callbackEntry.Data.TxStatus)) - m.callbackList.Remove(front) - continue - } - - success, retry := m.sender.Send(m.url, callbackEntry.Token, callbackEntry.Data) - if !retry || success { - m.callbackList.Remove(front) - continue - } - m.logger.Error("failed to send single callback", slog.String("url", m.url)) + m.processQueueSingle() } } }() } +func (m *SendManager) backfillQueue() { + capacityLeft := m.bufferSize - m.callbackList.Len() + if capacityLeft == 0 { + return + } + + callbacks, err := m.store.GetAndDelete(m.ctx, m.url, capacityLeft) + if err != nil { + m.logger.Error("Failed to load callbacks", slog.String("err", err.Error())) + return + } + + for _, callback := range callbacks { + m.Enqueue(toEntry(callback)) + } +} + +func (m *SendManager) processQueueSingle() { + front := m.callbackList.Front() + if front == nil { + return + } + + callbackEntry, ok := front.Value.(callbacker.CallbackEntry) + if !ok { + return + } + + // If item is expired - dequeue without storing + if m.now().Sub(callbackEntry.Data.Timestamp) > m.expiration { + m.logger.Warn("callback expired", slog.Time("timestamp", callbackEntry.Data.Timestamp), slog.String("hash", callbackEntry.Data.TxID), slog.String("status", callbackEntry.Data.TxStatus)) + m.callbackList.Remove(front) + return + } + + success, retry := m.sender.Send(m.url, callbackEntry.Token, callbackEntry.Data) + if !retry || success { + m.callbackList.Remove(front) + return + } + m.logger.Error("failed to send single callback", slog.String("url", m.url)) +} + func (m *SendManager) storeToDB(entry callbacker.CallbackEntry) { callbackData := toStoreDto(m.url, entry) err := m.store.Set(context.Background(), callbackData) @@ -244,23 +261,22 @@ func toStoreDto(url string, entry callbacker.CallbackEntry) *store.CallbackData } } -// -//func toEntry(callbackData *store.CallbackData) callbacker.CallbackEntry { -// return callbacker.CallbackEntry{ -// Token: callbackData.Token, -// Data: &callbacker.Callback{ -// Timestamp: callbackData.Timestamp, -// CompetingTxs: callbackData.CompetingTxs, -// TxID: callbackData.TxID, -// TxStatus: callbackData.TxStatus, -// ExtraInfo: callbackData.ExtraInfo, -// MerklePath: callbackData.MerklePath, -// BlockHash: callbackData.BlockHash, -// BlockHeight: callbackData.BlockHeight, -// }, -// AllowBatch: callbackData.AllowBatch, -// } -//} +func toEntry(callbackData *store.CallbackData) callbacker.CallbackEntry { + return callbacker.CallbackEntry{ + Token: callbackData.Token, + Data: &callbacker.Callback{ + Timestamp: callbackData.Timestamp, + CompetingTxs: callbackData.CompetingTxs, + TxID: callbackData.TxID, + TxStatus: callbackData.TxStatus, + ExtraInfo: callbackData.ExtraInfo, + MerklePath: callbackData.MerklePath, + BlockHash: callbackData.BlockHash, + BlockHeight: callbackData.BlockHeight, + }, + AllowBatch: callbackData.AllowBatch, + } +} func (m *SendManager) dequeueAll() []callbacker.CallbackEntry { callbacks := make([]callbacker.CallbackEntry, 0, m.callbackList.Len()) diff --git a/internal/callbacker/send_manager/ordered/send_manager_test.go b/internal/callbacker/send_manager/ordered/send_manager_test.go index 33c558201..f72007be1 100644 --- a/internal/callbacker/send_manager/ordered/send_manager_test.go +++ b/internal/callbacker/send_manager/ordered/send_manager_test.go @@ -6,11 +6,12 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/bitcoin-sv/arc/internal/callbacker" - "github.com/bitcoin-sv/arc/internal/callbacker/mocks" "github.com/bitcoin-sv/arc/internal/callbacker/send_manager/ordered" + "github.com/bitcoin-sv/arc/internal/callbacker/send_manager/ordered/mocks" "github.com/bitcoin-sv/arc/internal/callbacker/store" ) @@ -19,6 +20,7 @@ func TestSendManagerStart(t *testing.T) { name string callbacksEnqueued int singleSendInterval time.Duration + backfillInterval time.Duration callbackTimestamp time.Time expectedCallbacksEnqueued int @@ -30,6 +32,7 @@ func TestSendManagerStart(t *testing.T) { name: "enqueue 10 callbacks - 10ms interval", callbacksEnqueued: 10, singleSendInterval: 10 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, callbackTimestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC), expectedCallbacksEnqueued: 10, @@ -41,6 +44,7 @@ func TestSendManagerStart(t *testing.T) { name: "enqueue 10 callbacks - 100ms interval - store remaining at graceful stop", callbacksEnqueued: 10, singleSendInterval: 100 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, callbackTimestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC), expectedCallbacksEnqueued: 10, @@ -52,6 +56,7 @@ func TestSendManagerStart(t *testing.T) { name: "enqueue 10 callbacks - expired", callbacksEnqueued: 10, singleSendInterval: 10 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, callbackTimestamp: time.Date(2025, 1, 9, 12, 0, 0, 0, time.UTC), expectedCallbacksEnqueued: 10, @@ -63,6 +68,7 @@ func TestSendManagerStart(t *testing.T) { name: "enqueue 15 callbacks - buffer size reached", callbacksEnqueued: 15, singleSendInterval: 10 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, callbackTimestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC), expectedCallbacksEnqueued: 10, @@ -70,32 +76,52 @@ func TestSendManagerStart(t *testing.T) { expectedSetCalls: 5, expectedSendCalls: 10, }, + { + name: "enqueue 10 callbacks - 10ms interval - back fill queue", + callbacksEnqueued: 10, + singleSendInterval: 10 * time.Millisecond, + backfillInterval: 20 * time.Millisecond, + callbackTimestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC), + + expectedCallbacksEnqueued: 10, + expectedSetManyCalls: 1, + expectedSetCalls: 0, + expectedSendCalls: 15, + }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { // given - senderMock := &mocks.SenderIMock{ + senderMock := &mocks.SenderMock{ SendFunc: func(_, _ string, _ *callbacker.Callback) (bool, bool) { return true, false }, SendBatchFunc: func(_, _ string, _ []*callbacker.Callback) (bool, bool) { return true, false }, } - storeMock := &mocks.CallbackerStoreMock{ + storeMock := &mocks.SendManagerStoreMock{ SetManyFunc: func(_ context.Context, data []*store.CallbackData) error { return nil }, SetFunc: func(_ context.Context, data *store.CallbackData) error { return nil }, + GetAndDeleteFunc: func(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) { + var callbacks []*store.CallbackData + for range limit { + callbacks = append(callbacks, &store.CallbackData{Timestamp: tc.callbackTimestamp}) + } + return callbacks, nil + }, } - sut := ordered.New("https://abc.com", senderMock, storeMock, slog.Default(), + sut := ordered.New("https://abcdefg.com", senderMock, storeMock, slog.Default(), ordered.WithBufferSize(10), ordered.WithNow(func() time.Time { return time.Date(2025, 1, 10, 12, 0, 0, 0, time.UTC) }), ordered.WithSingleSendInterval(tc.singleSendInterval), ordered.WithExpiration(time.Hour), + ordered.WithBackfillQueueInterval(tc.backfillInterval), ) // add callbacks before starting the manager to queue them @@ -111,10 +137,10 @@ func TestSendManagerStart(t *testing.T) { time.Sleep(150 * time.Millisecond) sut.GracefulStop() - require.Equal(t, 0, sut.CallbacksQueued()) - require.Equal(t, tc.expectedSetManyCalls, len(storeMock.SetManyCalls())) - require.Equal(t, tc.expectedSetCalls, len(storeMock.SetCalls())) - require.Equal(t, tc.expectedSendCalls, len(senderMock.SendCalls())) + assert.Equal(t, 0, sut.CallbacksQueued()) + assert.Equal(t, tc.expectedSetManyCalls, len(storeMock.SetManyCalls())) + assert.Equal(t, tc.expectedSetCalls, len(storeMock.SetCalls())) + assert.Equal(t, tc.expectedSendCalls, len(senderMock.SendCalls())) }) } } From b61f2475e1e7e6536735645d3f7cd1880e730e52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Fri, 17 Jan 2025 13:28:11 +0100 Subject: [PATCH 05/18] feat(ARCO-291): Get and delete callbacks ordered by timestamp --- internal/callbacker/store/postgresql/postgres.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/callbacker/store/postgresql/postgres.go b/internal/callbacker/store/postgresql/postgres.go index 0b57b9d07..c31ae111d 100644 --- a/internal/callbacker/store/postgresql/postgres.go +++ b/internal/callbacker/store/postgresql/postgres.go @@ -258,7 +258,6 @@ func (p *PostgreSQL) GetAndDelete(ctx context.Context, url string, limit int) ([ ,block_height ,competing_txs ,timestamp - ,postponed_until ,allow_batch` rows, err := p.db.QueryContext(ctx, q, url, limit) From 8f8ae484cb1854449040aafadaa895e3c3eb4459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Fri, 17 Jan 2025 14:18:31 +0100 Subject: [PATCH 06/18] feat(ARCO-291): Sort queue --- .../send_manager/ordered/send_manager.go | 88 ++++++++------ .../send_manager/ordered/send_manager_test.go | 114 ++++++++++++------ 2 files changed, 132 insertions(+), 70 deletions(-) diff --git a/internal/callbacker/send_manager/ordered/send_manager.go b/internal/callbacker/send_manager/ordered/send_manager.go index 38089617f..3705e9754 100644 --- a/internal/callbacker/send_manager/ordered/send_manager.go +++ b/internal/callbacker/send_manager/ordered/send_manager.go @@ -3,6 +3,7 @@ package ordered import ( "container/list" "context" + "errors" "log/slog" "sync" "time" @@ -40,7 +41,8 @@ type SendManager struct { singleSendInterval time.Duration //batchSendInterval time.Duration //delayDuration time.Duration - backfillQueueInterval time.Duration + backfillQueueInterval time.Duration + sortByTimestampInterval time.Duration bufferSize int callbackList *list.List @@ -49,11 +51,12 @@ type SendManager struct { } const ( - entriesBufferSize = 10000 - batchSendIntervalDefault = 5 * time.Second - singleSendIntervalDefault = 5 * time.Second - backfillQueueIntervalDefault = 5 * time.Second - expirationDefault = 24 * time.Hour + entriesBufferSize = 10000 + + singleSendIntervalDefault = 5 * time.Second + backfillQueueIntervalDefault = 5 * time.Second + expirationDefault = 24 * time.Hour + sortByTimestampIntervalDefault = 10 * time.Second ) func WithNow(nowFunc func() time.Time) func(*SendManager) { @@ -86,6 +89,12 @@ func WithExpiration(d time.Duration) func(*SendManager) { } } +func WithSortByTimestampInterval(d time.Duration) func(*SendManager) { + return func(m *SendManager) { + m.sortByTimestampInterval = d + } +} + func New(url string, sender callbacker.SenderI, store SendManagerStore, logger *slog.Logger, opts ...func(*SendManager)) *SendManager { m := &SendManager{ url: url, @@ -93,11 +102,10 @@ func New(url string, sender callbacker.SenderI, store SendManagerStore, logger * store: store, logger: logger, - singleSendInterval: singleSendIntervalDefault, - //batchSendInterval: batchSendInterval, - //delayDuration: sendingConfig.DelayDuration, - expiration: expirationDefault, - backfillQueueInterval: backfillQueueIntervalDefault, + singleSendInterval: singleSendIntervalDefault, + expiration: expirationDefault, + backfillQueueInterval: backfillQueueIntervalDefault, + sortByTimestampInterval: sortByTimestampIntervalDefault, callbackList: list.New(), bufferSize: entriesBufferSize, @@ -123,26 +131,35 @@ func (m *SendManager) Enqueue(entry callbacker.CallbackEntry) { m.callbackList.PushBack(entry) } -//func (m *SendManager) sortByTimestamp() { -// current := m.callbackList.Front() -// if m.callbackList.Front() == nil { -// return -// } -// for current != nil { -// index := current.Next() -// for index != nil { -// currentTime := current.Value.(*callbacker.CallbackEntry).Data.Timestamp -// indexTime := index.Value.(*callbacker.CallbackEntry).Data.Timestamp -// if currentTime.Before(indexTime) { -// temp := current.Value -// current.Value = index.Value -// index.Value = temp -// } -// index = index.Next() -// } -// current = current.Next() -// } -//} +func (m *SendManager) sortByTimestamp() error { + current := m.callbackList.Front() + if m.callbackList.Front() == nil { + return nil + } + for current != nil { + index := current.Next() + for index != nil { + currentTime, ok := current.Value.(callbacker.CallbackEntry) + if !ok { + return errors.New("callback entry is not a CallbackEntry") + } + + indexTime, ok := index.Value.(callbacker.CallbackEntry) + if !ok { + return errors.New("callback entry is not a CallbackEntry") + } + if currentTime.Data.Timestamp.Before(indexTime.Data.Timestamp) { + temp := current.Value + current.Value = index.Value + index.Value = temp + } + index = index.Next() + } + current = current.Next() + } + + return nil +} func (m *SendManager) CallbacksQueued() int { return m.callbackList.Len() @@ -150,7 +167,7 @@ func (m *SendManager) CallbacksQueued() int { func (m *SendManager) Start() { queueTicker := time.NewTicker(m.singleSendInterval) - //sortTicker := time.NewTicker(10 * time.Second) + sortQueueTicker := time.NewTicker(m.sortByTimestampInterval) backfillQueueTicker := time.NewTicker(m.backfillQueueInterval) m.entriesWg.Add(1) @@ -178,8 +195,11 @@ func (m *SendManager) Start() { select { case <-m.ctx.Done(): return - //case <-sortTicker.C: - // m.sortByTimestamp() + case <-sortQueueTicker.C: + err = m.sortByTimestamp() + if err != nil { + m.logger.Error("Failed to sort by timestamp", slog.String("err", err.Error())) + } case <-backfillQueueTicker.C: m.backfillQueue() diff --git a/internal/callbacker/send_manager/ordered/send_manager_test.go b/internal/callbacker/send_manager/ordered/send_manager_test.go index f72007be1..8bf6b8782 100644 --- a/internal/callbacker/send_manager/ordered/send_manager_test.go +++ b/internal/callbacker/send_manager/ordered/send_manager_test.go @@ -3,6 +3,7 @@ package ordered_test import ( "context" "log/slog" + "math/rand" "testing" "time" @@ -15,13 +16,39 @@ import ( "github.com/bitcoin-sv/arc/internal/callbacker/store" ) +var ( + now = time.Date(2025, 1, 10, 12, 0, 0, 0, time.UTC) + ts = time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC) + tsExpired = time.Date(2025, 1, 9, 12, 0, 0, 0, time.UTC) +) + func TestSendManagerStart(t *testing.T) { + callbackEntries10 := make([]callbacker.CallbackEntry, 10) + for i := range 10 { + callbackEntries10[i] = callbacker.CallbackEntry{Data: &callbacker.Callback{Timestamp: ts}} + } + + callbackEntries15 := make([]callbacker.CallbackEntry, 15) + for i := range 15 { + callbackEntries15[i] = callbacker.CallbackEntry{Data: &callbacker.Callback{Timestamp: ts}} + } + + callbackEntries10Expired := make([]callbacker.CallbackEntry, 10) + for i := range 10 { + callbackEntries10Expired[i] = callbacker.CallbackEntry{Data: &callbacker.Callback{Timestamp: tsExpired}} + } + + callbackEntriesUnsorted := make([]callbacker.CallbackEntry, 10) + for i := range 10 { + callbackEntriesUnsorted[i] = callbacker.CallbackEntry{Data: &callbacker.Callback{Timestamp: time.Date(2025, 1, rand.Intn(31), rand.Intn(24), rand.Intn(59), 0, 0, time.UTC)}} + } + tcs := []struct { - name string - callbacksEnqueued int - singleSendInterval time.Duration - backfillInterval time.Duration - callbackTimestamp time.Time + name string + callbacksEnqueued []callbacker.CallbackEntry + singleSendInterval time.Duration + backfillInterval time.Duration + sortByTimestampInterval time.Duration expectedCallbacksEnqueued int expectedSetManyCalls int @@ -29,11 +56,11 @@ func TestSendManagerStart(t *testing.T) { expectedSendCalls int }{ { - name: "enqueue 10 callbacks - 10ms interval", - callbacksEnqueued: 10, - singleSendInterval: 10 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - callbackTimestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC), + name: "enqueue 10 callbacks - 10ms interval", + callbacksEnqueued: callbackEntries10, + singleSendInterval: 10 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, + sortByTimestampInterval: 500 * time.Millisecond, expectedCallbacksEnqueued: 10, expectedSetManyCalls: 0, @@ -41,11 +68,11 @@ func TestSendManagerStart(t *testing.T) { expectedSendCalls: 10, }, { - name: "enqueue 10 callbacks - 100ms interval - store remaining at graceful stop", - callbacksEnqueued: 10, - singleSendInterval: 100 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - callbackTimestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC), + name: "enqueue 10 callbacks - 100ms interval - store remaining at graceful stop", + callbacksEnqueued: callbackEntries10, + singleSendInterval: 100 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, + sortByTimestampInterval: 500 * time.Millisecond, expectedCallbacksEnqueued: 10, expectedSetManyCalls: 1, @@ -53,11 +80,11 @@ func TestSendManagerStart(t *testing.T) { expectedSendCalls: 1, }, { - name: "enqueue 10 callbacks - expired", - callbacksEnqueued: 10, - singleSendInterval: 10 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - callbackTimestamp: time.Date(2025, 1, 9, 12, 0, 0, 0, time.UTC), + name: "enqueue 10 callbacks - expired", + callbacksEnqueued: callbackEntries10Expired, + singleSendInterval: 10 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, + sortByTimestampInterval: 500 * time.Millisecond, expectedCallbacksEnqueued: 10, expectedSetManyCalls: 0, @@ -65,11 +92,11 @@ func TestSendManagerStart(t *testing.T) { expectedSendCalls: 0, }, { - name: "enqueue 15 callbacks - buffer size reached", - callbacksEnqueued: 15, - singleSendInterval: 10 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - callbackTimestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC), + name: "enqueue 15 callbacks - buffer size reached", + callbacksEnqueued: callbackEntries15, + singleSendInterval: 10 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, + sortByTimestampInterval: 500 * time.Millisecond, expectedCallbacksEnqueued: 10, expectedSetManyCalls: 0, @@ -77,17 +104,29 @@ func TestSendManagerStart(t *testing.T) { expectedSendCalls: 10, }, { - name: "enqueue 10 callbacks - 10ms interval - back fill queue", - callbacksEnqueued: 10, - singleSendInterval: 10 * time.Millisecond, - backfillInterval: 20 * time.Millisecond, - callbackTimestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC), + name: "enqueue 10 callbacks - 10ms interval - back fill queue", + callbacksEnqueued: callbackEntries10, + singleSendInterval: 10 * time.Millisecond, + backfillInterval: 20 * time.Millisecond, + sortByTimestampInterval: 500 * time.Millisecond, expectedCallbacksEnqueued: 10, expectedSetManyCalls: 1, expectedSetCalls: 0, expectedSendCalls: 15, }, + { + name: "enqueue 10 callbacks - 10ms interval - sort by timestamp", + callbacksEnqueued: callbackEntriesUnsorted, + singleSendInterval: 200 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, + sortByTimestampInterval: 20 * time.Millisecond, + + expectedCallbacksEnqueued: 10, + expectedSetManyCalls: 1, + expectedSetCalls: 0, + expectedSendCalls: 0, + }, } for _, tc := range tcs { @@ -100,6 +139,10 @@ func TestSendManagerStart(t *testing.T) { storeMock := &mocks.SendManagerStoreMock{ SetManyFunc: func(_ context.Context, data []*store.CallbackData) error { + for i := 0; i < len(data)-1; i++ { + assert.GreaterOrEqual(t, data[i].Timestamp, data[i+1].Timestamp) + } + return nil }, SetFunc: func(_ context.Context, data *store.CallbackData) error { @@ -108,7 +151,7 @@ func TestSendManagerStart(t *testing.T) { GetAndDeleteFunc: func(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) { var callbacks []*store.CallbackData for range limit { - callbacks = append(callbacks, &store.CallbackData{Timestamp: tc.callbackTimestamp}) + callbacks = append(callbacks, &store.CallbackData{Timestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC)}) } return callbacks, nil }, @@ -117,18 +160,17 @@ func TestSendManagerStart(t *testing.T) { sut := ordered.New("https://abcdefg.com", senderMock, storeMock, slog.Default(), ordered.WithBufferSize(10), ordered.WithNow(func() time.Time { - return time.Date(2025, 1, 10, 12, 0, 0, 0, time.UTC) + return now }), ordered.WithSingleSendInterval(tc.singleSendInterval), ordered.WithExpiration(time.Hour), ordered.WithBackfillQueueInterval(tc.backfillInterval), + ordered.WithSortByTimestampInterval(tc.sortByTimestampInterval), ) // add callbacks before starting the manager to queue them - for range tc.callbacksEnqueued { - sut.Enqueue(callbacker.CallbackEntry{Data: &callbacker.Callback{ - Timestamp: tc.callbackTimestamp, - }}) + for _, cb := range tc.callbacksEnqueued { + sut.Enqueue(cb) } require.Equal(t, tc.expectedCallbacksEnqueued, sut.CallbacksQueued()) From 827ab39cfcd712e0b6e0de7b2ecd78c331ea3f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Fri, 17 Jan 2025 14:58:20 +0100 Subject: [PATCH 07/18] feat(ARCO-291): Rm comment --- internal/callbacker/send_manager/ordered/send_manager.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/callbacker/send_manager/ordered/send_manager.go b/internal/callbacker/send_manager/ordered/send_manager.go index 3705e9754..9fcf43b6b 100644 --- a/internal/callbacker/send_manager/ordered/send_manager.go +++ b/internal/callbacker/send_manager/ordered/send_manager.go @@ -38,9 +38,7 @@ type SendManager struct { cancelAll context.CancelFunc ctx context.Context - singleSendInterval time.Duration - //batchSendInterval time.Duration - //delayDuration time.Duration + singleSendInterval time.Duration backfillQueueInterval time.Duration sortByTimestampInterval time.Duration From 887bdf3fafaab7f668b8eb5037dd827acc3b949e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Fri, 17 Jan 2025 15:02:14 +0100 Subject: [PATCH 08/18] feat(ARCO-291): make lint --- internal/callbacker/send_manager/ordered/send_manager_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/callbacker/send_manager/ordered/send_manager_test.go b/internal/callbacker/send_manager/ordered/send_manager_test.go index 8bf6b8782..d0674a6d4 100644 --- a/internal/callbacker/send_manager/ordered/send_manager_test.go +++ b/internal/callbacker/send_manager/ordered/send_manager_test.go @@ -145,10 +145,10 @@ func TestSendManagerStart(t *testing.T) { return nil }, - SetFunc: func(_ context.Context, data *store.CallbackData) error { + SetFunc: func(_ context.Context, _ *store.CallbackData) error { return nil }, - GetAndDeleteFunc: func(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) { + GetAndDeleteFunc: func(_ context.Context, _ string, limit int) ([]*store.CallbackData, error) { var callbacks []*store.CallbackData for range limit { callbacks = append(callbacks, &store.CallbackData{Timestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC)}) From b024b24f7d0ff06bbb81b911019623866673b81c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Fri, 17 Jan 2025 15:04:50 +0100 Subject: [PATCH 09/18] feat(ARCO-291): Fix test --- internal/callbacker/store/postgresql/postgres.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/callbacker/store/postgresql/postgres.go b/internal/callbacker/store/postgresql/postgres.go index c31ae111d..0b57b9d07 100644 --- a/internal/callbacker/store/postgresql/postgres.go +++ b/internal/callbacker/store/postgresql/postgres.go @@ -258,6 +258,7 @@ func (p *PostgreSQL) GetAndDelete(ctx context.Context, url string, limit int) ([ ,block_height ,competing_txs ,timestamp + ,postponed_until ,allow_batch` rows, err := p.db.QueryContext(ctx, q, url, limit) From 4533bdae9b0dfdac503c0a7dc62a286c8a0aff02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Fri, 17 Jan 2025 17:06:55 +0100 Subject: [PATCH 10/18] feat(ARCO-291): Ordered send manager with batch callbacks --- .../send_manager/ordered/send_manager.go | 208 ++++++++++++++---- .../send_manager/ordered/send_manager_test.go | 151 +++++++++++-- 2 files changed, 290 insertions(+), 69 deletions(-) diff --git a/internal/callbacker/send_manager/ordered/send_manager.go b/internal/callbacker/send_manager/ordered/send_manager.go index 9fcf43b6b..04bb20671 100644 --- a/internal/callbacker/send_manager/ordered/send_manager.go +++ b/internal/callbacker/send_manager/ordered/send_manager.go @@ -38,23 +38,30 @@ type SendManager struct { cancelAll context.CancelFunc ctx context.Context - singleSendInterval time.Duration + queueProcessInterval time.Duration backfillQueueInterval time.Duration sortByTimestampInterval time.Duration + batchSendInterval time.Duration + batchSize int - bufferSize int - callbackList *list.List + bufferSize int + callbackQueue *list.List now func() time.Time } const ( - entriesBufferSize = 10000 - - singleSendIntervalDefault = 5 * time.Second + entriesBufferSize = 10000 + batchSizeDefault = 50 + queueProcessIntervalDefault = 5 * time.Second backfillQueueIntervalDefault = 5 * time.Second expirationDefault = 24 * time.Hour sortByTimestampIntervalDefault = 10 * time.Second + batchSendIntervalDefault = 5 * time.Second +) + +var ( + ErrSendBatchedCallbacks = errors.New("failed to send batched callback") ) func WithNow(nowFunc func() time.Time) func(*SendManager) { @@ -69,9 +76,9 @@ func WithBufferSize(size int) func(*SendManager) { } } -func WithSingleSendInterval(d time.Duration) func(*SendManager) { +func WithQueueProcessInterval(d time.Duration) func(*SendManager) { return func(m *SendManager) { - m.singleSendInterval = d + m.queueProcessInterval = d } } @@ -93,6 +100,18 @@ func WithSortByTimestampInterval(d time.Duration) func(*SendManager) { } } +func WithBatchSendInterval(d time.Duration) func(*SendManager) { + return func(m *SendManager) { + m.batchSendInterval = d + } +} + +func WithBatchSize(size int) func(*SendManager) { + return func(m *SendManager) { + m.batchSize = size + } +} + func New(url string, sender callbacker.SenderI, store SendManagerStore, logger *slog.Logger, opts ...func(*SendManager)) *SendManager { m := &SendManager{ url: url, @@ -100,13 +119,15 @@ func New(url string, sender callbacker.SenderI, store SendManagerStore, logger * store: store, logger: logger, - singleSendInterval: singleSendIntervalDefault, + queueProcessInterval: queueProcessIntervalDefault, expiration: expirationDefault, backfillQueueInterval: backfillQueueIntervalDefault, sortByTimestampInterval: sortByTimestampIntervalDefault, + batchSendInterval: batchSendIntervalDefault, + batchSize: batchSizeDefault, - callbackList: list.New(), - bufferSize: entriesBufferSize, + callbackQueue: list.New(), + bufferSize: entriesBufferSize, } for _, opt := range opts { @@ -121,17 +142,17 @@ func New(url string, sender callbacker.SenderI, store SendManagerStore, logger * } func (m *SendManager) Enqueue(entry callbacker.CallbackEntry) { - if m.callbackList.Len() >= m.bufferSize { + if m.callbackQueue.Len() >= m.bufferSize { m.storeToDB(entry) return } - m.callbackList.PushBack(entry) + m.callbackQueue.PushBack(entry) } func (m *SendManager) sortByTimestamp() error { - current := m.callbackList.Front() - if m.callbackList.Front() == nil { + current := m.callbackQueue.Front() + if m.callbackQueue.Front() == nil { return nil } for current != nil { @@ -160,20 +181,31 @@ func (m *SendManager) sortByTimestamp() error { } func (m *SendManager) CallbacksQueued() int { - return m.callbackList.Len() + return m.callbackQueue.Len() } func (m *SendManager) Start() { - queueTicker := time.NewTicker(m.singleSendInterval) + queueTicker := time.NewTicker(m.queueProcessInterval) sortQueueTicker := time.NewTicker(m.sortByTimestampInterval) backfillQueueTicker := time.NewTicker(m.backfillQueueInterval) + batchSendTicker := time.NewTicker(m.batchSendInterval) m.entriesWg.Add(1) + var callbackBatch []*list.Element + go func() { var err error defer func() { // read all from callback queue and store in database - data := make([]*store.CallbackData, m.callbackList.Len()) + data := make([]*store.CallbackData, m.callbackQueue.Len()+len(callbackBatch)) + + for _, callbackElement := range callbackBatch { + entry, ok := callbackElement.Value.(callbacker.CallbackEntry) + if !ok { + continue + } + m.callbackQueue.PushBack(entry) + } for i, entry := range m.dequeueAll() { data[i] = toStoreDto(m.url, entry) @@ -189,6 +221,8 @@ func (m *SendManager) Start() { m.entriesWg.Done() }() + lastIterationWasBatch := false + for { select { case <-m.ctx.Done(): @@ -202,54 +236,132 @@ func (m *SendManager) Start() { case <-backfillQueueTicker.C: m.backfillQueue() + m.logger.Debug("Callback queue backfilled", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) + case <-batchSendTicker.C: + if len(callbackBatch) == 0 { + continue + } + + err = m.sendElementBatch(callbackBatch) + if err != nil { + m.logger.Error("Failed to send batch of callbacks", slog.String("url", m.url)) + continue + } + + callbackBatch = callbackBatch[:0] + m.logger.Debug("Batched callbacks sent on interval", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) case <-queueTicker.C: - m.processQueueSingle() + front := m.callbackQueue.Front() + if front == nil { + continue + } + + callbackEntry, ok := front.Value.(callbacker.CallbackEntry) + if !ok { + continue + } + + // If item is expired - dequeue without storing + if m.now().Sub(callbackEntry.Data.Timestamp) > m.expiration { + m.logger.Warn("Callback expired", slog.Time("timestamp", callbackEntry.Data.Timestamp), slog.String("hash", callbackEntry.Data.TxID), slog.String("status", callbackEntry.Data.TxStatus)) + m.callbackQueue.Remove(front) + continue + } + + if callbackEntry.AllowBatch { + lastIterationWasBatch = true + + if len(callbackBatch) < m.batchSize { + callbackBatch = append(callbackBatch, front) + queueTicker.Reset(m.queueProcessInterval) + m.callbackQueue.Remove(front) + continue + } + + err = m.sendElementBatch(callbackBatch) + if err != nil { + m.logger.Error("Failed to send batch of callbacks", slog.String("url", m.url)) + continue + } + + callbackBatch = callbackBatch[:0] + m.logger.Debug("Batched callbacks sent", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) + continue + } + + if lastIterationWasBatch { + lastIterationWasBatch = false + if len(callbackBatch) > 0 { + // if entry is not a batched entry, but last one was, send batch to keep the order + err = m.sendElementBatch(callbackBatch) + if err != nil { + m.logger.Error("Failed to send batch of callbacks", slog.String("url", m.url)) + continue + } + callbackBatch = callbackBatch[:0] + m.logger.Debug("Batched callbacks sent before sending single callback", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) + } + } + + success, retry := m.sender.Send(m.url, callbackEntry.Token, callbackEntry.Data) + if !retry || success { + m.callbackQueue.Remove(front) + m.logger.Debug("Single callback sent", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) + continue + } + m.logger.Error("Failed to send single callback", slog.String("url", m.url)) } } }() } -func (m *SendManager) backfillQueue() { - capacityLeft := m.bufferSize - m.callbackList.Len() - if capacityLeft == 0 { - return +func (m *SendManager) sendElementBatch(callbackElements []*list.Element) error { + var callbackElement *list.Element + callbackBatch := make([]callbacker.CallbackEntry, 0, len(callbackElements)) + for _, element := range callbackElements { + callback, ok := element.Value.(callbacker.CallbackEntry) + if !ok { + continue + } + callbackBatch = append(callbackBatch, callback) } + success, retry := m.sendBatch(callbackBatch) + if !retry || success { + for _, callbackElement = range callbackElements { + m.callbackQueue.Remove(callbackElement) + } - callbacks, err := m.store.GetAndDelete(m.ctx, m.url, capacityLeft) - if err != nil { - m.logger.Error("Failed to load callbacks", slog.String("err", err.Error())) - return + return nil } - for _, callback := range callbacks { - m.Enqueue(toEntry(callback)) - } + return ErrSendBatchedCallbacks } -func (m *SendManager) processQueueSingle() { - front := m.callbackList.Front() - if front == nil { - return +func (m *SendManager) sendBatch(batch []callbacker.CallbackEntry) (success, retry bool) { + token := batch[0].Token + callbacks := make([]*callbacker.Callback, len(batch)) + for i, e := range batch { + callbacks[i] = e.Data } - callbackEntry, ok := front.Value.(callbacker.CallbackEntry) - if !ok { + return m.sender.SendBatch(m.url, token, callbacks) +} + +func (m *SendManager) backfillQueue() { + capacityLeft := m.bufferSize - m.callbackQueue.Len() + if capacityLeft == 0 { return } - // If item is expired - dequeue without storing - if m.now().Sub(callbackEntry.Data.Timestamp) > m.expiration { - m.logger.Warn("callback expired", slog.Time("timestamp", callbackEntry.Data.Timestamp), slog.String("hash", callbackEntry.Data.TxID), slog.String("status", callbackEntry.Data.TxStatus)) - m.callbackList.Remove(front) + callbacks, err := m.store.GetAndDelete(m.ctx, m.url, capacityLeft) + if err != nil { + m.logger.Error("Failed to load callbacks", slog.String("err", err.Error())) return } - success, retry := m.sender.Send(m.url, callbackEntry.Token, callbackEntry.Data) - if !retry || success { - m.callbackList.Remove(front) - return + for _, callback := range callbacks { + m.Enqueue(toEntry(callback)) } - m.logger.Error("failed to send single callback", slog.String("url", m.url)) } func (m *SendManager) storeToDB(entry callbacker.CallbackEntry) { @@ -297,10 +409,10 @@ func toEntry(callbackData *store.CallbackData) callbacker.CallbackEntry { } func (m *SendManager) dequeueAll() []callbacker.CallbackEntry { - callbacks := make([]callbacker.CallbackEntry, 0, m.callbackList.Len()) + callbacks := make([]callbacker.CallbackEntry, 0, m.callbackQueue.Len()) var next *list.Element - for front := m.callbackList.Front(); front != nil; front = next { + for front := m.callbackQueue.Front(); front != nil; front = next { next = front.Next() entry, ok := front.Value.(callbacker.CallbackEntry) if !ok { @@ -308,7 +420,7 @@ func (m *SendManager) dequeueAll() []callbacker.CallbackEntry { } callbacks = append(callbacks, entry) - m.callbackList.Remove(front) + m.callbackQueue.Remove(front) } return callbacks diff --git a/internal/callbacker/send_manager/ordered/send_manager_test.go b/internal/callbacker/send_manager/ordered/send_manager_test.go index d0674a6d4..d05eb52fb 100644 --- a/internal/callbacker/send_manager/ordered/send_manager_test.go +++ b/internal/callbacker/send_manager/ordered/send_manager_test.go @@ -2,8 +2,10 @@ package ordered_test import ( "context" + "fmt" "log/slog" "math/rand" + "os" "testing" "time" @@ -43,102 +45,204 @@ func TestSendManagerStart(t *testing.T) { callbackEntriesUnsorted[i] = callbacker.CallbackEntry{Data: &callbacker.Callback{Timestamp: time.Date(2025, 1, rand.Intn(31), rand.Intn(24), rand.Intn(59), 0, 0, time.UTC)}} } + callbackEntriesBatch3Expired10 := make([]callbacker.CallbackEntry, 10) + for i := range 10 { + if i >= 3 && i < 6 { + callbackEntriesBatch3Expired10[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: tsExpired}, AllowBatch: true} + continue + } + callbackEntriesBatch3Expired10[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: ts}, AllowBatch: true} + } + + callbackEntriesBatch10 := make([]callbacker.CallbackEntry, 10) + for i := range 10 { + callbackEntriesBatch10[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: ts}, AllowBatch: true} + } + + callbackEntriesBatched10Mixed := make([]callbacker.CallbackEntry, 10) + for i := range 10 { + if i >= 3 && i < 7 { + callbackEntriesBatched10Mixed[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: ts}, AllowBatch: true} + continue + } + callbackEntriesBatched10Mixed[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: ts}, AllowBatch: false} + } + + callbackEntriesBatched10Mixed2 := make([]callbacker.CallbackEntry, 10) + for i := range 10 { + if i >= 2 && i < 8 { + callbackEntriesBatched10Mixed2[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: ts}, AllowBatch: true} + continue + } + callbackEntriesBatched10Mixed2[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: ts}, AllowBatch: false} + } + tcs := []struct { name string callbacksEnqueued []callbacker.CallbackEntry - singleSendInterval time.Duration + queueProcessInterval time.Duration backfillInterval time.Duration sortByTimestampInterval time.Duration + batchInterval time.Duration expectedCallbacksEnqueued int - expectedSetManyCalls int + expectedSetMany int expectedSetCalls int expectedSendCalls int + expectedSendBatchCalls []int }{ { name: "enqueue 10 callbacks - 10ms interval", callbacksEnqueued: callbackEntries10, - singleSendInterval: 10 * time.Millisecond, + queueProcessInterval: 10 * time.Millisecond, backfillInterval: 500 * time.Millisecond, sortByTimestampInterval: 500 * time.Millisecond, + batchInterval: 500 * time.Millisecond, expectedCallbacksEnqueued: 10, - expectedSetManyCalls: 0, expectedSetCalls: 0, expectedSendCalls: 10, }, { name: "enqueue 10 callbacks - 100ms interval - store remaining at graceful stop", callbacksEnqueued: callbackEntries10, - singleSendInterval: 100 * time.Millisecond, + queueProcessInterval: 100 * time.Millisecond, backfillInterval: 500 * time.Millisecond, sortByTimestampInterval: 500 * time.Millisecond, + batchInterval: 500 * time.Millisecond, expectedCallbacksEnqueued: 10, - expectedSetManyCalls: 1, + expectedSetMany: 9, expectedSetCalls: 0, expectedSendCalls: 1, }, { name: "enqueue 10 callbacks - expired", callbacksEnqueued: callbackEntries10Expired, - singleSendInterval: 10 * time.Millisecond, + queueProcessInterval: 10 * time.Millisecond, backfillInterval: 500 * time.Millisecond, sortByTimestampInterval: 500 * time.Millisecond, + batchInterval: 500 * time.Millisecond, expectedCallbacksEnqueued: 10, - expectedSetManyCalls: 0, expectedSetCalls: 0, expectedSendCalls: 0, }, { name: "enqueue 15 callbacks - buffer size reached", callbacksEnqueued: callbackEntries15, - singleSendInterval: 10 * time.Millisecond, + queueProcessInterval: 10 * time.Millisecond, backfillInterval: 500 * time.Millisecond, sortByTimestampInterval: 500 * time.Millisecond, + batchInterval: 500 * time.Millisecond, expectedCallbacksEnqueued: 10, - expectedSetManyCalls: 0, expectedSetCalls: 5, expectedSendCalls: 10, }, { - name: "enqueue 10 callbacks - 10ms interval - back fill queue", + name: "enqueue 10 callbacks - back fill queue", callbacksEnqueued: callbackEntries10, - singleSendInterval: 10 * time.Millisecond, + queueProcessInterval: 10 * time.Millisecond, backfillInterval: 20 * time.Millisecond, sortByTimestampInterval: 500 * time.Millisecond, + batchInterval: 500 * time.Millisecond, expectedCallbacksEnqueued: 10, - expectedSetManyCalls: 1, + expectedSetMany: 9, expectedSetCalls: 0, expectedSendCalls: 15, }, { - name: "enqueue 10 callbacks - 10ms interval - sort by timestamp", + name: "enqueue 10 callbacks - sort by timestamp", callbacksEnqueued: callbackEntriesUnsorted, - singleSendInterval: 200 * time.Millisecond, + queueProcessInterval: 200 * time.Millisecond, backfillInterval: 500 * time.Millisecond, sortByTimestampInterval: 20 * time.Millisecond, + batchInterval: 500 * time.Millisecond, expectedCallbacksEnqueued: 10, - expectedSetManyCalls: 1, + expectedSetMany: 10, expectedSetCalls: 0, expectedSendCalls: 0, }, + { + name: "enqueue 10 batched callbacks - 3 expired", + callbacksEnqueued: callbackEntriesBatch3Expired10, + queueProcessInterval: 10 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, + sortByTimestampInterval: 500 * time.Millisecond, + batchInterval: 500 * time.Millisecond, + + expectedCallbacksEnqueued: 10, + expectedSetMany: 2, + expectedSetCalls: 0, + expectedSendCalls: 0, + expectedSendBatchCalls: []int{5}, + }, + { + name: "enqueue 10 batched callbacks - 60ms batch interval", + callbacksEnqueued: callbackEntriesBatch10, + queueProcessInterval: 10 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, + sortByTimestampInterval: 500 * time.Millisecond, + batchInterval: 60 * time.Millisecond, + + expectedCallbacksEnqueued: 10, + expectedSetCalls: 0, + expectedSendCalls: 0, + expectedSendBatchCalls: []int{5, 5}, + }, + { + name: "enqueue 10 batched callbacks - 4 batched, 6 single", + callbacksEnqueued: callbackEntriesBatched10Mixed, + queueProcessInterval: 10 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, + sortByTimestampInterval: 500 * time.Millisecond, + batchInterval: 500 * time.Millisecond, + + expectedCallbacksEnqueued: 10, + expectedSetCalls: 0, + expectedSendCalls: 6, + expectedSendBatchCalls: []int{4}, + }, + { + name: "enqueue 10 batched callbacks - 6 batched, 4 single", + callbacksEnqueued: callbackEntriesBatched10Mixed2, + queueProcessInterval: 10 * time.Millisecond, + backfillInterval: 500 * time.Millisecond, + sortByTimestampInterval: 500 * time.Millisecond, + batchInterval: 500 * time.Millisecond, + + expectedCallbacksEnqueued: 10, + expectedSetCalls: 0, + expectedSendCalls: 4, + expectedSendBatchCalls: []int{5, 1}, + }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { // given + + counter := 0 senderMock := &mocks.SenderMock{ - SendFunc: func(_, _ string, _ *callbacker.Callback) (bool, bool) { return true, false }, - SendBatchFunc: func(_, _ string, _ []*callbacker.Callback) (bool, bool) { return true, false }, + SendFunc: func(_, _ string, _ *callbacker.Callback) (bool, bool) { return true, false }, + SendBatchFunc: func(_, _ string, batch []*callbacker.Callback) (bool, bool) { + if counter >= len(tc.expectedSendBatchCalls) { + t.Fail() + } else { + assert.Equal(t, tc.expectedSendBatchCalls[counter], len(batch)) + counter++ + } + return true, false + }, } storeMock := &mocks.SendManagerStoreMock{ SetManyFunc: func(_ context.Context, data []*store.CallbackData) error { + assert.Equal(t, tc.expectedSetMany, len(data)) + for i := 0; i < len(data)-1; i++ { assert.GreaterOrEqual(t, data[i].Timestamp, data[i+1].Timestamp) } @@ -157,15 +261,19 @@ func TestSendManagerStart(t *testing.T) { }, } - sut := ordered.New("https://abcdefg.com", senderMock, storeMock, slog.Default(), + logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug})) + + sut := ordered.New("https://abcdefg.com", senderMock, storeMock, logger, ordered.WithBufferSize(10), ordered.WithNow(func() time.Time { return now }), - ordered.WithSingleSendInterval(tc.singleSendInterval), + ordered.WithQueueProcessInterval(tc.queueProcessInterval), ordered.WithExpiration(time.Hour), ordered.WithBackfillQueueInterval(tc.backfillInterval), ordered.WithSortByTimestampInterval(tc.sortByTimestampInterval), + ordered.WithBatchSendInterval(tc.batchInterval), + ordered.WithBatchSize(5), ) // add callbacks before starting the manager to queue them @@ -180,9 +288,10 @@ func TestSendManagerStart(t *testing.T) { sut.GracefulStop() assert.Equal(t, 0, sut.CallbacksQueued()) - assert.Equal(t, tc.expectedSetManyCalls, len(storeMock.SetManyCalls())) + assert.Equal(t, tc.expectedSetMany > 0, len(storeMock.SetManyCalls()) == 1) assert.Equal(t, tc.expectedSetCalls, len(storeMock.SetCalls())) assert.Equal(t, tc.expectedSendCalls, len(senderMock.SendCalls())) + assert.Equal(t, len(tc.expectedSendBatchCalls), len(senderMock.SendBatchCalls())) }) } } From 8119a32c92c18b7aec93d3730e953848a38d14a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Thu, 23 Jan 2025 09:45:36 +0100 Subject: [PATCH 11/18] feat(ARCO-291): Refactor ordered send manager --- .../send_manager/ordered/mocks/sender_mock.go | 144 ------ .../send_manager/ordered/mocks/store_mock.go | 189 -------- .../send_manager/ordered/ordered_mocks.go | 5 - .../send_manager/ordered/send_manager.go | 436 ------------------ .../send_manager/ordered/send_manager_test.go | 297 ------------ 5 files changed, 1071 deletions(-) delete mode 100644 internal/callbacker/send_manager/ordered/mocks/sender_mock.go delete mode 100644 internal/callbacker/send_manager/ordered/mocks/store_mock.go delete mode 100644 internal/callbacker/send_manager/ordered/ordered_mocks.go delete mode 100644 internal/callbacker/send_manager/ordered/send_manager.go delete mode 100644 internal/callbacker/send_manager/ordered/send_manager_test.go diff --git a/internal/callbacker/send_manager/ordered/mocks/sender_mock.go b/internal/callbacker/send_manager/ordered/mocks/sender_mock.go deleted file mode 100644 index b3401d262..000000000 --- a/internal/callbacker/send_manager/ordered/mocks/sender_mock.go +++ /dev/null @@ -1,144 +0,0 @@ -// Code generated by moq; DO NOT EDIT. -// github.com/matryer/moq - -package mocks - -import ( - "github.com/bitcoin-sv/arc/internal/callbacker" - "github.com/bitcoin-sv/arc/internal/callbacker/send_manager/ordered" - "sync" -) - -// Ensure, that SenderMock does implement ordered.Sender. -// If this is not the case, regenerate this file with moq. -var _ ordered.Sender = &SenderMock{} - -// SenderMock is a mock implementation of ordered.Sender. -// -// func TestSomethingThatUsesSender(t *testing.T) { -// -// // make and configure a mocked ordered.Sender -// mockedSender := &SenderMock{ -// SendFunc: func(url string, token string, callback *callbacker.Callback) (bool, bool) { -// panic("mock out the Send method") -// }, -// SendBatchFunc: func(url string, token string, callbacks []*callbacker.Callback) (bool, bool) { -// panic("mock out the SendBatch method") -// }, -// } -// -// // use mockedSender in code that requires ordered.Sender -// // and then make assertions. -// -// } -type SenderMock struct { - // SendFunc mocks the Send method. - SendFunc func(url string, token string, callback *callbacker.Callback) (bool, bool) - - // SendBatchFunc mocks the SendBatch method. - SendBatchFunc func(url string, token string, callbacks []*callbacker.Callback) (bool, bool) - - // calls tracks calls to the methods. - calls struct { - // Send holds details about calls to the Send method. - Send []struct { - // URL is the url argument value. - URL string - // Token is the token argument value. - Token string - // Callback is the callback argument value. - Callback *callbacker.Callback - } - // SendBatch holds details about calls to the SendBatch method. - SendBatch []struct { - // URL is the url argument value. - URL string - // Token is the token argument value. - Token string - // Callbacks is the callbacks argument value. - Callbacks []*callbacker.Callback - } - } - lockSend sync.RWMutex - lockSendBatch sync.RWMutex -} - -// Send calls SendFunc. -func (mock *SenderMock) Send(url string, token string, callback *callbacker.Callback) (bool, bool) { - if mock.SendFunc == nil { - panic("SenderMock.SendFunc: method is nil but Sender.Send was just called") - } - callInfo := struct { - URL string - Token string - Callback *callbacker.Callback - }{ - URL: url, - Token: token, - Callback: callback, - } - mock.lockSend.Lock() - mock.calls.Send = append(mock.calls.Send, callInfo) - mock.lockSend.Unlock() - return mock.SendFunc(url, token, callback) -} - -// SendCalls gets all the calls that were made to Send. -// Check the length with: -// -// len(mockedSender.SendCalls()) -func (mock *SenderMock) SendCalls() []struct { - URL string - Token string - Callback *callbacker.Callback -} { - var calls []struct { - URL string - Token string - Callback *callbacker.Callback - } - mock.lockSend.RLock() - calls = mock.calls.Send - mock.lockSend.RUnlock() - return calls -} - -// SendBatch calls SendBatchFunc. -func (mock *SenderMock) SendBatch(url string, token string, callbacks []*callbacker.Callback) (bool, bool) { - if mock.SendBatchFunc == nil { - panic("SenderMock.SendBatchFunc: method is nil but Sender.SendBatch was just called") - } - callInfo := struct { - URL string - Token string - Callbacks []*callbacker.Callback - }{ - URL: url, - Token: token, - Callbacks: callbacks, - } - mock.lockSendBatch.Lock() - mock.calls.SendBatch = append(mock.calls.SendBatch, callInfo) - mock.lockSendBatch.Unlock() - return mock.SendBatchFunc(url, token, callbacks) -} - -// SendBatchCalls gets all the calls that were made to SendBatch. -// Check the length with: -// -// len(mockedSender.SendBatchCalls()) -func (mock *SenderMock) SendBatchCalls() []struct { - URL string - Token string - Callbacks []*callbacker.Callback -} { - var calls []struct { - URL string - Token string - Callbacks []*callbacker.Callback - } - mock.lockSendBatch.RLock() - calls = mock.calls.SendBatch - mock.lockSendBatch.RUnlock() - return calls -} diff --git a/internal/callbacker/send_manager/ordered/mocks/store_mock.go b/internal/callbacker/send_manager/ordered/mocks/store_mock.go deleted file mode 100644 index d3695e3bb..000000000 --- a/internal/callbacker/send_manager/ordered/mocks/store_mock.go +++ /dev/null @@ -1,189 +0,0 @@ -// Code generated by moq; DO NOT EDIT. -// github.com/matryer/moq - -package mocks - -import ( - "context" - "github.com/bitcoin-sv/arc/internal/callbacker/send_manager/ordered" - "github.com/bitcoin-sv/arc/internal/callbacker/store" - "sync" -) - -// Ensure, that SendManagerStoreMock does implement ordered.SendManagerStore. -// If this is not the case, regenerate this file with moq. -var _ ordered.SendManagerStore = &SendManagerStoreMock{} - -// SendManagerStoreMock is a mock implementation of ordered.SendManagerStore. -// -// func TestSomethingThatUsesSendManagerStore(t *testing.T) { -// -// // make and configure a mocked ordered.SendManagerStore -// mockedSendManagerStore := &SendManagerStoreMock{ -// GetAndDeleteFunc: func(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) { -// panic("mock out the GetAndDelete method") -// }, -// SetFunc: func(ctx context.Context, dto *store.CallbackData) error { -// panic("mock out the Set method") -// }, -// SetManyFunc: func(ctx context.Context, data []*store.CallbackData) error { -// panic("mock out the SetMany method") -// }, -// } -// -// // use mockedSendManagerStore in code that requires ordered.SendManagerStore -// // and then make assertions. -// -// } -type SendManagerStoreMock struct { - // GetAndDeleteFunc mocks the GetAndDelete method. - GetAndDeleteFunc func(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) - - // SetFunc mocks the Set method. - SetFunc func(ctx context.Context, dto *store.CallbackData) error - - // SetManyFunc mocks the SetMany method. - SetManyFunc func(ctx context.Context, data []*store.CallbackData) error - - // calls tracks calls to the methods. - calls struct { - // GetAndDelete holds details about calls to the GetAndDelete method. - GetAndDelete []struct { - // Ctx is the ctx argument value. - Ctx context.Context - // URL is the url argument value. - URL string - // Limit is the limit argument value. - Limit int - } - // Set holds details about calls to the Set method. - Set []struct { - // Ctx is the ctx argument value. - Ctx context.Context - // Dto is the dto argument value. - Dto *store.CallbackData - } - // SetMany holds details about calls to the SetMany method. - SetMany []struct { - // Ctx is the ctx argument value. - Ctx context.Context - // Data is the data argument value. - Data []*store.CallbackData - } - } - lockGetAndDelete sync.RWMutex - lockSet sync.RWMutex - lockSetMany sync.RWMutex -} - -// GetAndDelete calls GetAndDeleteFunc. -func (mock *SendManagerStoreMock) GetAndDelete(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) { - if mock.GetAndDeleteFunc == nil { - panic("SendManagerStoreMock.GetAndDeleteFunc: method is nil but SendManagerStore.GetAndDelete was just called") - } - callInfo := struct { - Ctx context.Context - URL string - Limit int - }{ - Ctx: ctx, - URL: url, - Limit: limit, - } - mock.lockGetAndDelete.Lock() - mock.calls.GetAndDelete = append(mock.calls.GetAndDelete, callInfo) - mock.lockGetAndDelete.Unlock() - return mock.GetAndDeleteFunc(ctx, url, limit) -} - -// GetAndDeleteCalls gets all the calls that were made to GetAndDelete. -// Check the length with: -// -// len(mockedSendManagerStore.GetAndDeleteCalls()) -func (mock *SendManagerStoreMock) GetAndDeleteCalls() []struct { - Ctx context.Context - URL string - Limit int -} { - var calls []struct { - Ctx context.Context - URL string - Limit int - } - mock.lockGetAndDelete.RLock() - calls = mock.calls.GetAndDelete - mock.lockGetAndDelete.RUnlock() - return calls -} - -// Set calls SetFunc. -func (mock *SendManagerStoreMock) Set(ctx context.Context, dto *store.CallbackData) error { - if mock.SetFunc == nil { - panic("SendManagerStoreMock.SetFunc: method is nil but SendManagerStore.Set was just called") - } - callInfo := struct { - Ctx context.Context - Dto *store.CallbackData - }{ - Ctx: ctx, - Dto: dto, - } - mock.lockSet.Lock() - mock.calls.Set = append(mock.calls.Set, callInfo) - mock.lockSet.Unlock() - return mock.SetFunc(ctx, dto) -} - -// SetCalls gets all the calls that were made to Set. -// Check the length with: -// -// len(mockedSendManagerStore.SetCalls()) -func (mock *SendManagerStoreMock) SetCalls() []struct { - Ctx context.Context - Dto *store.CallbackData -} { - var calls []struct { - Ctx context.Context - Dto *store.CallbackData - } - mock.lockSet.RLock() - calls = mock.calls.Set - mock.lockSet.RUnlock() - return calls -} - -// SetMany calls SetManyFunc. -func (mock *SendManagerStoreMock) SetMany(ctx context.Context, data []*store.CallbackData) error { - if mock.SetManyFunc == nil { - panic("SendManagerStoreMock.SetManyFunc: method is nil but SendManagerStore.SetMany was just called") - } - callInfo := struct { - Ctx context.Context - Data []*store.CallbackData - }{ - Ctx: ctx, - Data: data, - } - mock.lockSetMany.Lock() - mock.calls.SetMany = append(mock.calls.SetMany, callInfo) - mock.lockSetMany.Unlock() - return mock.SetManyFunc(ctx, data) -} - -// SetManyCalls gets all the calls that were made to SetMany. -// Check the length with: -// -// len(mockedSendManagerStore.SetManyCalls()) -func (mock *SendManagerStoreMock) SetManyCalls() []struct { - Ctx context.Context - Data []*store.CallbackData -} { - var calls []struct { - Ctx context.Context - Data []*store.CallbackData - } - mock.lockSetMany.RLock() - calls = mock.calls.SetMany - mock.lockSetMany.RUnlock() - return calls -} diff --git a/internal/callbacker/send_manager/ordered/ordered_mocks.go b/internal/callbacker/send_manager/ordered/ordered_mocks.go deleted file mode 100644 index 1900542c2..000000000 --- a/internal/callbacker/send_manager/ordered/ordered_mocks.go +++ /dev/null @@ -1,5 +0,0 @@ -package ordered - -//go:generate moq -pkg mocks -out ./mocks/store_mock.go . SendManagerStore - -//go:generate moq -pkg mocks -out ./mocks/sender_mock.go . Sender diff --git a/internal/callbacker/send_manager/ordered/send_manager.go b/internal/callbacker/send_manager/ordered/send_manager.go deleted file mode 100644 index 04bb20671..000000000 --- a/internal/callbacker/send_manager/ordered/send_manager.go +++ /dev/null @@ -1,436 +0,0 @@ -package ordered - -import ( - "container/list" - "context" - "errors" - "log/slog" - "sync" - "time" - - "github.com/bitcoin-sv/arc/internal/callbacker" - "github.com/bitcoin-sv/arc/internal/callbacker/store" -) - -type SendManagerStore interface { - Set(ctx context.Context, dto *store.CallbackData) error - SetMany(ctx context.Context, data []*store.CallbackData) error - GetAndDelete(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) -} - -type Sender interface { - Send(url, token string, callback *callbacker.Callback) (success, retry bool) - SendBatch(url, token string, callbacks []*callbacker.Callback) (success, retry bool) -} - -type SendManager struct { - url string - - // dependencies - sender Sender - store SendManagerStore - logger *slog.Logger - - expiration time.Duration - - // internal state - entriesWg sync.WaitGroup - cancelAll context.CancelFunc - ctx context.Context - - queueProcessInterval time.Duration - backfillQueueInterval time.Duration - sortByTimestampInterval time.Duration - batchSendInterval time.Duration - batchSize int - - bufferSize int - callbackQueue *list.List - - now func() time.Time -} - -const ( - entriesBufferSize = 10000 - batchSizeDefault = 50 - queueProcessIntervalDefault = 5 * time.Second - backfillQueueIntervalDefault = 5 * time.Second - expirationDefault = 24 * time.Hour - sortByTimestampIntervalDefault = 10 * time.Second - batchSendIntervalDefault = 5 * time.Second -) - -var ( - ErrSendBatchedCallbacks = errors.New("failed to send batched callback") -) - -func WithNow(nowFunc func() time.Time) func(*SendManager) { - return func(m *SendManager) { - m.now = nowFunc - } -} - -func WithBufferSize(size int) func(*SendManager) { - return func(m *SendManager) { - m.bufferSize = size - } -} - -func WithQueueProcessInterval(d time.Duration) func(*SendManager) { - return func(m *SendManager) { - m.queueProcessInterval = d - } -} - -func WithBackfillQueueInterval(d time.Duration) func(*SendManager) { - return func(m *SendManager) { - m.backfillQueueInterval = d - } -} - -func WithExpiration(d time.Duration) func(*SendManager) { - return func(m *SendManager) { - m.expiration = d - } -} - -func WithSortByTimestampInterval(d time.Duration) func(*SendManager) { - return func(m *SendManager) { - m.sortByTimestampInterval = d - } -} - -func WithBatchSendInterval(d time.Duration) func(*SendManager) { - return func(m *SendManager) { - m.batchSendInterval = d - } -} - -func WithBatchSize(size int) func(*SendManager) { - return func(m *SendManager) { - m.batchSize = size - } -} - -func New(url string, sender callbacker.SenderI, store SendManagerStore, logger *slog.Logger, opts ...func(*SendManager)) *SendManager { - m := &SendManager{ - url: url, - sender: sender, - store: store, - logger: logger, - - queueProcessInterval: queueProcessIntervalDefault, - expiration: expirationDefault, - backfillQueueInterval: backfillQueueIntervalDefault, - sortByTimestampInterval: sortByTimestampIntervalDefault, - batchSendInterval: batchSendIntervalDefault, - batchSize: batchSizeDefault, - - callbackQueue: list.New(), - bufferSize: entriesBufferSize, - } - - for _, opt := range opts { - opt(m) - } - - ctx, cancelAll := context.WithCancel(context.Background()) - m.cancelAll = cancelAll - m.ctx = ctx - - return m -} - -func (m *SendManager) Enqueue(entry callbacker.CallbackEntry) { - if m.callbackQueue.Len() >= m.bufferSize { - m.storeToDB(entry) - return - } - - m.callbackQueue.PushBack(entry) -} - -func (m *SendManager) sortByTimestamp() error { - current := m.callbackQueue.Front() - if m.callbackQueue.Front() == nil { - return nil - } - for current != nil { - index := current.Next() - for index != nil { - currentTime, ok := current.Value.(callbacker.CallbackEntry) - if !ok { - return errors.New("callback entry is not a CallbackEntry") - } - - indexTime, ok := index.Value.(callbacker.CallbackEntry) - if !ok { - return errors.New("callback entry is not a CallbackEntry") - } - if currentTime.Data.Timestamp.Before(indexTime.Data.Timestamp) { - temp := current.Value - current.Value = index.Value - index.Value = temp - } - index = index.Next() - } - current = current.Next() - } - - return nil -} - -func (m *SendManager) CallbacksQueued() int { - return m.callbackQueue.Len() -} - -func (m *SendManager) Start() { - queueTicker := time.NewTicker(m.queueProcessInterval) - sortQueueTicker := time.NewTicker(m.sortByTimestampInterval) - backfillQueueTicker := time.NewTicker(m.backfillQueueInterval) - batchSendTicker := time.NewTicker(m.batchSendInterval) - - m.entriesWg.Add(1) - var callbackBatch []*list.Element - - go func() { - var err error - defer func() { - // read all from callback queue and store in database - data := make([]*store.CallbackData, m.callbackQueue.Len()+len(callbackBatch)) - - for _, callbackElement := range callbackBatch { - entry, ok := callbackElement.Value.(callbacker.CallbackEntry) - if !ok { - continue - } - m.callbackQueue.PushBack(entry) - } - - for i, entry := range m.dequeueAll() { - data[i] = toStoreDto(m.url, entry) - } - - if len(data) > 0 { - err = m.store.SetMany(context.Background(), data) - if err != nil { - m.logger.Error("Failed to set remaining callbacks from queue", slog.String("err", err.Error())) - } - } - - m.entriesWg.Done() - }() - - lastIterationWasBatch := false - - for { - select { - case <-m.ctx.Done(): - return - case <-sortQueueTicker.C: - err = m.sortByTimestamp() - if err != nil { - m.logger.Error("Failed to sort by timestamp", slog.String("err", err.Error())) - } - - case <-backfillQueueTicker.C: - m.backfillQueue() - - m.logger.Debug("Callback queue backfilled", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) - case <-batchSendTicker.C: - if len(callbackBatch) == 0 { - continue - } - - err = m.sendElementBatch(callbackBatch) - if err != nil { - m.logger.Error("Failed to send batch of callbacks", slog.String("url", m.url)) - continue - } - - callbackBatch = callbackBatch[:0] - m.logger.Debug("Batched callbacks sent on interval", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) - case <-queueTicker.C: - front := m.callbackQueue.Front() - if front == nil { - continue - } - - callbackEntry, ok := front.Value.(callbacker.CallbackEntry) - if !ok { - continue - } - - // If item is expired - dequeue without storing - if m.now().Sub(callbackEntry.Data.Timestamp) > m.expiration { - m.logger.Warn("Callback expired", slog.Time("timestamp", callbackEntry.Data.Timestamp), slog.String("hash", callbackEntry.Data.TxID), slog.String("status", callbackEntry.Data.TxStatus)) - m.callbackQueue.Remove(front) - continue - } - - if callbackEntry.AllowBatch { - lastIterationWasBatch = true - - if len(callbackBatch) < m.batchSize { - callbackBatch = append(callbackBatch, front) - queueTicker.Reset(m.queueProcessInterval) - m.callbackQueue.Remove(front) - continue - } - - err = m.sendElementBatch(callbackBatch) - if err != nil { - m.logger.Error("Failed to send batch of callbacks", slog.String("url", m.url)) - continue - } - - callbackBatch = callbackBatch[:0] - m.logger.Debug("Batched callbacks sent", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) - continue - } - - if lastIterationWasBatch { - lastIterationWasBatch = false - if len(callbackBatch) > 0 { - // if entry is not a batched entry, but last one was, send batch to keep the order - err = m.sendElementBatch(callbackBatch) - if err != nil { - m.logger.Error("Failed to send batch of callbacks", slog.String("url", m.url)) - continue - } - callbackBatch = callbackBatch[:0] - m.logger.Debug("Batched callbacks sent before sending single callback", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) - } - } - - success, retry := m.sender.Send(m.url, callbackEntry.Token, callbackEntry.Data) - if !retry || success { - m.callbackQueue.Remove(front) - m.logger.Debug("Single callback sent", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) - continue - } - m.logger.Error("Failed to send single callback", slog.String("url", m.url)) - } - } - }() -} - -func (m *SendManager) sendElementBatch(callbackElements []*list.Element) error { - var callbackElement *list.Element - callbackBatch := make([]callbacker.CallbackEntry, 0, len(callbackElements)) - for _, element := range callbackElements { - callback, ok := element.Value.(callbacker.CallbackEntry) - if !ok { - continue - } - callbackBatch = append(callbackBatch, callback) - } - success, retry := m.sendBatch(callbackBatch) - if !retry || success { - for _, callbackElement = range callbackElements { - m.callbackQueue.Remove(callbackElement) - } - - return nil - } - - return ErrSendBatchedCallbacks -} - -func (m *SendManager) sendBatch(batch []callbacker.CallbackEntry) (success, retry bool) { - token := batch[0].Token - callbacks := make([]*callbacker.Callback, len(batch)) - for i, e := range batch { - callbacks[i] = e.Data - } - - return m.sender.SendBatch(m.url, token, callbacks) -} - -func (m *SendManager) backfillQueue() { - capacityLeft := m.bufferSize - m.callbackQueue.Len() - if capacityLeft == 0 { - return - } - - callbacks, err := m.store.GetAndDelete(m.ctx, m.url, capacityLeft) - if err != nil { - m.logger.Error("Failed to load callbacks", slog.String("err", err.Error())) - return - } - - for _, callback := range callbacks { - m.Enqueue(toEntry(callback)) - } -} - -func (m *SendManager) storeToDB(entry callbacker.CallbackEntry) { - callbackData := toStoreDto(m.url, entry) - err := m.store.Set(context.Background(), callbackData) - if err != nil { - m.logger.Error("Failed to set callback data", slog.String("hash", callbackData.TxID), slog.String("status", callbackData.TxStatus), slog.String("err", err.Error())) - } -} - -func toStoreDto(url string, entry callbacker.CallbackEntry) *store.CallbackData { - return &store.CallbackData{ - URL: url, - Token: entry.Token, - Timestamp: entry.Data.Timestamp, - - CompetingTxs: entry.Data.CompetingTxs, - TxID: entry.Data.TxID, - TxStatus: entry.Data.TxStatus, - ExtraInfo: entry.Data.ExtraInfo, - MerklePath: entry.Data.MerklePath, - - BlockHash: entry.Data.BlockHash, - BlockHeight: entry.Data.BlockHeight, - - AllowBatch: entry.AllowBatch, - } -} - -func toEntry(callbackData *store.CallbackData) callbacker.CallbackEntry { - return callbacker.CallbackEntry{ - Token: callbackData.Token, - Data: &callbacker.Callback{ - Timestamp: callbackData.Timestamp, - CompetingTxs: callbackData.CompetingTxs, - TxID: callbackData.TxID, - TxStatus: callbackData.TxStatus, - ExtraInfo: callbackData.ExtraInfo, - MerklePath: callbackData.MerklePath, - BlockHash: callbackData.BlockHash, - BlockHeight: callbackData.BlockHeight, - }, - AllowBatch: callbackData.AllowBatch, - } -} - -func (m *SendManager) dequeueAll() []callbacker.CallbackEntry { - callbacks := make([]callbacker.CallbackEntry, 0, m.callbackQueue.Len()) - - var next *list.Element - for front := m.callbackQueue.Front(); front != nil; front = next { - next = front.Next() - entry, ok := front.Value.(callbacker.CallbackEntry) - if !ok { - continue - } - callbacks = append(callbacks, entry) - - m.callbackQueue.Remove(front) - } - - return callbacks -} - -// GracefulStop On service termination, any unsent callbacks are persisted in the store, ensuring no loss of data during shutdown. -func (m *SendManager) GracefulStop() { - if m.cancelAll != nil { - m.cancelAll() - } - - m.entriesWg.Wait() -} diff --git a/internal/callbacker/send_manager/ordered/send_manager_test.go b/internal/callbacker/send_manager/ordered/send_manager_test.go deleted file mode 100644 index d05eb52fb..000000000 --- a/internal/callbacker/send_manager/ordered/send_manager_test.go +++ /dev/null @@ -1,297 +0,0 @@ -package ordered_test - -import ( - "context" - "fmt" - "log/slog" - "math/rand" - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/bitcoin-sv/arc/internal/callbacker" - "github.com/bitcoin-sv/arc/internal/callbacker/send_manager/ordered" - "github.com/bitcoin-sv/arc/internal/callbacker/send_manager/ordered/mocks" - "github.com/bitcoin-sv/arc/internal/callbacker/store" -) - -var ( - now = time.Date(2025, 1, 10, 12, 0, 0, 0, time.UTC) - ts = time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC) - tsExpired = time.Date(2025, 1, 9, 12, 0, 0, 0, time.UTC) -) - -func TestSendManagerStart(t *testing.T) { - callbackEntries10 := make([]callbacker.CallbackEntry, 10) - for i := range 10 { - callbackEntries10[i] = callbacker.CallbackEntry{Data: &callbacker.Callback{Timestamp: ts}} - } - - callbackEntries15 := make([]callbacker.CallbackEntry, 15) - for i := range 15 { - callbackEntries15[i] = callbacker.CallbackEntry{Data: &callbacker.Callback{Timestamp: ts}} - } - - callbackEntries10Expired := make([]callbacker.CallbackEntry, 10) - for i := range 10 { - callbackEntries10Expired[i] = callbacker.CallbackEntry{Data: &callbacker.Callback{Timestamp: tsExpired}} - } - - callbackEntriesUnsorted := make([]callbacker.CallbackEntry, 10) - for i := range 10 { - callbackEntriesUnsorted[i] = callbacker.CallbackEntry{Data: &callbacker.Callback{Timestamp: time.Date(2025, 1, rand.Intn(31), rand.Intn(24), rand.Intn(59), 0, 0, time.UTC)}} - } - - callbackEntriesBatch3Expired10 := make([]callbacker.CallbackEntry, 10) - for i := range 10 { - if i >= 3 && i < 6 { - callbackEntriesBatch3Expired10[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: tsExpired}, AllowBatch: true} - continue - } - callbackEntriesBatch3Expired10[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: ts}, AllowBatch: true} - } - - callbackEntriesBatch10 := make([]callbacker.CallbackEntry, 10) - for i := range 10 { - callbackEntriesBatch10[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: ts}, AllowBatch: true} - } - - callbackEntriesBatched10Mixed := make([]callbacker.CallbackEntry, 10) - for i := range 10 { - if i >= 3 && i < 7 { - callbackEntriesBatched10Mixed[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: ts}, AllowBatch: true} - continue - } - callbackEntriesBatched10Mixed[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: ts}, AllowBatch: false} - } - - callbackEntriesBatched10Mixed2 := make([]callbacker.CallbackEntry, 10) - for i := range 10 { - if i >= 2 && i < 8 { - callbackEntriesBatched10Mixed2[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: ts}, AllowBatch: true} - continue - } - callbackEntriesBatched10Mixed2[i] = callbacker.CallbackEntry{Token: fmt.Sprintf("token: %d", i), Data: &callbacker.Callback{Timestamp: ts}, AllowBatch: false} - } - - tcs := []struct { - name string - callbacksEnqueued []callbacker.CallbackEntry - queueProcessInterval time.Duration - backfillInterval time.Duration - sortByTimestampInterval time.Duration - batchInterval time.Duration - - expectedCallbacksEnqueued int - expectedSetMany int - expectedSetCalls int - expectedSendCalls int - expectedSendBatchCalls []int - }{ - { - name: "enqueue 10 callbacks - 10ms interval", - callbacksEnqueued: callbackEntries10, - queueProcessInterval: 10 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - sortByTimestampInterval: 500 * time.Millisecond, - batchInterval: 500 * time.Millisecond, - - expectedCallbacksEnqueued: 10, - expectedSetCalls: 0, - expectedSendCalls: 10, - }, - { - name: "enqueue 10 callbacks - 100ms interval - store remaining at graceful stop", - callbacksEnqueued: callbackEntries10, - queueProcessInterval: 100 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - sortByTimestampInterval: 500 * time.Millisecond, - batchInterval: 500 * time.Millisecond, - - expectedCallbacksEnqueued: 10, - expectedSetMany: 9, - expectedSetCalls: 0, - expectedSendCalls: 1, - }, - { - name: "enqueue 10 callbacks - expired", - callbacksEnqueued: callbackEntries10Expired, - queueProcessInterval: 10 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - sortByTimestampInterval: 500 * time.Millisecond, - batchInterval: 500 * time.Millisecond, - - expectedCallbacksEnqueued: 10, - expectedSetCalls: 0, - expectedSendCalls: 0, - }, - { - name: "enqueue 15 callbacks - buffer size reached", - callbacksEnqueued: callbackEntries15, - queueProcessInterval: 10 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - sortByTimestampInterval: 500 * time.Millisecond, - batchInterval: 500 * time.Millisecond, - - expectedCallbacksEnqueued: 10, - expectedSetCalls: 5, - expectedSendCalls: 10, - }, - { - name: "enqueue 10 callbacks - back fill queue", - callbacksEnqueued: callbackEntries10, - queueProcessInterval: 10 * time.Millisecond, - backfillInterval: 20 * time.Millisecond, - sortByTimestampInterval: 500 * time.Millisecond, - batchInterval: 500 * time.Millisecond, - - expectedCallbacksEnqueued: 10, - expectedSetMany: 9, - expectedSetCalls: 0, - expectedSendCalls: 15, - }, - { - name: "enqueue 10 callbacks - sort by timestamp", - callbacksEnqueued: callbackEntriesUnsorted, - queueProcessInterval: 200 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - sortByTimestampInterval: 20 * time.Millisecond, - batchInterval: 500 * time.Millisecond, - - expectedCallbacksEnqueued: 10, - expectedSetMany: 10, - expectedSetCalls: 0, - expectedSendCalls: 0, - }, - { - name: "enqueue 10 batched callbacks - 3 expired", - callbacksEnqueued: callbackEntriesBatch3Expired10, - queueProcessInterval: 10 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - sortByTimestampInterval: 500 * time.Millisecond, - batchInterval: 500 * time.Millisecond, - - expectedCallbacksEnqueued: 10, - expectedSetMany: 2, - expectedSetCalls: 0, - expectedSendCalls: 0, - expectedSendBatchCalls: []int{5}, - }, - { - name: "enqueue 10 batched callbacks - 60ms batch interval", - callbacksEnqueued: callbackEntriesBatch10, - queueProcessInterval: 10 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - sortByTimestampInterval: 500 * time.Millisecond, - batchInterval: 60 * time.Millisecond, - - expectedCallbacksEnqueued: 10, - expectedSetCalls: 0, - expectedSendCalls: 0, - expectedSendBatchCalls: []int{5, 5}, - }, - { - name: "enqueue 10 batched callbacks - 4 batched, 6 single", - callbacksEnqueued: callbackEntriesBatched10Mixed, - queueProcessInterval: 10 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - sortByTimestampInterval: 500 * time.Millisecond, - batchInterval: 500 * time.Millisecond, - - expectedCallbacksEnqueued: 10, - expectedSetCalls: 0, - expectedSendCalls: 6, - expectedSendBatchCalls: []int{4}, - }, - { - name: "enqueue 10 batched callbacks - 6 batched, 4 single", - callbacksEnqueued: callbackEntriesBatched10Mixed2, - queueProcessInterval: 10 * time.Millisecond, - backfillInterval: 500 * time.Millisecond, - sortByTimestampInterval: 500 * time.Millisecond, - batchInterval: 500 * time.Millisecond, - - expectedCallbacksEnqueued: 10, - expectedSetCalls: 0, - expectedSendCalls: 4, - expectedSendBatchCalls: []int{5, 1}, - }, - } - - for _, tc := range tcs { - t.Run(tc.name, func(t *testing.T) { - // given - - counter := 0 - senderMock := &mocks.SenderMock{ - SendFunc: func(_, _ string, _ *callbacker.Callback) (bool, bool) { return true, false }, - SendBatchFunc: func(_, _ string, batch []*callbacker.Callback) (bool, bool) { - if counter >= len(tc.expectedSendBatchCalls) { - t.Fail() - } else { - assert.Equal(t, tc.expectedSendBatchCalls[counter], len(batch)) - counter++ - } - return true, false - }, - } - - storeMock := &mocks.SendManagerStoreMock{ - SetManyFunc: func(_ context.Context, data []*store.CallbackData) error { - assert.Equal(t, tc.expectedSetMany, len(data)) - - for i := 0; i < len(data)-1; i++ { - assert.GreaterOrEqual(t, data[i].Timestamp, data[i+1].Timestamp) - } - - return nil - }, - SetFunc: func(_ context.Context, _ *store.CallbackData) error { - return nil - }, - GetAndDeleteFunc: func(_ context.Context, _ string, limit int) ([]*store.CallbackData, error) { - var callbacks []*store.CallbackData - for range limit { - callbacks = append(callbacks, &store.CallbackData{Timestamp: time.Date(2025, 1, 10, 11, 30, 0, 0, time.UTC)}) - } - return callbacks, nil - }, - } - - logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug})) - - sut := ordered.New("https://abcdefg.com", senderMock, storeMock, logger, - ordered.WithBufferSize(10), - ordered.WithNow(func() time.Time { - return now - }), - ordered.WithQueueProcessInterval(tc.queueProcessInterval), - ordered.WithExpiration(time.Hour), - ordered.WithBackfillQueueInterval(tc.backfillInterval), - ordered.WithSortByTimestampInterval(tc.sortByTimestampInterval), - ordered.WithBatchSendInterval(tc.batchInterval), - ordered.WithBatchSize(5), - ) - - // add callbacks before starting the manager to queue them - for _, cb := range tc.callbacksEnqueued { - sut.Enqueue(cb) - } - require.Equal(t, tc.expectedCallbacksEnqueued, sut.CallbacksQueued()) - - sut.Start() - - time.Sleep(150 * time.Millisecond) - sut.GracefulStop() - - assert.Equal(t, 0, sut.CallbacksQueued()) - assert.Equal(t, tc.expectedSetMany > 0, len(storeMock.SetManyCalls()) == 1) - assert.Equal(t, tc.expectedSetCalls, len(storeMock.SetCalls())) - assert.Equal(t, tc.expectedSendCalls, len(senderMock.SendCalls())) - assert.Equal(t, len(tc.expectedSendBatchCalls), len(senderMock.SendBatchCalls())) - }) - } -} From b55877a8b679fb6047d083494f543272b562c56a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Thu, 23 Jan 2025 09:46:57 +0100 Subject: [PATCH 12/18] feat(ARCO-291): Ordered callbacks --- cmd/arc/services/callbacker.go | 52 +-- config/config.go | 21 +- config/defaults.go | 14 +- config/example_config.yaml | 11 +- docker-compose.yaml | 3 +- internal/callbacker/background_workers.go | 160 ------- .../callbacker/background_workers_test.go | 201 --------- internal/callbacker/callbacker.go | 6 + .../callbacker_api/callbacker_api.pb.go | 169 ++++---- .../callbacker_api/callbacker_api.proto | 1 + internal/callbacker/callbacker_mocks.go | 7 +- internal/callbacker/dispatcher.go | 50 +-- internal/callbacker/dispatcher_test.go | 67 +-- internal/callbacker/mocks/dipatcher_mock.go | 30 +- .../callbacker/mocks/processor_store_mock.go | 151 +++++++ .../callbacker/mocks/send_manager_mock.go | 149 +++++++ .../{callbacker_mock.go => sender_mock.go} | 0 internal/callbacker/mocks/store_mock.go | 326 -------------- internal/callbacker/processor.go | 160 ++++++- internal/callbacker/processor_test.go | 122 +++++- internal/callbacker/send_manager.go | 354 --------------- internal/callbacker/send_manager_test.go | 306 ------------- internal/callbacker/server.go | 6 +- internal/callbacker/server_test.go | 55 +-- .../callbacker.callbacks.yaml | 23 +- .../callbacker.callbacks.yaml | 0 .../pop_failed_many/callbacker.callbacks.yaml | 408 ------------------ .../store/postgresql/internal/tests/utils.go | 35 +- .../000005_drop_postponed_until.down.sql | 3 + .../000005_drop_postponed_until.up.sql | 3 + .../callbacker/store/postgresql/postgres.go | 162 ++----- .../store/postgresql/postgres_test.go | 164 ++----- internal/callbacker/store/store.go | 15 +- internal/metamorph/processor.go | 4 +- internal/metamorph/processor_helpers.go | 9 +- .../nats_jetstream/nats_jetstream_client.go | 2 +- test/config/config.yaml | 13 +- 37 files changed, 841 insertions(+), 2421 deletions(-) delete mode 100644 internal/callbacker/background_workers.go delete mode 100644 internal/callbacker/background_workers_test.go create mode 100644 internal/callbacker/mocks/send_manager_mock.go rename internal/callbacker/mocks/{callbacker_mock.go => sender_mock.go} (100%) delete mode 100644 internal/callbacker/mocks/store_mock.go delete mode 100644 internal/callbacker/send_manager.go delete mode 100644 internal/callbacker/send_manager_test.go rename internal/callbacker/store/postgresql/fixtures/{delete_failed_older_than => delete_older_than}/callbacker.callbacks.yaml (94%) rename internal/callbacker/store/postgresql/fixtures/{pop_many => get_and_delete}/callbacker.callbacks.yaml (100%) delete mode 100644 internal/callbacker/store/postgresql/fixtures/pop_failed_many/callbacker.callbacks.yaml create mode 100644 internal/callbacker/store/postgresql/migrations/000005_drop_postponed_until.down.sql create mode 100644 internal/callbacker/store/postgresql/migrations/000005_drop_postponed_until.up.sql diff --git a/cmd/arc/services/callbacker.go b/cmd/arc/services/callbacker.go index 6566dae74..033ef6e1a 100644 --- a/cmd/arc/services/callbacker.go +++ b/cmd/arc/services/callbacker.go @@ -31,7 +31,7 @@ import ( "github.com/bitcoin-sv/arc/config" "github.com/bitcoin-sv/arc/internal/callbacker" - "github.com/bitcoin-sv/arc/internal/callbacker/store" + "github.com/bitcoin-sv/arc/internal/callbacker/send_manager" "github.com/bitcoin-sv/arc/internal/callbacker/store/postgresql" "github.com/bitcoin-sv/arc/internal/grpc_opts" "github.com/bitcoin-sv/arc/pkg/message_queue/nats/client/nats_jetstream" @@ -48,7 +48,6 @@ func StartCallbacker(logger *slog.Logger, arcConfig *config.ArcConfig) (func(), callbackerStore *postgresql.PostgreSQL sender *callbacker.CallbackSender dispatcher *callbacker.CallbackDispatcher - workers *callbacker.BackgroundWorkers server *callbacker.Server healthServer *grpc_opts.GrpcServer mqClient callbacker.MessageQueueClient @@ -58,7 +57,7 @@ func StartCallbacker(logger *slog.Logger, arcConfig *config.ArcConfig) (func(), stopFn := func() { logger.Info("Shutting down callbacker") - dispose(logger, server, workers, dispatcher, sender, callbackerStore, healthServer, processor, mqClient) + dispose(logger, server, dispatcher, sender, callbackerStore, healthServer, processor, mqClient) logger.Info("Shutdown complete") } @@ -73,24 +72,18 @@ func StartCallbacker(logger *slog.Logger, arcConfig *config.ArcConfig) (func(), return nil, fmt.Errorf("failed to create callback sender: %v", err) } - sendConfig := callbacker.SendConfig{ - Expiration: cfg.Expiration, - Delay: cfg.Delay, - DelayDuration: cfg.DelayDuration, - PauseAfterSingleModeSuccessfulSend: cfg.Pause, - BatchSendInterval: cfg.BatchSendInterval, - } + runNewManager := func(url string) callbacker.SendManagerI { + manager := send_manager.New(url, sender, callbackerStore, logger, + send_manager.WithQueueProcessInterval(cfg.Pause), + send_manager.WithBatchSendInterval(cfg.BatchSendInterval), + send_manager.WithExpiration(cfg.Expiration), + ) + manager.Start() - dispatcher = callbacker.NewCallbackDispatcher(sender, callbackerStore, logger, &sendConfig) - workers = callbacker.NewBackgroundWorkers(callbackerStore, dispatcher, logger) - err = workers.DispatchPersistedCallbacks() - if err != nil { - stopFn() - return nil, fmt.Errorf("failed to dispatch previously persisted callbacks: %v", err) + return manager } - workers.StartCallbackStoreCleanup(cfg.PruneInterval, cfg.PruneOlderThan) - workers.StartFailedCallbacksDispatch(cfg.FailedCallbackCheckInterval) + dispatcher = callbacker.NewCallbackDispatcher(sender, runNewManager) natsConnection, err := nats_connection.New(arcConfig.MessageQueue.URL, logger) if err != nil { @@ -119,6 +112,9 @@ func StartCallbacker(logger *slog.Logger, arcConfig *config.ArcConfig) (func(), return nil, err } + processor.StartCallbackStoreCleanup(cfg.PruneInterval, cfg.PruneOlderThan) + processor.DispatchPersistedCallbacks() + err = processor.Start() if err != nil { stopFn() @@ -168,37 +164,31 @@ func newStore(dbConfig *config.DbConfig) (s *postgresql.PostgreSQL, err error) { return s, err } -func dispose(l *slog.Logger, server *callbacker.Server, workers *callbacker.BackgroundWorkers, +func dispose(l *slog.Logger, server *callbacker.Server, dispatcher *callbacker.CallbackDispatcher, sender *callbacker.CallbackSender, - store store.CallbackerStore, healthServer *grpc_opts.GrpcServer, processor *callbacker.Processor, mqClient callbacker.MessageQueueClient) { + store *postgresql.PostgreSQL, healthServer *grpc_opts.GrpcServer, processor *callbacker.Processor, mqClient callbacker.MessageQueueClient) { // dispose the dependencies in the correct order: // 1. server - ensure no new callbacks will be received - // 2. background workers - ensure no callbacks from background will be accepted - // 3. dispatcher - ensure all already accepted callbacks are proccessed + // 2. dispatcher - ensure all already accepted callbacks are processed + // 3. processor - remove all URL mappings // 4. sender - finally, stop the sender as there are no callbacks left to send // 5. store if server != nil { server.GracefulStop() } - if workers != nil { - workers.GracefulStop() - } if dispatcher != nil { dispatcher.GracefulStop() } - if sender != nil { - sender.GracefulStop() - } - if processor != nil { processor.GracefulStop() } - + if sender != nil { + sender.GracefulStop() + } if mqClient != nil { mqClient.Shutdown() } - if store != nil { err := store.Close() if err != nil { diff --git a/config/config.go b/config/config.go index 0808b6011..1462964a8 100644 --- a/config/config.go +++ b/config/config.go @@ -199,16 +199,13 @@ type K8sWatcherConfig struct { } type CallbackerConfig struct { - ListenAddr string `mapstructure:"listenAddr"` - DialAddr string `mapstructure:"dialAddr"` - Health *HealthConfig `mapstructure:"health"` - Delay time.Duration `mapstructure:"delay"` - Pause time.Duration `mapstructure:"pause"` - BatchSendInterval time.Duration `mapstructure:"batchSendInterval"` - Db *DbConfig `mapstructure:"db"` - PruneInterval time.Duration `mapstructure:"pruneInterval"` - PruneOlderThan time.Duration `mapstructure:"pruneOlderThan"` - DelayDuration time.Duration `mapstructure:"delayDuration"` - FailedCallbackCheckInterval time.Duration `mapstructure:"failedCallbackCheckInterval"` - Expiration time.Duration `mapstructure:"expiration"` + ListenAddr string `mapstructure:"listenAddr"` + DialAddr string `mapstructure:"dialAddr"` + Health *HealthConfig `mapstructure:"health"` + Pause time.Duration `mapstructure:"pause"` + BatchSendInterval time.Duration `mapstructure:"batchSendInterval"` + PruneOlderThan time.Duration `mapstructure:"pruneOlderThan"` + PruneInterval time.Duration `mapstructure:"pruneInterval"` + Expiration time.Duration `mapstructure:"expiration"` + Db *DbConfig `mapstructure:"db"` } diff --git a/config/defaults.go b/config/defaults.go index 8084652ee..5c2feebf5 100644 --- a/config/defaults.go +++ b/config/defaults.go @@ -184,14 +184,12 @@ func getCallbackerConfig() *CallbackerConfig { Health: &HealthConfig{ SeverDialAddr: "localhost:8025", }, - Delay: 0, - Pause: 0, - BatchSendInterval: time.Duration(5 * time.Second), - Db: getDbConfig("callbacker"), - PruneInterval: 24 * time.Hour, - PruneOlderThan: 14 * 24 * time.Hour, - FailedCallbackCheckInterval: time.Minute, - Expiration: 24 * time.Hour, + Pause: 0, + BatchSendInterval: 5 * time.Second, + PruneOlderThan: 14 * 24 * time.Hour, + PruneInterval: 24 * time.Hour, + Expiration: 24 * time.Hour, + Db: getDbConfig("callbacker"), } } diff --git a/config/example_config.yaml b/config/example_config.yaml index 0a7e28ad6..7d3781ade 100644 --- a/config/example_config.yaml +++ b/config/example_config.yaml @@ -180,9 +180,11 @@ callbacker: dialAddr: localhost:8021 # address for other services to dial callbacker service health: serverDialAddr: localhost:8025 # address at which the grpc health server is exposed - delay: 0s # delay before the callback (or batch of callbacks) is actually sent - pause: 0s # pause between sending next callback to the same receiver + pause: 1s # pause between sending next callback to the same receiver - must be greater 0s batchSendInterval: 5s # interval at witch batched callbacks are send (default 5s) + pruneOlderThan: 336h # age threshold for pruning callbacks (older than this value will be removed) + pruneInterval: 24h # interval at which old or failed callbacks are pruned from the store + expiration: 24h # maximum time a callback can remain unsent before it's put as permanently failed db: mode: postgres # db mode indicates which db to use. At the moment only postgres is offered postgres: # postgres db configuration in case that mode: postgres @@ -194,8 +196,3 @@ callbacker: maxIdleConns: 10 # maximum idle connections maxOpenConns: 80 # maximum open connections sslMode: disable - pruneInterval: 24h # interval at which old or failed callbacks are pruned from the store - pruneOlderThan: 336h # age threshold for pruning callbacks (older than this value will be removed) - failedCallbackCheckInterval: 1m # interval at which the store is checked for failed callbacks to be re-sent - delayDuration: 5s # we try callbacks a few times with this delay after which if it fails consistently we store them in db - expiration: 24h # maximum time a callback can remain unsent before it's put as permanently failed diff --git a/docker-compose.yaml b/docker-compose.yaml index e135323ef..028d877e0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -180,7 +180,6 @@ services: condition: service_healthy migrate-blocktx: condition: service_completed_successfully - healthcheck: test: ["CMD", "/bin/grpc_health_probe", "-addr=:8006", "-service=liveness", "-rpc-timeout=5s"] interval: 10s @@ -204,7 +203,7 @@ services: migrate-callbacker: condition: service_completed_successfully healthcheck: - test: ["CMD", "/bin/grpc_health_probe", "-addr=:8022", "-service=liveness", "-rpc-timeout=5s"] + test: ["CMD", "/bin/grpc_health_probe", "-addr=:8025", "-service=liveness", "-rpc-timeout=5s"] interval: 10s timeout: 5s retries: 3 diff --git a/internal/callbacker/background_workers.go b/internal/callbacker/background_workers.go deleted file mode 100644 index a5cff73c0..000000000 --- a/internal/callbacker/background_workers.go +++ /dev/null @@ -1,160 +0,0 @@ -package callbacker - -import ( - "context" - "errors" - "log/slog" - "sync" - "time" - - "github.com/bitcoin-sv/arc/internal/callbacker/store" -) - -var ( - ErrFailedPopMany = errors.New("failed to pop many") -) - -type BackgroundWorkers struct { - callbackerStore store.CallbackerStore - logger *slog.Logger - dispatcher *CallbackDispatcher - - workersWg sync.WaitGroup - ctx context.Context - cancelAll func() -} - -func NewBackgroundWorkers(s store.CallbackerStore, dispatcher *CallbackDispatcher, logger *slog.Logger) *BackgroundWorkers { - ctx, cancel := context.WithCancel(context.Background()) - - return &BackgroundWorkers{ - callbackerStore: s, - dispatcher: dispatcher, - logger: logger.With(slog.String("module", "background workers")), - - ctx: ctx, - cancelAll: cancel, - } -} - -func (w *BackgroundWorkers) StartCallbackStoreCleanup(interval, olderThanDuration time.Duration) { - ctx := context.Background() - ticker := time.NewTicker(interval) - - w.workersWg.Add(1) - go func() { - for { - select { - case <-ticker.C: - n := time.Now() - midnight := time.Date(n.Year(), n.Month(), n.Day(), 0, 0, 0, 0, time.UTC) - olderThan := midnight.Add(-1 * olderThanDuration) - - err := w.callbackerStore.DeleteFailedOlderThan(ctx, olderThan) - if err != nil { - w.logger.Error("Failed to delete old callbacks in delay", slog.String("err", err.Error())) - } - - case <-w.ctx.Done(): - w.workersWg.Done() - return - } - } - }() -} - -func (w *BackgroundWorkers) StartFailedCallbacksDispatch(interval time.Duration) { - const batchSize = 100 - - ctx := context.Background() - ticker := time.NewTicker(interval) - - w.workersWg.Add(1) - go func() { - for { - select { - case <-ticker.C: - - callbacks, err := w.callbackerStore.PopFailedMany(ctx, time.Now(), batchSize) - if err != nil { - w.logger.Error("Failed to load callbacks from store", slog.String("err", err.Error())) - continue - } - - if len(callbacks) == 0 { - continue - } - w.logger.Info("Loaded callbacks from store", slog.Any("count", len(callbacks))) - - for _, c := range callbacks { - callbackEntry := &CallbackEntry{ - Token: c.Token, - Data: toCallback(c), - postponedUntil: c.PostponedUntil, - } - - w.dispatcher.Dispatch(c.URL, callbackEntry, c.AllowBatch) - } - - case <-w.ctx.Done(): - w.workersWg.Done() - return - } - } - }() -} - -func (w *BackgroundWorkers) DispatchPersistedCallbacks() error { - w.logger.Info("Load and dispatch stored callbacks") - - const batchSize = 100 - ctx := context.Background() - - for { - callbacks, err := w.callbackerStore.PopMany(ctx, batchSize) - if err != nil { - return errors.Join(ErrFailedPopMany, err) - } - - if len(callbacks) == 0 { - return nil - } - - for _, c := range callbacks { - callbackEntry := &CallbackEntry{ - Token: c.Token, - Data: toCallback(c), - } - - w.dispatcher.Dispatch(c.URL, callbackEntry, c.AllowBatch) - } - - time.Sleep(500 * time.Millisecond) - } -} - -func (w *BackgroundWorkers) GracefulStop() { - w.logger.Info("Shutting down") - - w.cancelAll() - w.workersWg.Wait() - - w.logger.Info("Shutdown complete") -} - -func toCallback(dto *store.CallbackData) *Callback { - d := &Callback{ - Timestamp: dto.Timestamp, - - CompetingTxs: dto.CompetingTxs, - TxID: dto.TxID, - TxStatus: dto.TxStatus, - ExtraInfo: dto.ExtraInfo, - MerklePath: dto.MerklePath, - - BlockHash: dto.BlockHash, - BlockHeight: dto.BlockHeight, - } - - return d -} diff --git a/internal/callbacker/background_workers_test.go b/internal/callbacker/background_workers_test.go deleted file mode 100644 index 36a4e0c35..000000000 --- a/internal/callbacker/background_workers_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package callbacker_test - -import ( - "context" - "errors" - "log/slog" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/bitcoin-sv/arc/internal/callbacker" - "github.com/bitcoin-sv/arc/internal/callbacker/mocks" - "github.com/bitcoin-sv/arc/internal/callbacker/store" -) - -func TestStartCallbackStoreCleanup(t *testing.T) { - tt := []struct { - name string - deleteFailedOlderThanErr error - - expectedIterations int - }{ - { - name: "success", - - expectedIterations: 4, - }, - { - name: "error deleting failed older than", - deleteFailedOlderThanErr: errors.New("some error"), - - expectedIterations: 4, - }, - } - - for _, tc := range tt { - t.Run(tc.name, func(t *testing.T) { - cbStore := &mocks.CallbackerStoreMock{ - DeleteFailedOlderThanFunc: func(_ context.Context, _ time.Time) error { - return tc.deleteFailedOlderThanErr - }, - } - logger := slog.Default() - - bw := callbacker.NewBackgroundWorkers(cbStore, nil, logger) - defer bw.GracefulStop() - - bw.StartCallbackStoreCleanup(20*time.Millisecond, 50*time.Second) - - time.Sleep(90 * time.Millisecond) - - require.Equal(t, tc.expectedIterations, len(cbStore.DeleteFailedOlderThanCalls())) - }) - } -} - -func TestStartFailedCallbacksDispatch(t *testing.T) { - tt := []struct { - name string - storedCallbacks []*store.CallbackData - popFailedManyErr error - - expectedIterations int - expectedSends int - }{ - { - name: "success", - storedCallbacks: []*store.CallbackData{{ - URL: "https://test.com", - Token: "1234", - }}, - - expectedIterations: 4, - expectedSends: 4, - }, - { - name: "success - no callbacks in store", - storedCallbacks: []*store.CallbackData{}, - - expectedIterations: 4, - expectedSends: 0, - }, - { - name: "error getting stored callbacks", - storedCallbacks: []*store.CallbackData{}, - popFailedManyErr: errors.New("some error"), - - expectedIterations: 4, - expectedSends: 0, - }, - } - - for _, tc := range tt { - t.Run(tc.name, func(t *testing.T) { - cbStore := &mocks.CallbackerStoreMock{ - PopFailedManyFunc: func(_ context.Context, _ time.Time, limit int) ([]*store.CallbackData, error) { - require.Equal(t, 100, limit) - return tc.storedCallbacks, tc.popFailedManyErr - }, - SetFunc: func(_ context.Context, _ *store.CallbackData) error { - return nil - }, - } - sender := &mocks.SenderIMock{ - SendFunc: func(_ string, _ string, _ *callbacker.Callback) (bool, bool) { - return true, false - }, - } - logger := slog.Default() - dispatcher := callbacker.NewCallbackDispatcher(sender, cbStore, logger, &callbacker.SendConfig{}) - - bw := callbacker.NewBackgroundWorkers(cbStore, dispatcher, logger) - defer bw.GracefulStop() - - bw.StartFailedCallbacksDispatch(20 * time.Millisecond) - - time.Sleep(90 * time.Millisecond) - - require.Equal(t, tc.expectedIterations, len(cbStore.PopFailedManyCalls())) - require.Equal(t, tc.expectedSends, len(sender.SendCalls())) - }) - } -} - -func TestDispatchPersistedCallbacks(t *testing.T) { - tt := []struct { - name string - storedCallbacks []*store.CallbackData - popManyErr error - - expectedErr error - expectedIterations int - expectedSends int - }{ - { - name: "success - no stored callbacks", - storedCallbacks: []*store.CallbackData{}, - - expectedIterations: 1, - }, - { - name: "success - 1 stored callback", - storedCallbacks: []*store.CallbackData{{ - URL: "https://test.com", - Token: "1234", - }}, - - expectedIterations: 2, - expectedSends: 1, - }, - { - name: "error deleting failed older than", - popManyErr: errors.New("some error"), - - expectedErr: callbacker.ErrFailedPopMany, - expectedIterations: 1, - }, - } - - for _, tc := range tt { - t.Run(tc.name, func(t *testing.T) { - counter := 0 - - cbStore := &mocks.CallbackerStoreMock{ - PopManyFunc: func(_ context.Context, limit int) ([]*store.CallbackData, error) { - require.Equal(t, 100, limit) - if counter > 0 { - return []*store.CallbackData{}, nil - } - counter++ - - return tc.storedCallbacks, tc.popManyErr - }, - SetFunc: func(_ context.Context, _ *store.CallbackData) error { - return nil - }, - } - sender := &mocks.SenderIMock{ - SendFunc: func(_ string, _ string, _ *callbacker.Callback) (bool, bool) { - return true, false - }, - } - logger := slog.Default() - dispatcher := callbacker.NewCallbackDispatcher(sender, cbStore, logger, &callbacker.SendConfig{}) - - bw := callbacker.NewBackgroundWorkers(cbStore, dispatcher, logger) - defer bw.GracefulStop() - - actualError := bw.DispatchPersistedCallbacks() - - require.Equal(t, tc.expectedIterations, len(cbStore.PopManyCalls())) - require.Equal(t, tc.expectedSends, len(sender.SendCalls())) - if tc.expectedErr == nil { - require.NoError(t, actualError) - return - } - require.ErrorIs(t, actualError, tc.expectedErr) - }) - } -} diff --git a/internal/callbacker/callbacker.go b/internal/callbacker/callbacker.go index c2fa3704f..326da5729 100644 --- a/internal/callbacker/callbacker.go +++ b/internal/callbacker/callbacker.go @@ -9,6 +9,12 @@ type SenderI interface { SendBatch(url, token string, callbacks []*Callback) (success, retry bool) } +type SendManagerI interface { + Enqueue(entry CallbackEntry) + Start() + GracefulStop() +} + type Callback struct { Timestamp time.Time `json:"timestamp"` diff --git a/internal/callbacker/callbacker_api/callbacker_api.pb.go b/internal/callbacker/callbacker_api/callbacker_api.pb.go index 9604ddab1..74f700ee2 100644 --- a/internal/callbacker/callbacker_api/callbacker_api.pb.go +++ b/internal/callbacker/callbacker_api/callbacker_api.pb.go @@ -161,14 +161,15 @@ type SendRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - CallbackRouting *CallbackRouting `protobuf:"bytes,1,opt,name=callback_routing,json=callbackRouting,proto3" json:"callback_routing,omitempty"` - Txid string `protobuf:"bytes,2,opt,name=txid,proto3" json:"txid,omitempty"` - Status Status `protobuf:"varint,3,opt,name=status,proto3,enum=callbacker_api.Status" json:"status,omitempty"` - MerklePath string `protobuf:"bytes,4,opt,name=merkle_path,json=merklePath,proto3" json:"merkle_path,omitempty"` - ExtraInfo string `protobuf:"bytes,5,opt,name=extra_info,json=extraInfo,proto3" json:"extra_info,omitempty"` - CompetingTxs []string `protobuf:"bytes,6,rep,name=competing_txs,json=competingTxs,proto3" json:"competing_txs,omitempty"` - BlockHash string `protobuf:"bytes,7,opt,name=block_hash,json=blockHash,proto3" json:"block_hash,omitempty"` - BlockHeight uint64 `protobuf:"varint,8,opt,name=block_height,json=blockHeight,proto3" json:"block_height,omitempty"` + CallbackRouting *CallbackRouting `protobuf:"bytes,1,opt,name=callback_routing,json=callbackRouting,proto3" json:"callback_routing,omitempty"` + Txid string `protobuf:"bytes,2,opt,name=txid,proto3" json:"txid,omitempty"` + Status Status `protobuf:"varint,3,opt,name=status,proto3,enum=callbacker_api.Status" json:"status,omitempty"` + MerklePath string `protobuf:"bytes,4,opt,name=merkle_path,json=merklePath,proto3" json:"merkle_path,omitempty"` + ExtraInfo string `protobuf:"bytes,5,opt,name=extra_info,json=extraInfo,proto3" json:"extra_info,omitempty"` + CompetingTxs []string `protobuf:"bytes,6,rep,name=competing_txs,json=competingTxs,proto3" json:"competing_txs,omitempty"` + BlockHash string `protobuf:"bytes,7,opt,name=block_hash,json=blockHash,proto3" json:"block_hash,omitempty"` + BlockHeight uint64 `protobuf:"varint,8,opt,name=block_height,json=blockHeight,proto3" json:"block_height,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,9,opt,name=timestamp,proto3" json:"timestamp,omitempty"` } func (x *SendRequest) Reset() { @@ -259,6 +260,13 @@ func (x *SendRequest) GetBlockHeight() uint64 { return 0 } +func (x *SendRequest) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp + } + return nil +} + // swagger:model SendCallbackRequest type SendCallbackRequest struct { state protoimpl.MessageState @@ -443,7 +451,7 @@ var file_internal_callbacker_callbacker_api_callbacker_api_proto_rawDesc = []byt 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x22, 0xc4, 0x02, 0x0a, 0x0b, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, + 0x61, 0x6d, 0x70, 0x22, 0xfe, 0x02, 0x0a, 0x0b, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4a, 0x0a, 0x10, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x43, @@ -463,64 +471,68 @@ var file_internal_callbacker_callbacker_api_callbacker_api_proto_rawDesc = []byt 0x6b, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0xce, 0x02, 0x0a, 0x13, 0x53, - 0x65, 0x6e, 0x64, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x4c, 0x0a, 0x11, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x72, - 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, - 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x43, - 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x10, - 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x73, - 0x12, 0x12, 0x0a, 0x04, 0x74, 0x78, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x74, 0x78, 0x69, 0x64, 0x12, 0x2e, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x65, - 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x65, 0x72, 0x6b, 0x6c, 0x65, 0x5f, 0x70, - 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x65, 0x72, 0x6b, 0x6c, - 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x69, - 0x6e, 0x66, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x78, 0x74, 0x72, 0x61, - 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x65, 0x74, 0x69, 0x6e, - 0x67, 0x5f, 0x74, 0x78, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x6f, 0x6d, - 0x70, 0x65, 0x74, 0x69, 0x6e, 0x67, 0x54, 0x78, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x6c, 0x6f, - 0x63, 0x6b, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x6c, 0x6f, 0x63, - 0x6b, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, - 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x5a, 0x0a, 0x0f, 0x43, - 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x10, - 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, - 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, - 0x62, 0x61, 0x74, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x61, 0x6c, 0x6c, - 0x6f, 0x77, 0x42, 0x61, 0x74, 0x63, 0x68, 0x2a, 0x9d, 0x02, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, - 0x0a, 0x0a, 0x06, 0x51, 0x55, 0x45, 0x55, 0x45, 0x44, 0x10, 0x0a, 0x12, 0x0c, 0x0a, 0x08, 0x52, - 0x45, 0x43, 0x45, 0x49, 0x56, 0x45, 0x44, 0x10, 0x14, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x4f, - 0x52, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x18, 0x0a, 0x14, 0x41, 0x4e, 0x4e, 0x4f, 0x55, 0x4e, 0x43, - 0x45, 0x44, 0x5f, 0x54, 0x4f, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, 0x28, 0x12, - 0x18, 0x0a, 0x14, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x45, 0x44, 0x5f, 0x42, 0x59, 0x5f, - 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, 0x32, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x45, 0x4e, - 0x54, 0x5f, 0x54, 0x4f, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, 0x3c, 0x12, 0x17, - 0x0a, 0x13, 0x41, 0x43, 0x43, 0x45, 0x50, 0x54, 0x45, 0x44, 0x5f, 0x42, 0x59, 0x5f, 0x4e, 0x45, - 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, 0x46, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x45, 0x45, 0x4e, 0x5f, - 0x49, 0x4e, 0x5f, 0x4f, 0x52, 0x50, 0x48, 0x41, 0x4e, 0x5f, 0x4d, 0x45, 0x4d, 0x50, 0x4f, 0x4f, - 0x4c, 0x10, 0x50, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x45, 0x45, 0x4e, 0x5f, 0x4f, 0x4e, 0x5f, 0x4e, - 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, 0x5a, 0x12, 0x1a, 0x0a, 0x16, 0x44, 0x4f, 0x55, 0x42, - 0x4c, 0x45, 0x5f, 0x53, 0x50, 0x45, 0x4e, 0x44, 0x5f, 0x41, 0x54, 0x54, 0x45, 0x4d, 0x50, 0x54, - 0x45, 0x44, 0x10, 0x64, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x45, 0x4a, 0x45, 0x43, 0x54, 0x45, 0x44, - 0x10, 0x6e, 0x12, 0x18, 0x0a, 0x14, 0x4d, 0x49, 0x4e, 0x45, 0x44, 0x5f, 0x49, 0x4e, 0x5f, 0x53, - 0x54, 0x41, 0x4c, 0x45, 0x5f, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x10, 0x73, 0x12, 0x09, 0x0a, 0x05, - 0x4d, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x78, 0x32, 0xa2, 0x01, 0x0a, 0x0d, 0x43, 0x61, 0x6c, 0x6c, - 0x62, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x41, 0x50, 0x49, 0x12, 0x42, 0x0a, 0x06, 0x48, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1e, 0x2e, 0x63, 0x61, - 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x48, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4d, 0x0a, - 0x0c, 0x53, 0x65, 0x6e, 0x64, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x23, 0x2e, - 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, - 0x65, 0x6e, 0x64, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x42, 0x12, 0x5a, 0x10, - 0x2e, 0x3b, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x22, 0xce, 0x02, 0x0a, 0x13, 0x53, 0x65, 0x6e, 0x64, 0x43, 0x61, 0x6c, + 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4c, 0x0a, 0x11, + 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, + 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, + 0x6b, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x10, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, + 0x63, 0x6b, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x78, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x78, 0x69, 0x64, 0x12, 0x2e, + 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, + 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, + 0x0a, 0x0b, 0x6d, 0x65, 0x72, 0x6b, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x65, 0x72, 0x6b, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, + 0x1d, 0x0a, 0x0a, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x78, 0x74, 0x72, 0x61, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x23, + 0x0a, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x65, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x78, 0x73, 0x18, + 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x6f, 0x6d, 0x70, 0x65, 0x74, 0x69, 0x6e, 0x67, + 0x54, 0x78, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, 0x61, 0x73, + 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x61, + 0x73, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, 0x65, 0x69, 0x67, + 0x68, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x48, + 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x5a, 0x0a, 0x0f, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, + 0x6b, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, + 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, + 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x62, 0x61, 0x74, 0x63, 0x68, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x42, 0x61, 0x74, 0x63, + 0x68, 0x2a, 0x9d, 0x02, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, + 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x51, 0x55, 0x45, + 0x55, 0x45, 0x44, 0x10, 0x0a, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x45, 0x43, 0x45, 0x49, 0x56, 0x45, + 0x44, 0x10, 0x14, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x4f, 0x52, 0x45, 0x44, 0x10, 0x1e, 0x12, + 0x18, 0x0a, 0x14, 0x41, 0x4e, 0x4e, 0x4f, 0x55, 0x4e, 0x43, 0x45, 0x44, 0x5f, 0x54, 0x4f, 0x5f, + 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, 0x28, 0x12, 0x18, 0x0a, 0x14, 0x52, 0x45, 0x51, + 0x55, 0x45, 0x53, 0x54, 0x45, 0x44, 0x5f, 0x42, 0x59, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, + 0x4b, 0x10, 0x32, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x4f, 0x5f, 0x4e, + 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, 0x3c, 0x12, 0x17, 0x0a, 0x13, 0x41, 0x43, 0x43, 0x45, + 0x50, 0x54, 0x45, 0x44, 0x5f, 0x42, 0x59, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, + 0x46, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x45, 0x45, 0x4e, 0x5f, 0x49, 0x4e, 0x5f, 0x4f, 0x52, 0x50, + 0x48, 0x41, 0x4e, 0x5f, 0x4d, 0x45, 0x4d, 0x50, 0x4f, 0x4f, 0x4c, 0x10, 0x50, 0x12, 0x13, 0x0a, + 0x0f, 0x53, 0x45, 0x45, 0x4e, 0x5f, 0x4f, 0x4e, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, + 0x10, 0x5a, 0x12, 0x1a, 0x0a, 0x16, 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x5f, 0x53, 0x50, 0x45, + 0x4e, 0x44, 0x5f, 0x41, 0x54, 0x54, 0x45, 0x4d, 0x50, 0x54, 0x45, 0x44, 0x10, 0x64, 0x12, 0x0c, + 0x0a, 0x08, 0x52, 0x45, 0x4a, 0x45, 0x43, 0x54, 0x45, 0x44, 0x10, 0x6e, 0x12, 0x18, 0x0a, 0x14, + 0x4d, 0x49, 0x4e, 0x45, 0x44, 0x5f, 0x49, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x4c, 0x45, 0x5f, 0x42, + 0x4c, 0x4f, 0x43, 0x4b, 0x10, 0x73, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x49, 0x4e, 0x45, 0x44, 0x10, + 0x78, 0x32, 0xa2, 0x01, 0x0a, 0x0d, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x72, + 0x41, 0x50, 0x49, 0x12, 0x42, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x16, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1e, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, + 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0c, 0x53, 0x65, 0x6e, 0x64, 0x43, + 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x23, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, + 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x43, 0x61, 0x6c, + 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x42, 0x12, 0x5a, 0x10, 0x2e, 0x3b, 0x63, 0x61, 0x6c, 0x6c, + 0x62, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( @@ -550,17 +562,18 @@ var file_internal_callbacker_callbacker_api_callbacker_api_proto_depIdxs = []int 5, // 0: callbacker_api.HealthResponse.timestamp:type_name -> google.protobuf.Timestamp 4, // 1: callbacker_api.SendRequest.callback_routing:type_name -> callbacker_api.CallbackRouting 0, // 2: callbacker_api.SendRequest.status:type_name -> callbacker_api.Status - 4, // 3: callbacker_api.SendCallbackRequest.callback_routings:type_name -> callbacker_api.CallbackRouting - 0, // 4: callbacker_api.SendCallbackRequest.status:type_name -> callbacker_api.Status - 6, // 5: callbacker_api.CallbackerAPI.Health:input_type -> google.protobuf.Empty - 3, // 6: callbacker_api.CallbackerAPI.SendCallback:input_type -> callbacker_api.SendCallbackRequest - 1, // 7: callbacker_api.CallbackerAPI.Health:output_type -> callbacker_api.HealthResponse - 6, // 8: callbacker_api.CallbackerAPI.SendCallback:output_type -> google.protobuf.Empty - 7, // [7:9] is the sub-list for method output_type - 5, // [5:7] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 5, // 3: callbacker_api.SendRequest.timestamp:type_name -> google.protobuf.Timestamp + 4, // 4: callbacker_api.SendCallbackRequest.callback_routings:type_name -> callbacker_api.CallbackRouting + 0, // 5: callbacker_api.SendCallbackRequest.status:type_name -> callbacker_api.Status + 6, // 6: callbacker_api.CallbackerAPI.Health:input_type -> google.protobuf.Empty + 3, // 7: callbacker_api.CallbackerAPI.SendCallback:input_type -> callbacker_api.SendCallbackRequest + 1, // 8: callbacker_api.CallbackerAPI.Health:output_type -> callbacker_api.HealthResponse + 6, // 9: callbacker_api.CallbackerAPI.SendCallback:output_type -> google.protobuf.Empty + 8, // [8:10] is the sub-list for method output_type + 6, // [6:8] is the sub-list for method input_type + 6, // [6:6] is the sub-list for extension type_name + 6, // [6:6] is the sub-list for extension extendee + 0, // [0:6] is the sub-list for field type_name } func init() { file_internal_callbacker_callbacker_api_callbacker_api_proto_init() } diff --git a/internal/callbacker/callbacker_api/callbacker_api.proto b/internal/callbacker/callbacker_api/callbacker_api.proto index 2083adf64..802baeb6e 100644 --- a/internal/callbacker/callbacker_api/callbacker_api.proto +++ b/internal/callbacker/callbacker_api/callbacker_api.proto @@ -47,6 +47,7 @@ message SendRequest { repeated string competing_txs = 6; string block_hash = 7; uint64 block_height = 8; + google.protobuf.Timestamp timestamp = 9; } // swagger:model SendCallbackRequest diff --git a/internal/callbacker/callbacker_mocks.go b/internal/callbacker/callbacker_mocks.go index aea59c4bd..ba3cb2af3 100644 --- a/internal/callbacker/callbacker_mocks.go +++ b/internal/callbacker/callbacker_mocks.go @@ -1,11 +1,10 @@ package callbacker -// from callbacker.go -//go:generate moq -pkg mocks -out ./mocks/callbacker_mock.go ./ SenderI +//go:generate moq -pkg mocks -out ./mocks/sender_mock.go ./ SenderI -//go:generate moq -pkg mocks -out ./mocks/callbacker_api_client_mock.go ./callbacker_api/ CallbackerAPIClient +//go:generate moq -pkg mocks -out ./mocks/send_manager_mock.go ./ SendManagerI -//go:generate moq -pkg mocks -out ./mocks/store_mock.go ./store/ CallbackerStore +//go:generate moq -pkg mocks -out ./mocks/callbacker_api_client_mock.go ./callbacker_api/ CallbackerAPIClient //go:generate moq -pkg mocks -out ./mocks/processor_store_mock.go ./store/ ProcessorStore diff --git a/internal/callbacker/dispatcher.go b/internal/callbacker/dispatcher.go index 51dd82bea..00d118fce 100644 --- a/internal/callbacker/dispatcher.go +++ b/internal/callbacker/dispatcher.go @@ -15,52 +15,34 @@ Graceful Shutdown: on service termination, the CallbackDispatcher ensures all ac */ import ( - "log/slog" "sync" - "time" - - "github.com/bitcoin-sv/arc/internal/callbacker/store" ) type CallbackDispatcher struct { sender SenderI - store store.CallbackerStore - logger *slog.Logger - managers map[string]*SendManager - managersMu sync.Mutex + runNewManager func(url string) SendManagerI - sendConfig *SendConfig + managers map[string]SendManagerI + managersMu sync.Mutex } type CallbackEntry struct { - Token string - Data *Callback - postponedUntil *time.Time - AllowBatch bool -} - -type SendConfig struct { - Delay time.Duration - PauseAfterSingleModeSuccessfulSend time.Duration - DelayDuration time.Duration - BatchSendInterval time.Duration - Expiration time.Duration + Token string + Data *Callback + AllowBatch bool } -func NewCallbackDispatcher(callbacker SenderI, cStore store.CallbackerStore, logger *slog.Logger, - sendingConfig *SendConfig) *CallbackDispatcher { +func NewCallbackDispatcher(callbacker SenderI, + runNewManager func(url string) SendManagerI) *CallbackDispatcher { return &CallbackDispatcher{ - sender: callbacker, - store: cStore, - logger: logger.With(slog.String("module", "dispatcher")), - - sendConfig: sendingConfig, - managers: make(map[string]*SendManager), + sender: callbacker, + runNewManager: runNewManager, + managers: make(map[string]SendManagerI), } } -func (d *CallbackDispatcher) GetLenMangers() int { +func (d *CallbackDispatcher) GetLenManagers() int { return len(d.managers) } @@ -73,15 +55,15 @@ func (d *CallbackDispatcher) GracefulStop() { } } -func (d *CallbackDispatcher) Dispatch(url string, dto *CallbackEntry, allowBatch bool) { +func (d *CallbackDispatcher) Dispatch(url string, dto *CallbackEntry) { d.managersMu.Lock() manager, ok := d.managers[url] if !ok { - manager = RunNewSendManager(url, d.sender, d.store, d.logger, d.sendConfig) + manager = d.runNewManager(url) d.managers[url] = manager } - d.managersMu.Unlock() + manager.Enqueue(*dto) - manager.Add(dto, allowBatch) + d.managersMu.Unlock() } diff --git a/internal/callbacker/dispatcher_test.go b/internal/callbacker/dispatcher_test.go index 4bc59b1e3..2dee728d5 100644 --- a/internal/callbacker/dispatcher_test.go +++ b/internal/callbacker/dispatcher_test.go @@ -1,10 +1,7 @@ package callbacker_test import ( - "context" "fmt" - "log/slog" - "sync" "testing" "time" @@ -12,7 +9,6 @@ import ( "github.com/bitcoin-sv/arc/internal/callbacker" "github.com/bitcoin-sv/arc/internal/callbacker/mocks" - "github.com/bitcoin-sv/arc/internal/callbacker/store" ) func TestCallbackDispatcher(t *testing.T) { @@ -21,21 +17,13 @@ func TestCallbackDispatcher(t *testing.T) { sendInterval time.Duration numOfReceivers int numOfSendPerReceiver int - stopDispatcher bool }{ { name: "send", - sendInterval: 0, + sendInterval: 5, numOfReceivers: 20, numOfSendPerReceiver: 1000, }, - { - name: "process callbacks on stopping", - sendInterval: 5 * time.Millisecond, // set interval to give time to call stop function - numOfReceivers: 100, - numOfSendPerReceiver: 200, - stopDispatcher: true, - }, } for _, tc := range tcs { @@ -45,23 +33,12 @@ func TestCallbackDispatcher(t *testing.T) { SendFunc: func(_, _ string, _ *callbacker.Callback) (bool, bool) { return true, false }, } - var savedCallbacks []*store.CallbackData - sMq := &mocks.CallbackerStoreMock{ - SetManyFunc: func(_ context.Context, data []*store.CallbackData) error { - savedCallbacks = append(savedCallbacks, data...) - return nil - }, - SetFunc: func(_ context.Context, _ *store.CallbackData) error { - return nil - }, - } - - sendingConfig := callbacker.SendConfig{ - PauseAfterSingleModeSuccessfulSend: tc.sendInterval, - Expiration: 24 * time.Hour, - } - - sut := callbacker.NewCallbackDispatcher(cMq, sMq, slog.Default(), &sendingConfig) + sut := callbacker.NewCallbackDispatcher(cMq, func(_ string) callbacker.SendManagerI { + return &mocks.SendManagerIMock{ + EnqueueFunc: func(_ callbacker.CallbackEntry) {}, + GracefulStopFunc: func() {}, + } + }) var receivers []string for i := range tc.numOfReceivers { @@ -69,35 +46,15 @@ func TestCallbackDispatcher(t *testing.T) { } // when - // send callbacks to receiver - wg := &sync.WaitGroup{} - wg.Add(tc.numOfSendPerReceiver) - for range tc.numOfSendPerReceiver { - go func() { - for _, url := range receivers { - sut.Dispatch(url, &callbacker.CallbackEntry{Token: "", Data: &callbacker.Callback{}}, false) - } - wg.Done() - }() - } - wg.Wait() - if tc.stopDispatcher { - sut.GracefulStop() - } else { - // give a chance to process - time.Sleep(100 * time.Millisecond) + for _, receiver := range receivers { + sut.Dispatch(receiver, &callbacker.CallbackEntry{Token: "", Data: &callbacker.Callback{}}) } + // send callbacks to receiver + sut.GracefulStop() // then - require.Equal(t, tc.numOfReceivers, sut.GetLenMangers()) - if tc.stopDispatcher { - require.NotEmpty(t, savedCallbacks) - require.Equal(t, tc.numOfReceivers*tc.numOfSendPerReceiver, len(cMq.SendCalls())+len(savedCallbacks)) - } else { - require.Empty(t, savedCallbacks) - require.Equal(t, tc.numOfReceivers*tc.numOfSendPerReceiver, len(cMq.SendCalls())) - } + require.Equal(t, tc.numOfReceivers, sut.GetLenManagers()) }) } } diff --git a/internal/callbacker/mocks/dipatcher_mock.go b/internal/callbacker/mocks/dipatcher_mock.go index e8ac24d92..f977c3075 100644 --- a/internal/callbacker/mocks/dipatcher_mock.go +++ b/internal/callbacker/mocks/dipatcher_mock.go @@ -18,7 +18,7 @@ var _ callbacker.Dispatcher = &DispatcherMock{} // // // make and configure a mocked callbacker.Dispatcher // mockedDispatcher := &DispatcherMock{ -// DispatchFunc: func(url string, dto *callbacker.CallbackEntry, allowBatch bool) { +// DispatchFunc: func(url string, dto *callbacker.CallbackEntry) { // panic("mock out the Dispatch method") // }, // } @@ -29,7 +29,7 @@ var _ callbacker.Dispatcher = &DispatcherMock{} // } type DispatcherMock struct { // DispatchFunc mocks the Dispatch method. - DispatchFunc func(url string, dto *callbacker.CallbackEntry, allowBatch bool) + DispatchFunc func(url string, dto *callbacker.CallbackEntry) // calls tracks calls to the methods. calls struct { @@ -39,31 +39,27 @@ type DispatcherMock struct { URL string // Dto is the dto argument value. Dto *callbacker.CallbackEntry - // AllowBatch is the allowBatch argument value. - AllowBatch bool } } lockDispatch sync.RWMutex } // Dispatch calls DispatchFunc. -func (mock *DispatcherMock) Dispatch(url string, dto *callbacker.CallbackEntry, allowBatch bool) { +func (mock *DispatcherMock) Dispatch(url string, dto *callbacker.CallbackEntry) { if mock.DispatchFunc == nil { panic("DispatcherMock.DispatchFunc: method is nil but Dispatcher.Dispatch was just called") } callInfo := struct { - URL string - Dto *callbacker.CallbackEntry - AllowBatch bool + URL string + Dto *callbacker.CallbackEntry }{ - URL: url, - Dto: dto, - AllowBatch: allowBatch, + URL: url, + Dto: dto, } mock.lockDispatch.Lock() mock.calls.Dispatch = append(mock.calls.Dispatch, callInfo) mock.lockDispatch.Unlock() - mock.DispatchFunc(url, dto, allowBatch) + mock.DispatchFunc(url, dto) } // DispatchCalls gets all the calls that were made to Dispatch. @@ -71,14 +67,12 @@ func (mock *DispatcherMock) Dispatch(url string, dto *callbacker.CallbackEntry, // // len(mockedDispatcher.DispatchCalls()) func (mock *DispatcherMock) DispatchCalls() []struct { - URL string - Dto *callbacker.CallbackEntry - AllowBatch bool + URL string + Dto *callbacker.CallbackEntry } { var calls []struct { - URL string - Dto *callbacker.CallbackEntry - AllowBatch bool + URL string + Dto *callbacker.CallbackEntry } mock.lockDispatch.RLock() calls = mock.calls.Dispatch diff --git a/internal/callbacker/mocks/processor_store_mock.go b/internal/callbacker/mocks/processor_store_mock.go index 96d5e4cd7..066632ed5 100644 --- a/internal/callbacker/mocks/processor_store_mock.go +++ b/internal/callbacker/mocks/processor_store_mock.go @@ -7,6 +7,7 @@ import ( "context" "github.com/bitcoin-sv/arc/internal/callbacker/store" "sync" + "time" ) // Ensure, that ProcessorStoreMock does implement store.ProcessorStore. @@ -19,12 +20,21 @@ var _ store.ProcessorStore = &ProcessorStoreMock{} // // // make and configure a mocked store.ProcessorStore // mockedProcessorStore := &ProcessorStoreMock{ +// DeleteOlderThanFunc: func(ctx context.Context, t time.Time) error { +// panic("mock out the DeleteOlderThan method") +// }, // DeleteURLMappingFunc: func(ctx context.Context, instance string) error { // panic("mock out the DeleteURLMapping method") // }, +// GetAndDeleteFunc: func(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) { +// panic("mock out the GetAndDelete method") +// }, // GetURLMappingsFunc: func(ctx context.Context) (map[string]string, error) { // panic("mock out the GetURLMappings method") // }, +// GetUnmappedURLFunc: func(ctx context.Context) (string, error) { +// panic("mock out the GetUnmappedURL method") +// }, // SetURLMappingFunc: func(ctx context.Context, m store.URLMapping) error { // panic("mock out the SetURLMapping method") // }, @@ -35,17 +45,33 @@ var _ store.ProcessorStore = &ProcessorStoreMock{} // // } type ProcessorStoreMock struct { + // DeleteOlderThanFunc mocks the DeleteOlderThan method. + DeleteOlderThanFunc func(ctx context.Context, t time.Time) error + // DeleteURLMappingFunc mocks the DeleteURLMapping method. DeleteURLMappingFunc func(ctx context.Context, instance string) error + // GetAndDeleteFunc mocks the GetAndDelete method. + GetAndDeleteFunc func(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) + // GetURLMappingsFunc mocks the GetURLMappings method. GetURLMappingsFunc func(ctx context.Context) (map[string]string, error) + // GetUnmappedURLFunc mocks the GetUnmappedURL method. + GetUnmappedURLFunc func(ctx context.Context) (string, error) + // SetURLMappingFunc mocks the SetURLMapping method. SetURLMappingFunc func(ctx context.Context, m store.URLMapping) error // calls tracks calls to the methods. calls struct { + // DeleteOlderThan holds details about calls to the DeleteOlderThan method. + DeleteOlderThan []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // T is the t argument value. + T time.Time + } // DeleteURLMapping holds details about calls to the DeleteURLMapping method. DeleteURLMapping []struct { // Ctx is the ctx argument value. @@ -53,11 +79,25 @@ type ProcessorStoreMock struct { // Instance is the instance argument value. Instance string } + // GetAndDelete holds details about calls to the GetAndDelete method. + GetAndDelete []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // URL is the url argument value. + URL string + // Limit is the limit argument value. + Limit int + } // GetURLMappings holds details about calls to the GetURLMappings method. GetURLMappings []struct { // Ctx is the ctx argument value. Ctx context.Context } + // GetUnmappedURL holds details about calls to the GetUnmappedURL method. + GetUnmappedURL []struct { + // Ctx is the ctx argument value. + Ctx context.Context + } // SetURLMapping holds details about calls to the SetURLMapping method. SetURLMapping []struct { // Ctx is the ctx argument value. @@ -66,11 +106,50 @@ type ProcessorStoreMock struct { M store.URLMapping } } + lockDeleteOlderThan sync.RWMutex lockDeleteURLMapping sync.RWMutex + lockGetAndDelete sync.RWMutex lockGetURLMappings sync.RWMutex + lockGetUnmappedURL sync.RWMutex lockSetURLMapping sync.RWMutex } +// DeleteOlderThan calls DeleteOlderThanFunc. +func (mock *ProcessorStoreMock) DeleteOlderThan(ctx context.Context, t time.Time) error { + if mock.DeleteOlderThanFunc == nil { + panic("ProcessorStoreMock.DeleteOlderThanFunc: method is nil but ProcessorStore.DeleteOlderThan was just called") + } + callInfo := struct { + Ctx context.Context + T time.Time + }{ + Ctx: ctx, + T: t, + } + mock.lockDeleteOlderThan.Lock() + mock.calls.DeleteOlderThan = append(mock.calls.DeleteOlderThan, callInfo) + mock.lockDeleteOlderThan.Unlock() + return mock.DeleteOlderThanFunc(ctx, t) +} + +// DeleteOlderThanCalls gets all the calls that were made to DeleteOlderThan. +// Check the length with: +// +// len(mockedProcessorStore.DeleteOlderThanCalls()) +func (mock *ProcessorStoreMock) DeleteOlderThanCalls() []struct { + Ctx context.Context + T time.Time +} { + var calls []struct { + Ctx context.Context + T time.Time + } + mock.lockDeleteOlderThan.RLock() + calls = mock.calls.DeleteOlderThan + mock.lockDeleteOlderThan.RUnlock() + return calls +} + // DeleteURLMapping calls DeleteURLMappingFunc. func (mock *ProcessorStoreMock) DeleteURLMapping(ctx context.Context, instance string) error { if mock.DeleteURLMappingFunc == nil { @@ -107,6 +186,46 @@ func (mock *ProcessorStoreMock) DeleteURLMappingCalls() []struct { return calls } +// GetAndDelete calls GetAndDeleteFunc. +func (mock *ProcessorStoreMock) GetAndDelete(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) { + if mock.GetAndDeleteFunc == nil { + panic("ProcessorStoreMock.GetAndDeleteFunc: method is nil but ProcessorStore.GetAndDelete was just called") + } + callInfo := struct { + Ctx context.Context + URL string + Limit int + }{ + Ctx: ctx, + URL: url, + Limit: limit, + } + mock.lockGetAndDelete.Lock() + mock.calls.GetAndDelete = append(mock.calls.GetAndDelete, callInfo) + mock.lockGetAndDelete.Unlock() + return mock.GetAndDeleteFunc(ctx, url, limit) +} + +// GetAndDeleteCalls gets all the calls that were made to GetAndDelete. +// Check the length with: +// +// len(mockedProcessorStore.GetAndDeleteCalls()) +func (mock *ProcessorStoreMock) GetAndDeleteCalls() []struct { + Ctx context.Context + URL string + Limit int +} { + var calls []struct { + Ctx context.Context + URL string + Limit int + } + mock.lockGetAndDelete.RLock() + calls = mock.calls.GetAndDelete + mock.lockGetAndDelete.RUnlock() + return calls +} + // GetURLMappings calls GetURLMappingsFunc. func (mock *ProcessorStoreMock) GetURLMappings(ctx context.Context) (map[string]string, error) { if mock.GetURLMappingsFunc == nil { @@ -139,6 +258,38 @@ func (mock *ProcessorStoreMock) GetURLMappingsCalls() []struct { return calls } +// GetUnmappedURL calls GetUnmappedURLFunc. +func (mock *ProcessorStoreMock) GetUnmappedURL(ctx context.Context) (string, error) { + if mock.GetUnmappedURLFunc == nil { + panic("ProcessorStoreMock.GetUnmappedURLFunc: method is nil but ProcessorStore.GetUnmappedURL was just called") + } + callInfo := struct { + Ctx context.Context + }{ + Ctx: ctx, + } + mock.lockGetUnmappedURL.Lock() + mock.calls.GetUnmappedURL = append(mock.calls.GetUnmappedURL, callInfo) + mock.lockGetUnmappedURL.Unlock() + return mock.GetUnmappedURLFunc(ctx) +} + +// GetUnmappedURLCalls gets all the calls that were made to GetUnmappedURL. +// Check the length with: +// +// len(mockedProcessorStore.GetUnmappedURLCalls()) +func (mock *ProcessorStoreMock) GetUnmappedURLCalls() []struct { + Ctx context.Context +} { + var calls []struct { + Ctx context.Context + } + mock.lockGetUnmappedURL.RLock() + calls = mock.calls.GetUnmappedURL + mock.lockGetUnmappedURL.RUnlock() + return calls +} + // SetURLMapping calls SetURLMappingFunc. func (mock *ProcessorStoreMock) SetURLMapping(ctx context.Context, m store.URLMapping) error { if mock.SetURLMappingFunc == nil { diff --git a/internal/callbacker/mocks/send_manager_mock.go b/internal/callbacker/mocks/send_manager_mock.go new file mode 100644 index 000000000..f218090f6 --- /dev/null +++ b/internal/callbacker/mocks/send_manager_mock.go @@ -0,0 +1,149 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mocks + +import ( + "github.com/bitcoin-sv/arc/internal/callbacker" + "sync" +) + +// Ensure, that SendManagerIMock does implement callbacker.SendManagerI. +// If this is not the case, regenerate this file with moq. +var _ callbacker.SendManagerI = &SendManagerIMock{} + +// SendManagerIMock is a mock implementation of callbacker.SendManagerI. +// +// func TestSomethingThatUsesSendManagerI(t *testing.T) { +// +// // make and configure a mocked callbacker.SendManagerI +// mockedSendManagerI := &SendManagerIMock{ +// EnqueueFunc: func(entry callbacker.CallbackEntry) { +// panic("mock out the Enqueue method") +// }, +// GracefulStopFunc: func() { +// panic("mock out the GracefulStop method") +// }, +// StartFunc: func() { +// panic("mock out the Start method") +// }, +// } +// +// // use mockedSendManagerI in code that requires callbacker.SendManagerI +// // and then make assertions. +// +// } +type SendManagerIMock struct { + // EnqueueFunc mocks the Enqueue method. + EnqueueFunc func(entry callbacker.CallbackEntry) + + // GracefulStopFunc mocks the GracefulStop method. + GracefulStopFunc func() + + // StartFunc mocks the Start method. + StartFunc func() + + // calls tracks calls to the methods. + calls struct { + // Enqueue holds details about calls to the Enqueue method. + Enqueue []struct { + // Entry is the entry argument value. + Entry callbacker.CallbackEntry + } + // GracefulStop holds details about calls to the GracefulStop method. + GracefulStop []struct { + } + // Start holds details about calls to the Start method. + Start []struct { + } + } + lockEnqueue sync.RWMutex + lockGracefulStop sync.RWMutex + lockStart sync.RWMutex +} + +// Enqueue calls EnqueueFunc. +func (mock *SendManagerIMock) Enqueue(entry callbacker.CallbackEntry) { + if mock.EnqueueFunc == nil { + panic("SendManagerIMock.EnqueueFunc: method is nil but SendManagerI.Enqueue was just called") + } + callInfo := struct { + Entry callbacker.CallbackEntry + }{ + Entry: entry, + } + mock.lockEnqueue.Lock() + mock.calls.Enqueue = append(mock.calls.Enqueue, callInfo) + mock.lockEnqueue.Unlock() + mock.EnqueueFunc(entry) +} + +// EnqueueCalls gets all the calls that were made to Enqueue. +// Check the length with: +// +// len(mockedSendManagerI.EnqueueCalls()) +func (mock *SendManagerIMock) EnqueueCalls() []struct { + Entry callbacker.CallbackEntry +} { + var calls []struct { + Entry callbacker.CallbackEntry + } + mock.lockEnqueue.RLock() + calls = mock.calls.Enqueue + mock.lockEnqueue.RUnlock() + return calls +} + +// GracefulStop calls GracefulStopFunc. +func (mock *SendManagerIMock) GracefulStop() { + if mock.GracefulStopFunc == nil { + panic("SendManagerIMock.GracefulStopFunc: method is nil but SendManagerI.GracefulStop was just called") + } + callInfo := struct { + }{} + mock.lockGracefulStop.Lock() + mock.calls.GracefulStop = append(mock.calls.GracefulStop, callInfo) + mock.lockGracefulStop.Unlock() + mock.GracefulStopFunc() +} + +// GracefulStopCalls gets all the calls that were made to GracefulStop. +// Check the length with: +// +// len(mockedSendManagerI.GracefulStopCalls()) +func (mock *SendManagerIMock) GracefulStopCalls() []struct { +} { + var calls []struct { + } + mock.lockGracefulStop.RLock() + calls = mock.calls.GracefulStop + mock.lockGracefulStop.RUnlock() + return calls +} + +// Start calls StartFunc. +func (mock *SendManagerIMock) Start() { + if mock.StartFunc == nil { + panic("SendManagerIMock.StartFunc: method is nil but SendManagerI.Start was just called") + } + callInfo := struct { + }{} + mock.lockStart.Lock() + mock.calls.Start = append(mock.calls.Start, callInfo) + mock.lockStart.Unlock() + mock.StartFunc() +} + +// StartCalls gets all the calls that were made to Start. +// Check the length with: +// +// len(mockedSendManagerI.StartCalls()) +func (mock *SendManagerIMock) StartCalls() []struct { +} { + var calls []struct { + } + mock.lockStart.RLock() + calls = mock.calls.Start + mock.lockStart.RUnlock() + return calls +} diff --git a/internal/callbacker/mocks/callbacker_mock.go b/internal/callbacker/mocks/sender_mock.go similarity index 100% rename from internal/callbacker/mocks/callbacker_mock.go rename to internal/callbacker/mocks/sender_mock.go diff --git a/internal/callbacker/mocks/store_mock.go b/internal/callbacker/mocks/store_mock.go deleted file mode 100644 index 537bbd561..000000000 --- a/internal/callbacker/mocks/store_mock.go +++ /dev/null @@ -1,326 +0,0 @@ -// Code generated by moq; DO NOT EDIT. -// github.com/matryer/moq - -package mocks - -import ( - "context" - "github.com/bitcoin-sv/arc/internal/callbacker/store" - "sync" - "time" -) - -// Ensure, that CallbackerStoreMock does implement store.CallbackerStore. -// If this is not the case, regenerate this file with moq. -var _ store.CallbackerStore = &CallbackerStoreMock{} - -// CallbackerStoreMock is a mock implementation of store.CallbackerStore. -// -// func TestSomethingThatUsesCallbackerStore(t *testing.T) { -// -// // make and configure a mocked store.CallbackerStore -// mockedCallbackerStore := &CallbackerStoreMock{ -// CloseFunc: func() error { -// panic("mock out the Close method") -// }, -// DeleteFailedOlderThanFunc: func(ctx context.Context, t time.Time) error { -// panic("mock out the DeleteFailedOlderThan method") -// }, -// PopFailedManyFunc: func(ctx context.Context, t time.Time, limit int) ([]*store.CallbackData, error) { -// panic("mock out the PopFailedMany method") -// }, -// PopManyFunc: func(ctx context.Context, limit int) ([]*store.CallbackData, error) { -// panic("mock out the PopMany method") -// }, -// SetFunc: func(ctx context.Context, dto *store.CallbackData) error { -// panic("mock out the Set method") -// }, -// SetManyFunc: func(ctx context.Context, data []*store.CallbackData) error { -// panic("mock out the SetMany method") -// }, -// } -// -// // use mockedCallbackerStore in code that requires store.CallbackerStore -// // and then make assertions. -// -// } -type CallbackerStoreMock struct { - // CloseFunc mocks the Close method. - CloseFunc func() error - - // DeleteFailedOlderThanFunc mocks the DeleteFailedOlderThan method. - DeleteFailedOlderThanFunc func(ctx context.Context, t time.Time) error - - // PopFailedManyFunc mocks the PopFailedMany method. - PopFailedManyFunc func(ctx context.Context, t time.Time, limit int) ([]*store.CallbackData, error) - - // PopManyFunc mocks the PopMany method. - PopManyFunc func(ctx context.Context, limit int) ([]*store.CallbackData, error) - - // SetFunc mocks the Set method. - SetFunc func(ctx context.Context, dto *store.CallbackData) error - - // SetManyFunc mocks the SetMany method. - SetManyFunc func(ctx context.Context, data []*store.CallbackData) error - - // calls tracks calls to the methods. - calls struct { - // Close holds details about calls to the Close method. - Close []struct { - } - // DeleteFailedOlderThan holds details about calls to the DeleteFailedOlderThan method. - DeleteFailedOlderThan []struct { - // Ctx is the ctx argument value. - Ctx context.Context - // T is the t argument value. - T time.Time - } - // PopFailedMany holds details about calls to the PopFailedMany method. - PopFailedMany []struct { - // Ctx is the ctx argument value. - Ctx context.Context - // T is the t argument value. - T time.Time - // Limit is the limit argument value. - Limit int - } - // PopMany holds details about calls to the PopMany method. - PopMany []struct { - // Ctx is the ctx argument value. - Ctx context.Context - // Limit is the limit argument value. - Limit int - } - // Set holds details about calls to the Set method. - Set []struct { - // Ctx is the ctx argument value. - Ctx context.Context - // Dto is the dto argument value. - Dto *store.CallbackData - } - // SetMany holds details about calls to the SetMany method. - SetMany []struct { - // Ctx is the ctx argument value. - Ctx context.Context - // Data is the data argument value. - Data []*store.CallbackData - } - } - lockClose sync.RWMutex - lockDeleteFailedOlderThan sync.RWMutex - lockPopFailedMany sync.RWMutex - lockPopMany sync.RWMutex - lockSet sync.RWMutex - lockSetMany sync.RWMutex -} - -// Close calls CloseFunc. -func (mock *CallbackerStoreMock) Close() error { - if mock.CloseFunc == nil { - panic("CallbackerStoreMock.CloseFunc: method is nil but CallbackerStore.Close was just called") - } - callInfo := struct { - }{} - mock.lockClose.Lock() - mock.calls.Close = append(mock.calls.Close, callInfo) - mock.lockClose.Unlock() - return mock.CloseFunc() -} - -// CloseCalls gets all the calls that were made to Close. -// Check the length with: -// -// len(mockedCallbackerStore.CloseCalls()) -func (mock *CallbackerStoreMock) CloseCalls() []struct { -} { - var calls []struct { - } - mock.lockClose.RLock() - calls = mock.calls.Close - mock.lockClose.RUnlock() - return calls -} - -// DeleteFailedOlderThan calls DeleteFailedOlderThanFunc. -func (mock *CallbackerStoreMock) DeleteFailedOlderThan(ctx context.Context, t time.Time) error { - if mock.DeleteFailedOlderThanFunc == nil { - panic("CallbackerStoreMock.DeleteFailedOlderThanFunc: method is nil but CallbackerStore.DeleteFailedOlderThan was just called") - } - callInfo := struct { - Ctx context.Context - T time.Time - }{ - Ctx: ctx, - T: t, - } - mock.lockDeleteFailedOlderThan.Lock() - mock.calls.DeleteFailedOlderThan = append(mock.calls.DeleteFailedOlderThan, callInfo) - mock.lockDeleteFailedOlderThan.Unlock() - return mock.DeleteFailedOlderThanFunc(ctx, t) -} - -// DeleteFailedOlderThanCalls gets all the calls that were made to DeleteFailedOlderThan. -// Check the length with: -// -// len(mockedCallbackerStore.DeleteFailedOlderThanCalls()) -func (mock *CallbackerStoreMock) DeleteFailedOlderThanCalls() []struct { - Ctx context.Context - T time.Time -} { - var calls []struct { - Ctx context.Context - T time.Time - } - mock.lockDeleteFailedOlderThan.RLock() - calls = mock.calls.DeleteFailedOlderThan - mock.lockDeleteFailedOlderThan.RUnlock() - return calls -} - -// PopFailedMany calls PopFailedManyFunc. -func (mock *CallbackerStoreMock) PopFailedMany(ctx context.Context, t time.Time, limit int) ([]*store.CallbackData, error) { - if mock.PopFailedManyFunc == nil { - panic("CallbackerStoreMock.PopFailedManyFunc: method is nil but CallbackerStore.PopFailedMany was just called") - } - callInfo := struct { - Ctx context.Context - T time.Time - Limit int - }{ - Ctx: ctx, - T: t, - Limit: limit, - } - mock.lockPopFailedMany.Lock() - mock.calls.PopFailedMany = append(mock.calls.PopFailedMany, callInfo) - mock.lockPopFailedMany.Unlock() - return mock.PopFailedManyFunc(ctx, t, limit) -} - -// PopFailedManyCalls gets all the calls that were made to PopFailedMany. -// Check the length with: -// -// len(mockedCallbackerStore.PopFailedManyCalls()) -func (mock *CallbackerStoreMock) PopFailedManyCalls() []struct { - Ctx context.Context - T time.Time - Limit int -} { - var calls []struct { - Ctx context.Context - T time.Time - Limit int - } - mock.lockPopFailedMany.RLock() - calls = mock.calls.PopFailedMany - mock.lockPopFailedMany.RUnlock() - return calls -} - -// PopMany calls PopManyFunc. -func (mock *CallbackerStoreMock) PopMany(ctx context.Context, limit int) ([]*store.CallbackData, error) { - if mock.PopManyFunc == nil { - panic("CallbackerStoreMock.PopManyFunc: method is nil but CallbackerStore.PopMany was just called") - } - callInfo := struct { - Ctx context.Context - Limit int - }{ - Ctx: ctx, - Limit: limit, - } - mock.lockPopMany.Lock() - mock.calls.PopMany = append(mock.calls.PopMany, callInfo) - mock.lockPopMany.Unlock() - return mock.PopManyFunc(ctx, limit) -} - -// PopManyCalls gets all the calls that were made to PopMany. -// Check the length with: -// -// len(mockedCallbackerStore.PopManyCalls()) -func (mock *CallbackerStoreMock) PopManyCalls() []struct { - Ctx context.Context - Limit int -} { - var calls []struct { - Ctx context.Context - Limit int - } - mock.lockPopMany.RLock() - calls = mock.calls.PopMany - mock.lockPopMany.RUnlock() - return calls -} - -// Set calls SetFunc. -func (mock *CallbackerStoreMock) Set(ctx context.Context, dto *store.CallbackData) error { - if mock.SetFunc == nil { - panic("CallbackerStoreMock.SetFunc: method is nil but CallbackerStore.Set was just called") - } - callInfo := struct { - Ctx context.Context - Dto *store.CallbackData - }{ - Ctx: ctx, - Dto: dto, - } - mock.lockSet.Lock() - mock.calls.Set = append(mock.calls.Set, callInfo) - mock.lockSet.Unlock() - return mock.SetFunc(ctx, dto) -} - -// SetCalls gets all the calls that were made to Set. -// Check the length with: -// -// len(mockedCallbackerStore.SetCalls()) -func (mock *CallbackerStoreMock) SetCalls() []struct { - Ctx context.Context - Dto *store.CallbackData -} { - var calls []struct { - Ctx context.Context - Dto *store.CallbackData - } - mock.lockSet.RLock() - calls = mock.calls.Set - mock.lockSet.RUnlock() - return calls -} - -// SetMany calls SetManyFunc. -func (mock *CallbackerStoreMock) SetMany(ctx context.Context, data []*store.CallbackData) error { - if mock.SetManyFunc == nil { - panic("CallbackerStoreMock.SetManyFunc: method is nil but CallbackerStore.SetMany was just called") - } - callInfo := struct { - Ctx context.Context - Data []*store.CallbackData - }{ - Ctx: ctx, - Data: data, - } - mock.lockSetMany.Lock() - mock.calls.SetMany = append(mock.calls.SetMany, callInfo) - mock.lockSetMany.Unlock() - return mock.SetManyFunc(ctx, data) -} - -// SetManyCalls gets all the calls that were made to SetMany. -// Check the length with: -// -// len(mockedCallbackerStore.SetManyCalls()) -func (mock *CallbackerStoreMock) SetManyCalls() []struct { - Ctx context.Context - Data []*store.CallbackData -} { - var calls []struct { - Ctx context.Context - Data []*store.CallbackData - } - mock.lockSetMany.RLock() - calls = mock.calls.SetMany - mock.lockSetMany.RUnlock() - return calls -} diff --git a/internal/callbacker/processor.go b/internal/callbacker/processor.go index 955e21d55..3cb263c3d 100644 --- a/internal/callbacker/processor.go +++ b/internal/callbacker/processor.go @@ -16,11 +16,12 @@ import ( ) type Dispatcher interface { - Dispatch(url string, dto *CallbackEntry, allowBatch bool) + Dispatch(url string, dto *CallbackEntry) } const ( - syncInterval = 5 * time.Second + syncInterval = 5 * time.Second + dispatchPersistedIntervalDefault = 20 * time.Second ) var ( @@ -31,29 +32,36 @@ var ( ) type Processor struct { - mqClient MessageQueueClient - dispatcher Dispatcher - store store.ProcessorStore - logger *slog.Logger - - hostName string - waitGroup *sync.WaitGroup - cancelAll context.CancelFunc - ctx context.Context + mqClient MessageQueueClient + dispatcher Dispatcher + store store.ProcessorStore + logger *slog.Logger + dispatchPersistedInterval time.Duration + hostName string + waitGroup *sync.WaitGroup + cancelAll context.CancelFunc + ctx context.Context mu sync.RWMutex urlMapping map[string]string } +func WithDispatchPersistedInterval(interval time.Duration) func(*Processor) { + return func(p *Processor) { + p.dispatchPersistedInterval = interval + } +} + func NewProcessor(dispatcher Dispatcher, processorStore store.ProcessorStore, mqClient MessageQueueClient, hostName string, logger *slog.Logger, opts ...func(*Processor)) (*Processor, error) { p := &Processor{ - hostName: hostName, - urlMapping: make(map[string]string), - dispatcher: dispatcher, - waitGroup: &sync.WaitGroup{}, - store: processorStore, - logger: logger, - mqClient: mqClient, + hostName: hostName, + urlMapping: make(map[string]string), + dispatcher: dispatcher, + waitGroup: &sync.WaitGroup{}, + store: processorStore, + logger: logger, + mqClient: mqClient, + dispatchPersistedInterval: dispatchPersistedIntervalDefault, } for _, opt := range opts { opt(p) @@ -72,7 +80,7 @@ func (p *Processor) handleCallbackMessage(msg jetstream.Msg) error { request := &callbacker_api.SendRequest{} err := proto.Unmarshal(msg.Data(), request) if err != nil { - nakErr := msg.Nak() + nakErr := msg.Ack() // Ack instead of nak. The same message will always fail to be unmarshalled if nakErr != nil { return errors.Join(errors.Join(ErrUnmarshal, err), errors.Join(ErrNakMessage, nakErr)) } @@ -119,7 +127,7 @@ func (p *Processor) handleCallbackMessage(msg jetstream.Msg) error { p.logger.Debug("dispatching callback", "instance", p.hostName, "url", request.CallbackRouting.Url) dto := sendRequestToDto(request) if request.CallbackRouting.Url != "" { - p.dispatcher.Dispatch(request.CallbackRouting.Url, &CallbackEntry{Token: request.CallbackRouting.Token, Data: dto}, request.CallbackRouting.AllowBatch) + p.dispatcher.Dispatch(request.CallbackRouting.Url, &CallbackEntry{Token: request.CallbackRouting.Token, Data: dto, AllowBatch: request.CallbackRouting.AllowBatch}) } } else { p.logger.Debug("not dispatching callback", "instance", p.hostName, "url", request.CallbackRouting.Url) @@ -144,6 +152,101 @@ func (p *Processor) Start() error { return nil } +func (p *Processor) StartCallbackStoreCleanup(interval, olderThanDuration time.Duration) { + ctx := context.Background() + ticker := time.NewTicker(interval) + + p.waitGroup.Add(1) + go func() { + for { + select { + case <-ticker.C: + n := time.Now() + midnight := time.Date(n.Year(), n.Month(), n.Day(), 0, 0, 0, 0, time.UTC) + olderThan := midnight.Add(-1 * olderThanDuration) + + err := p.store.DeleteOlderThan(ctx, olderThan) + if err != nil { + p.logger.Error("Failed to delete old callbacks in delay", slog.String("err", err.Error())) + } + + case <-p.ctx.Done(): + p.waitGroup.Done() + return + } + } + }() +} + +// DispatchPersistedCallbacks loads and dispatches persisted callbacks with unmapped URLs in intervals +func (p *Processor) DispatchPersistedCallbacks() { + const batchSize = 100 + ctx := context.Background() + + ticker := time.NewTicker(p.dispatchPersistedInterval) + + p.waitGroup.Add(1) + go func() { + defer func() { + ticker.Stop() + p.waitGroup.Done() + }() + + for { + select { + case <-p.ctx.Done(): + return + case <-ticker.C: + url, err := p.store.GetUnmappedURL(ctx) + if err != nil { + p.logger.Error("Failed to fetch unmapped url", slog.String("err", err.Error())) + continue + } + + if url == "" { + continue + } + + err = p.store.SetURLMapping(ctx, store.URLMapping{ + URL: url, + Instance: p.hostName, + }) + + if err != nil { + if errors.Is(err, store.ErrURLMappingDuplicateKey) { + p.logger.Debug("URL already mapped", slog.String("err", err.Error())) + continue + } + + p.logger.Error("Failed to set URL mapping", slog.String("err", err.Error())) + continue + } + + callbacks, err := p.store.GetAndDelete(ctx, url, batchSize) + if err != nil { + p.logger.Error("Failed to load callbacks", slog.String("err", err.Error())) + continue + } + + if len(callbacks) == 0 { + continue + } + p.logger.Info("Dispatching callbacks with unmapped URL", slog.String("url", url), slog.Int("callbacks", len(callbacks))) + + for _, c := range callbacks { + callbackEntry := &CallbackEntry{ + Token: c.Token, + Data: toCallback(c), + AllowBatch: c.AllowBatch, + } + + p.dispatcher.Dispatch(c.URL, callbackEntry) + } + } + } + }() +} + func (p *Processor) startSyncURLMapping() { p.waitGroup.Add(1) go func() { @@ -173,6 +276,23 @@ func (p *Processor) startSyncURLMapping() { }() } +func toCallback(dto *store.CallbackData) *Callback { + d := &Callback{ + Timestamp: dto.Timestamp, + + CompetingTxs: dto.CompetingTxs, + TxID: dto.TxID, + TxStatus: dto.TxStatus, + ExtraInfo: dto.ExtraInfo, + MerklePath: dto.MerklePath, + + BlockHash: dto.BlockHash, + BlockHeight: dto.BlockHeight, + } + + return d +} + func (p *Processor) GracefulStop() { err := p.store.DeleteURLMapping(p.ctx, p.hostName) if err != nil { diff --git a/internal/callbacker/processor_test.go b/internal/callbacker/processor_test.go index a18c6681c..98a46c171 100644 --- a/internal/callbacker/processor_test.go +++ b/internal/callbacker/processor_test.go @@ -164,7 +164,7 @@ func TestProcessorStart(t *testing.T) { for _, tc := range tt { t.Run(tc.name, func(t *testing.T) { dispatcher := &mocks.DispatcherMock{ - DispatchFunc: func(_ string, _ *callbacker.CallbackEntry, _ bool) {}, + DispatchFunc: func(_ string, _ *callbacker.CallbackEntry) {}, } processorStore := &mocks.ProcessorStoreMock{ SetURLMappingFunc: func(_ context.Context, _ store.URLMapping) error { return tc.setURLMappingErr }, @@ -223,3 +223,123 @@ func TestProcessorStart(t *testing.T) { }) } } + +func TestDispatchPersistedCallbacks(t *testing.T) { + tt := []struct { + name string + storedCallbacks []*store.CallbackData + getAndDeleteErr error + setURLMappingErr error + + expectedDispatch int + }{ + { + name: "success - no stored callbacks", + storedCallbacks: []*store.CallbackData{}, + + expectedDispatch: 0, + }, + { + name: "success - 1 stored callback", + storedCallbacks: []*store.CallbackData{{ + URL: "https://test.com", + Token: "1234", + }}, + + expectedDispatch: 1, + }, + { + name: "URL already mapped", + storedCallbacks: []*store.CallbackData{{ + URL: "https://test.com", + Token: "1234", + }}, + setURLMappingErr: store.ErrURLMappingDuplicateKey, + + expectedDispatch: 0, + }, + { + name: "error deleting failed older than", + getAndDeleteErr: errors.New("some error"), + + expectedDispatch: 0, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + logger := slog.Default() + dispatcher := &mocks.DispatcherMock{DispatchFunc: func(_ string, _ *callbacker.CallbackEntry) {}} + + mqClient := &mocks.MessageQueueClientMock{ + SubscribeMsgFunc: func(_ string, _ func(_ jetstream.Msg) error) error { + return nil + }, + ShutdownFunc: func() {}, + } + processorStore := &mocks.ProcessorStoreMock{ + SetURLMappingFunc: func(_ context.Context, _ store.URLMapping) error { return tc.setURLMappingErr }, + DeleteURLMappingFunc: func(_ context.Context, _ string) error { return nil }, + GetURLMappingsFunc: func(_ context.Context) (map[string]string, error) { return nil, nil }, + GetUnmappedURLFunc: func(_ context.Context) (string, error) { return "https://abcdefg.com", nil }, + GetAndDeleteFunc: func(_ context.Context, _ string, _ int) ([]*store.CallbackData, error) { + return tc.storedCallbacks, tc.getAndDeleteErr + }, + } + processor, err := callbacker.NewProcessor(dispatcher, processorStore, mqClient, "host1", logger, callbacker.WithDispatchPersistedInterval(20*time.Millisecond)) + require.NoError(t, err) + + defer processor.GracefulStop() + + processor.DispatchPersistedCallbacks() + time.Sleep(30 * time.Millisecond) + + require.Equal(t, tc.expectedDispatch, len(dispatcher.DispatchCalls())) + }) + } +} + +func TestStartCallbackStoreCleanup(t *testing.T) { + tt := []struct { + name string + deleteFailedOlderThanErr error + + expectedIterations int + }{ + { + name: "success", + + expectedIterations: 4, + }, + { + name: "error deleting failed older than", + deleteFailedOlderThanErr: errors.New("some error"), + + expectedIterations: 4, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + cbStore := &mocks.ProcessorStoreMock{ + DeleteOlderThanFunc: func(_ context.Context, _ time.Time) error { + return tc.deleteFailedOlderThanErr + }, + DeleteURLMappingFunc: func(ctx context.Context, instance string) error { + return nil + }, + } + dispatcher := &mocks.DispatcherMock{} + + processor, err := callbacker.NewProcessor(dispatcher, cbStore, nil, "hostname", slog.Default()) + require.NoError(t, err) + defer processor.GracefulStop() + + processor.StartCallbackStoreCleanup(20*time.Millisecond, 50*time.Second) + + time.Sleep(90 * time.Millisecond) + + require.Equal(t, tc.expectedIterations, len(cbStore.DeleteOlderThanCalls())) + }) + } +} diff --git a/internal/callbacker/send_manager.go b/internal/callbacker/send_manager.go deleted file mode 100644 index 943938fc7..000000000 --- a/internal/callbacker/send_manager.go +++ /dev/null @@ -1,354 +0,0 @@ -package callbacker - -/* SendManager */ -/* - -The SendManager is responsible for managing the sequential sending of callbacks to a specified URL. -It supports single and batched callbacks, handles failures by placing the URL in failed state, and ensures -safe storage of unsent callbacks during graceful shutdowns. -The manager operates in various modes: - - ActiveMode (normal sending) - - StoppingMode (for graceful shutdown). - -It processes callbacks from two channels, ensuring either single or batch dispatch, and manages retries based on a failure policy. - -Key components: -- SenderI : responsible for sending callbacks - -Sending logic: callbacks are sent to the designated URL one at a time, ensuring sequential and orderly processing. - -Failure handling: if a URL fails to respond with a success status, the URL is placed in failed state (based on a defined policy). - During this period, all callbacks for the failed URL are stored with a failed timestamp, preventing further dispatch attempts until the we retry again. - -Graceful Shutdown: on service termination, the SendManager ensures that any unsent callbacks are safely persisted in the store, ensuring no loss of data during shutdown. - -*/ - -import ( - "context" - "log/slog" - "math" - "sync" - "time" - - "github.com/bitcoin-sv/arc/internal/callbacker/store" -) - -type SendManager struct { - url string - - // dependencies - sender SenderI - store store.CallbackerStore - logger *slog.Logger - - expiration time.Duration - - // internal state - entries chan *CallbackEntry - batchEntries chan *CallbackEntry - - stop chan struct{} - - sendDelay time.Duration - singleSendSleep time.Duration - batchSendInterval time.Duration - delayDuration time.Duration - - modeMu sync.Mutex - mode mode -} - -type mode uint8 - -var infinity = time.Date(2999, time.January, 1, 0, 0, 0, 0, time.UTC) - -const ( - IdleMode mode = iota - ActiveMode - StoppingMode - - entriesBufferSize = 10000 -) - -func WithBufferSize(size int) func(*SendManager) { - return func(m *SendManager) { - m.entries = make(chan *CallbackEntry, size) - m.batchEntries = make(chan *CallbackEntry, size) - } -} - -func RunNewSendManager(url string, sender SenderI, store store.CallbackerStore, logger *slog.Logger, sendingConfig *SendConfig, opts ...func(*SendManager)) *SendManager { - const defaultBatchSendInterval = 5 * time.Second - - batchSendInterval := defaultBatchSendInterval - if sendingConfig.BatchSendInterval != 0 { - batchSendInterval = sendingConfig.BatchSendInterval - } - - m := &SendManager{ - url: url, - sender: sender, - store: store, - logger: logger, - - sendDelay: sendingConfig.Delay, - singleSendSleep: sendingConfig.PauseAfterSingleModeSuccessfulSend, - batchSendInterval: batchSendInterval, - delayDuration: sendingConfig.DelayDuration, - expiration: sendingConfig.Expiration, - - entries: make(chan *CallbackEntry, entriesBufferSize), - batchEntries: make(chan *CallbackEntry, entriesBufferSize), - stop: make(chan struct{}), - } - - for _, opt := range opts { - opt(m) - } - - m.run() - return m -} - -func (m *SendManager) Add(entry *CallbackEntry, batch bool) { - if batch { - select { - case m.batchEntries <- entry: - default: - m.logger.Warn("Batch entry buffer is full - storing entry on DB", - slog.String("url", m.url), - slog.String("token", entry.Token), - slog.String("hash", entry.Data.TxID), - ) - m.storeToDB(entry, ptrTo(time.Now())) - } - return - } - - select { - case m.entries <- entry: - default: - m.logger.Warn("Single entry buffer is full - storing entry on DB", - slog.String("url", m.url), - slog.String("token", entry.Token), - slog.String("hash", entry.Data.TxID), - ) - m.storeToDB(entry, ptrTo(time.Now())) - } -} - -func (m *SendManager) storeToDB(entry *CallbackEntry, postponeUntil *time.Time) { - if entry == nil { - return - } - callbackData := toStoreDto(m.url, entry, postponeUntil, false) - err := m.store.Set(context.Background(), callbackData) - if err != nil { - m.logger.Error("Failed to set callback data", slog.String("hash", callbackData.TxID), slog.String("status", callbackData.TxStatus), slog.String("err", err.Error())) - } -} - -func (m *SendManager) GracefulStop() { - m.setMode(StoppingMode) // signal the `run` goroutine to stop sending callbacks - - // signal the `run` goroutine to exit - close(m.entries) - close(m.batchEntries) - - <-m.stop // wait for the `run` goroutine to exit - close(m.stop) -} - -func (m *SendManager) run() { - m.setMode(ActiveMode) - - go func() { - var danglingCallbacks []*store.CallbackData - var danglingBatchedCallbacks []*store.CallbackData - - var runWg sync.WaitGroup - runWg.Add(2) - - go func() { - defer runWg.Done() - danglingCallbacks = m.consumeSingleCallbacks() - }() - - go func() { - defer runWg.Done() - danglingBatchedCallbacks = m.consumeBatchedCallbacks() - }() - - runWg.Wait() - - totalCallbacks := append(danglingCallbacks, danglingBatchedCallbacks...) - m.logger.Info("Storing pending callbacks", slog.Int("count", len(totalCallbacks))) - - // store unsent callbacks - err := m.store.SetMany(context.Background(), totalCallbacks) - if err != nil { - m.logger.Error("Failed to store pending callbacks", slog.Int("count", len(totalCallbacks))) - } - m.stop <- struct{}{} - }() -} - -func (m *SendManager) consumeSingleCallbacks() []*store.CallbackData { - var danglingCallbacks []*store.CallbackData - - for { - callback, ok := <-m.entries - if !ok { - break - } - - switch m.getMode() { - case ActiveMode: - m.send(callback) - case StoppingMode: - // add callback to save - danglingCallbacks = append(danglingCallbacks, toStoreDto(m.url, callback, nil, false)) - } - } - - return danglingCallbacks -} - -func (m *SendManager) consumeBatchedCallbacks() []*store.CallbackData { - const batchSize = 50 - var danglingCallbacks []*store.CallbackData - - var callbacks []*CallbackEntry - sendInterval := time.NewTicker(m.batchSendInterval) - -runLoop: - for { - select { - // put callback to process - case callback, ok := <-m.batchEntries: - if !ok { - break runLoop - } - callbacks = append(callbacks, callback) - - // process batch - case <-sendInterval.C: - if len(callbacks) == 0 { - continue - } - - switch m.getMode() { - case ActiveMode: - // send batch - n := int(math.Min(float64(len(callbacks)), batchSize)) - batch := callbacks[:n] // get n callbacks to send - - m.sendBatch(batch) - callbacks = callbacks[n:] // shrink slice - - case StoppingMode: - // add callback to save - danglingCallbacks = append(danglingCallbacks, toStoreDtoCollection(m.url, nil, true, callbacks)...) - callbacks = nil - } - - sendInterval.Reset(m.batchSendInterval) - } - } - - if len(callbacks) > 0 { - // add callback to save - danglingCallbacks = append(danglingCallbacks, toStoreDtoCollection(m.url, nil, true, callbacks)...) - } - - return danglingCallbacks -} - -func (m *SendManager) getMode() mode { - m.modeMu.Lock() - defer m.modeMu.Unlock() - - return m.mode -} - -func (m *SendManager) setMode(v mode) { - m.modeMu.Lock() - m.mode = v - m.modeMu.Unlock() -} - -func (m *SendManager) send(callback *CallbackEntry) { - // quick fix for client issue with to fast callbacks - time.Sleep(m.sendDelay) - - success, retry := m.sender.Send(m.url, callback.Token, callback.Data) - if !retry || success { - time.Sleep(m.singleSendSleep) - return - } - - until := time.Now().Add(m.delayDuration) - if time.Since(callback.Data.Timestamp) > m.expiration { - until = infinity - } - - err := m.store.Set(context.Background(), toStoreDto(m.url, callback, &until, false)) - if err != nil { - m.logger.Error("failed to store failed callback in db", slog.String("url", m.url), slog.String("err", err.Error())) - } -} - -func (m *SendManager) sendBatch(batch []*CallbackEntry) { - token := batch[0].Token - callbacks := make([]*Callback, len(batch)) - for i, e := range batch { - callbacks[i] = e.Data - } - - // quick fix for client issue with to fast callbacks - time.Sleep(m.sendDelay) - - success, retry := m.sender.SendBatch(m.url, token, callbacks) - if !retry || success { - return - } - - until := time.Now().Add(m.delayDuration) - if time.Since(batch[0].Data.Timestamp) > m.expiration { - until = infinity - } - err := m.store.SetMany(context.Background(), toStoreDtoCollection(m.url, &until, true, batch)) - if err != nil { - m.logger.Error("failed to store failed callbacks in db", slog.String("url", m.url), slog.String("err", err.Error())) - } -} - -func toStoreDto(url string, entry *CallbackEntry, postponedUntil *time.Time, allowBatch bool) *store.CallbackData { - return &store.CallbackData{ - URL: url, - Token: entry.Token, - Timestamp: entry.Data.Timestamp, - - CompetingTxs: entry.Data.CompetingTxs, - TxID: entry.Data.TxID, - TxStatus: entry.Data.TxStatus, - ExtraInfo: entry.Data.ExtraInfo, - MerklePath: entry.Data.MerklePath, - - BlockHash: entry.Data.BlockHash, - BlockHeight: entry.Data.BlockHeight, - - PostponedUntil: postponedUntil, - AllowBatch: allowBatch, - } -} - -func toStoreDtoCollection(url string, postponedUntil *time.Time, allowBatch bool, entries []*CallbackEntry) []*store.CallbackData { - res := make([]*store.CallbackData, len(entries)) - for i, e := range entries { - res[i] = toStoreDto(url, e, postponedUntil, allowBatch) - } - - return res -} diff --git a/internal/callbacker/send_manager_test.go b/internal/callbacker/send_manager_test.go deleted file mode 100644 index 0ae5747f7..000000000 --- a/internal/callbacker/send_manager_test.go +++ /dev/null @@ -1,306 +0,0 @@ -package callbacker_test - -import ( - "context" - "errors" - "fmt" - "log/slog" - "sync/atomic" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/bitcoin-sv/arc/internal/callbacker" - "github.com/bitcoin-sv/arc/internal/callbacker/mocks" - "github.com/bitcoin-sv/arc/internal/callbacker/store" -) - -func TestSendManager(t *testing.T) { - tcs := []struct { - name string - singleSendInterval time.Duration - numOfSingleCallbacks int - numOfBatchedCallbacks int - stopManager bool - setEntriesBufferSize int - setErr error - }{ - { - name: "send only single callbacks when run", - singleSendInterval: 0, - numOfSingleCallbacks: 100, - numOfBatchedCallbacks: 0, - }, - { - name: "save callbacks on stopping (only single callbacks)", - singleSendInterval: time.Millisecond, // set interval to give time to call stop function - numOfSingleCallbacks: 10, - numOfBatchedCallbacks: 0, - stopManager: true, - }, - { - name: "send only batched callbacks when run", - numOfSingleCallbacks: 0, - numOfBatchedCallbacks: 493, - }, - { - name: "save callbacks on stopping (only batched callbacks)", - numOfSingleCallbacks: 0, - numOfBatchedCallbacks: 451, - stopManager: true, - }, - { - name: "send mixed callbacks when run", - singleSendInterval: 0, - numOfSingleCallbacks: 100, - numOfBatchedCallbacks: 501, - }, - { - name: "entry buffer size exceeds limit", - singleSendInterval: 0, - numOfSingleCallbacks: 10, - numOfBatchedCallbacks: 10, - setEntriesBufferSize: 5, - stopManager: true, - }, - { - name: "entry buffer size exceeds limit - set error", - singleSendInterval: 0, - numOfSingleCallbacks: 10, - numOfBatchedCallbacks: 10, - setEntriesBufferSize: 5, - setErr: errors.New("failed to set entry"), - stopManager: true, - }, - { - name: "save callbacks on stopping (mixed callbacks)", - singleSendInterval: time.Millisecond, // set interval to give time to call stop function - numOfSingleCallbacks: 10, - numOfBatchedCallbacks: 375, - stopManager: true, - }, - } - - for _, tc := range tcs { - t.Run(tc.name, func(t *testing.T) { - // given - cMq := &mocks.SenderIMock{ - SendFunc: func(_, _ string, _ *callbacker.Callback) (bool, bool) { return true, false }, - SendBatchFunc: func(_, _ string, _ []*callbacker.Callback) (bool, bool) { return true, false }, - } - - var savedCallbacks []*store.CallbackData - sMq := &mocks.CallbackerStoreMock{ - SetManyFunc: func(_ context.Context, data []*store.CallbackData) error { - savedCallbacks = append(savedCallbacks, data...) - return nil - }, - SetFunc: func(_ context.Context, data *store.CallbackData) error { - savedCallbacks = append(savedCallbacks, data) - return tc.setErr - }, - } - - sendConfig := &callbacker.SendConfig{ - Delay: 0, - PauseAfterSingleModeSuccessfulSend: 0, - BatchSendInterval: time.Millisecond, - } - - var opts []func(manager *callbacker.SendManager) - if tc.setEntriesBufferSize > 0 { - opts = append(opts, callbacker.WithBufferSize(tc.setEntriesBufferSize)) - } - - sut := callbacker.RunNewSendManager("", cMq, sMq, slog.Default(), sendConfig, opts...) - - // add callbacks before starting the manager to queue them - for range tc.numOfSingleCallbacks { - sut.Add(&callbacker.CallbackEntry{Data: &callbacker.Callback{}}, false) - } - for range tc.numOfBatchedCallbacks { - sut.Add(&callbacker.CallbackEntry{Data: &callbacker.Callback{}}, true) - } - - if tc.stopManager { - sut.GracefulStop() - } else { - // give a chance to process - time.Sleep(200 * time.Millisecond) - } - - // then - // check if manager sends callbacks in correct way - if tc.numOfSingleCallbacks == 0 { - require.Equal(t, 0, len(cMq.SendCalls())) - } - if tc.numOfBatchedCallbacks == 0 { - require.Equal(t, 0, len(cMq.SendBatchCalls())) - } - - // check if all callbacks were consumed - expectedTotal := tc.numOfSingleCallbacks + tc.numOfBatchedCallbacks - actualBatchedSent := 0 - for _, c := range cMq.SendBatchCalls() { - actualBatchedSent += len(c.Callbacks) - } - - actualTotal := len(cMq.SendCalls()) + actualBatchedSent + len(savedCallbacks) - - require.Equal(t, expectedTotal, actualTotal) - - if tc.stopManager { - // manager should save some callbacks on stoping instead of sending all of them - require.NotEmpty(t, savedCallbacks) - } else { - // if manager was not stopped it should send all callbacks - require.Empty(t, savedCallbacks) - } - }) - } -} - -func TestSendManager_FailedCallbacks(t *testing.T) { - /* Failure scenario - 1. sending failed - 2. put manager in failed state for a specified duration - 3. store all callbacks during the failure period - 4. switch manager to active mode once failure duration is over - 5. send new callbacks again - */ - - tt := []struct { - name string - batch bool - }{ - { - name: "send single callbacks", - }, - { - name: "send batched callbacks", - batch: true, - }, - } - - for _, tc := range tt { - t.Run(tc.name, func(t *testing.T) { - // given - var sendOK int32 - senderMq := &mocks.SenderIMock{ - SendFunc: func(_, _ string, _ *callbacker.Callback) (bool, bool) { - return atomic.LoadInt32(&sendOK) == 1, atomic.LoadInt32(&sendOK) != 1 - }, - SendBatchFunc: func(_, _ string, _ []*callbacker.Callback) (bool, bool) { - return atomic.LoadInt32(&sendOK) == 1, atomic.LoadInt32(&sendOK) != 1 - }, - } - - storeMq := &mocks.CallbackerStoreMock{ - SetFunc: func(_ context.Context, _ *store.CallbackData) error { - return nil - }, - SetManyFunc: func(_ context.Context, _ []*store.CallbackData) error { - return nil - }, - } - - var preFailureCallbacks []*callbacker.CallbackEntry - for i := range 10 { - preFailureCallbacks = append(preFailureCallbacks, &callbacker.CallbackEntry{Data: &callbacker.Callback{TxID: fmt.Sprintf("q %d", i)}}) - } - - var postFailureCallbacks []*callbacker.CallbackEntry - for i := range 10 { - postFailureCallbacks = append(postFailureCallbacks, &callbacker.CallbackEntry{Data: &callbacker.Callback{TxID: fmt.Sprintf("a %d", i)}}) - } - - sendConfig := &callbacker.SendConfig{ - Delay: 0, - PauseAfterSingleModeSuccessfulSend: 0, - BatchSendInterval: time.Millisecond, - } - - sut := callbacker.RunNewSendManager("http://unittest.com", senderMq, storeMq, slog.Default(), sendConfig) - - // when - atomic.StoreInt32(&sendOK, 0) // trigger send failure - this should put the manager in failed state - - // add a few callbacks to send - all should be stored - for _, c := range preFailureCallbacks { - sut.Add(c, tc.batch) - } - - time.Sleep(500 * time.Millisecond) // wait for the failure period to complete - - atomic.StoreInt32(&sendOK, 1) // now all sends should complete successfully - // add a few callbacks to send - all should be sent - for _, c := range postFailureCallbacks { - sut.Add(c, tc.batch) - } - - // give a chance to process - time.Sleep(500 * time.Millisecond) - - // then - // check stored callbacks during failure - var storedCallbacks []*store.CallbackData - if tc.batch { - for _, c := range storeMq.SetManyCalls() { - storedCallbacks = append(storedCallbacks, c.Data...) - } - } else { - for _, c := range storeMq.SetCalls() { - storedCallbacks = append(storedCallbacks, c.Dto) - } - } - - require.Equal(t, len(preFailureCallbacks), len(storedCallbacks), "all callbacks sent during failure should be stored") - for _, c := range preFailureCallbacks { - _, ok := find(storedCallbacks, func(e *store.CallbackData) bool { - return e.TxID == c.Data.TxID - }) - - require.True(t, ok) - } - - // check sent callbacks - var sendCallbacks []*callbacker.Callback - if tc.batch { - for _, c := range senderMq.SendBatchCalls() { - sendCallbacks = append(sendCallbacks, c.Callbacks...) - } - } else { - for _, c := range senderMq.SendCalls() { - sendCallbacks = append(sendCallbacks, c.Callback) - } - } - - require.Equal(t, len(postFailureCallbacks)+len(preFailureCallbacks), len(sendCallbacks), "manager should attempt to send the callback that caused failure (first call) and all callbacks sent after failure") - - _, ok := find(sendCallbacks, func(e *callbacker.Callback) bool { - return e.TxID == preFailureCallbacks[0].Data.TxID - }) - - require.True(t, ok) - - for _, c := range postFailureCallbacks { - _, ok := find(sendCallbacks, func(e *callbacker.Callback) bool { - return e.TxID == c.Data.TxID - }) - - require.True(t, ok) - } - }) - } -} - -func find[T any](arr []T, predicate func(T) bool) (T, bool) { - for _, element := range arr { - if predicate(element) { - return element, true - } - } - var zero T - return zero, false -} diff --git a/internal/callbacker/server.go b/internal/callbacker/server.go index c835ce62e..8b6de461d 100644 --- a/internal/callbacker/server.go +++ b/internal/callbacker/server.go @@ -17,11 +17,11 @@ import ( type Server struct { callbacker_api.UnimplementedCallbackerAPIServer grpc_opts.GrpcServer - dispatcher *CallbackDispatcher + dispatcher Dispatcher } // NewServer will return a server instance -func NewServer(prometheusEndpoint string, maxMsgSize int, logger *slog.Logger, dispatcher *CallbackDispatcher, tracingConfig *config.TracingConfig) (*Server, error) { +func NewServer(prometheusEndpoint string, maxMsgSize int, logger *slog.Logger, dispatcher Dispatcher, tracingConfig *config.TracingConfig) (*Server, error) { logger = logger.With(slog.String("module", "server")) grpcServer, err := grpc_opts.NewGrpcServer(logger, "callbacker", prometheusEndpoint, maxMsgSize, tracingConfig) @@ -50,7 +50,7 @@ func (s *Server) SendCallback(_ context.Context, request *callbacker_api.SendCal dto := toCallbackDto(request) for _, r := range request.CallbackRoutings { if r.Url != "" { - s.dispatcher.Dispatch(r.Url, &CallbackEntry{Token: r.Token, Data: dto}, r.AllowBatch) + s.dispatcher.Dispatch(r.Url, &CallbackEntry{Token: r.Token, Data: dto, AllowBatch: r.AllowBatch}) } } diff --git a/internal/callbacker/server_test.go b/internal/callbacker/server_test.go index 4803dc3ab..795f44388 100644 --- a/internal/callbacker/server_test.go +++ b/internal/callbacker/server_test.go @@ -13,7 +13,6 @@ import ( "github.com/bitcoin-sv/arc/internal/callbacker" "github.com/bitcoin-sv/arc/internal/callbacker/callbacker_api" "github.com/bitcoin-sv/arc/internal/callbacker/mocks" - "github.com/bitcoin-sv/arc/internal/callbacker/store" ) func TestNewServer(t *testing.T) { @@ -56,27 +55,16 @@ func TestHealth(t *testing.T) { func TestSendCallback(t *testing.T) { t.Run("dispatches callback for each routing", func(t *testing.T) { // Given - sendOK := true - senderMq := &mocks.SenderIMock{ - SendFunc: func(_, _ string, _ *callbacker.Callback) (bool, bool) { return sendOK, false }, - SendBatchFunc: func(_, _ string, _ []*callbacker.Callback) (bool, bool) { return sendOK, false }, - } - - storeMq := &mocks.CallbackerStoreMock{ - SetFunc: func(_ context.Context, _ *store.CallbackData) error { - return nil - }, - SetManyFunc: func(_ context.Context, _ []*store.CallbackData) error { - return nil - }, - } - - mockDispatcher := callbacker.NewCallbackDispatcher( - senderMq, - storeMq, - slog.Default(), - &callbacker.SendConfig{Expiration: time.Duration(24 * time.Hour)}, - ) + mockDispatcher := &mocks.DispatcherMock{DispatchFunc: func(url string, dto *callbacker.CallbackEntry) { + switch url { + case "https://example.com/callback1": + require.Equal(t, dto.Token, "token1") + case "https://example.com/callback2": + require.Equal(t, dto.Token, "token2") + default: + t.Fatalf("unexpected callback URL: %s", url) + } + }} server, err := callbacker.NewServer("", 0, slog.Default(), mockDispatcher, nil) require.NoError(t, err) @@ -85,8 +73,8 @@ func TestSendCallback(t *testing.T) { Txid: "1234", Status: callbacker_api.Status_SEEN_ON_NETWORK, CallbackRoutings: []*callbacker_api.CallbackRouting{ - {Url: "http://example.com/callback1", Token: "token1", AllowBatch: false}, - {Url: "http://example.com/callback2", Token: "token2", AllowBatch: false}, + {Url: "https://example.com/callback1", Token: "token1", AllowBatch: false}, + {Url: "https://example.com/callback2", Token: "token2", AllowBatch: false}, }, BlockHash: "abcd1234", BlockHeight: 100, @@ -102,24 +90,7 @@ func TestSendCallback(t *testing.T) { assert.NoError(t, err) assert.IsType(t, &emptypb.Empty{}, resp) time.Sleep(100 * time.Millisecond) - require.Equal(t, 2, len(senderMq.SendCalls()), "Expected two dispatch calls") - - calls0 := senderMq.SendCalls()[0] - calls1 := senderMq.SendCalls()[1] - - switch calls0.URL { - case "http://example.com/callback1": - require.Equal(t, calls0.Token, "token1") - require.Equal(t, calls1.URL, "http://example.com/callback2") - require.Equal(t, calls1.Token, "token2") - case "http://example.com/callback2": - require.Equal(t, calls0.Token, "token2") - require.Equal(t, calls1.URL, "http://example.com/callback1") - require.Equal(t, calls1.Token, "token1") - default: - t.Fatalf("unexpected callback URL: %s", calls0.URL) - } + require.Equal(t, 2, len(mockDispatcher.DispatchCalls()), "Expected two dispatch calls") - mockDispatcher.GracefulStop() }) } diff --git a/internal/callbacker/store/postgresql/fixtures/delete_failed_older_than/callbacker.callbacks.yaml b/internal/callbacker/store/postgresql/fixtures/delete_older_than/callbacker.callbacks.yaml similarity index 94% rename from internal/callbacker/store/postgresql/fixtures/delete_failed_older_than/callbacker.callbacks.yaml rename to internal/callbacker/store/postgresql/fixtures/delete_older_than/callbacker.callbacks.yaml index 8510e0593..ab345ebef 100644 --- a/internal/callbacker/store/postgresql/fixtures/delete_failed_older_than/callbacker.callbacks.yaml +++ b/internal/callbacker/store/postgresql/fixtures/delete_older_than/callbacker.callbacks.yaml @@ -2,22 +2,21 @@ token: token tx_id: 96cbf8ba96dc3bad6ecc19ce34d1edbf57b2bc6f76cc3d80efdca95599cf5c28 tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:00 + timestamp: 2024-09-01 12:30:00 - url: https://arc-callback-1/callback token: token tx_id: 96cbf8ba96dc3bad6ecc19ce34d1edbf57b2bc6f76cc3d80efdca95599cf5c28 tx_status: "MINED" - timestamp: 2024-09-01 12:01:00 + timestamp: 2024-09-01 12:30:00 block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-2/callback token: token tx_id: 96cbf8ba96dc3bad6ecc19ce34d1edbf57b2bc6f76cc3d80efdca95599cf5c28 tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:00 + timestamp: 2024-09-01 12:30:00 - url: https://arc-callback-2/callback token: token @@ -26,7 +25,6 @@ timestamp: 2024-09-01 12:01:00 block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-1/callback token: token @@ -41,7 +39,6 @@ timestamp: 2024-09-01 12:01:01 block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-2/callback token: token @@ -62,7 +59,6 @@ tx_id: 277fb619a6ee37757123301fce61884e741ab4e01a0dea7ec465ae74f43f82cc tx_status: "SEEN_ON_NETWORK" timestamp: 2024-09-01 12:00:02 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-1/callback token: token @@ -77,7 +73,6 @@ tx_id: 277fb619a6ee37757123301fce61884e741ab4e01a0dea7ec465ae74f43f82cc tx_status: "SEEN_ON_NETWORK" timestamp: 2024-09-01 12:00:02 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-2/callback token: token @@ -100,7 +95,6 @@ timestamp: 2024-09-01 12:01:02 block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-2/callback token: token @@ -115,7 +109,6 @@ timestamp: 2024-09-01 12:01:02 block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-1/callback token: token @@ -130,7 +123,6 @@ timestamp: 2024-09-01 12:01:02 block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-2/callback token: token @@ -151,7 +143,6 @@ tx_id: 0e27376ded97b656ccc02ecd6948d2e775d8904e7093eb89e8d2bf1eb7a60ea9 tx_status: "SEEN_ON_NETWORK" timestamp: 2024-09-01 12:00:02 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-1/callback token: token @@ -166,7 +157,6 @@ tx_id: 0e27376ded97b656ccc02ecd6948d2e775d8904e7093eb89e8d2bf1eb7a60ea9 tx_status: "SEEN_ON_NETWORK" timestamp: 2024-09-01 12:00:02 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-2/callback token: token @@ -203,7 +193,6 @@ timestamp: 2024-09-01 12:01:02 block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-1/callback token: token @@ -218,14 +207,12 @@ timestamp: 2024-09-01 12:01:02 block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad block_height: 860339 - postponed_until: 2024-09-01 12:31:10 - url: https://arc-callback-2/callback token: token tx_id: c50eeb84c58780bb0eb39f430deb93e5f362fe16b73aa4f811c089e14a1815ae tx_status: "SEEN_ON_NETWORK" timestamp: 2024-09-01 12:00:02 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-2/callback token: token @@ -254,7 +241,6 @@ tx_id: 6de67f42990ab43f7cc480d6339f13932fe67fa8971a59aa34b93b5dac734c3e tx_status: "SEEN_ON_NETWORK" timestamp: 2024-09-01 12:00:02 - postponed_until: 2024-09-01 12:11:10 - url: https://arc-callback-2/callback token: token @@ -277,7 +263,6 @@ timestamp: 2024-09-01 12:01:03 block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad block_height: 860339 - postponed_until: 2024-09-01 12:31:10 - url: https://arc-callback-2/callback token: token @@ -390,7 +375,6 @@ timestamp: 2024-09-01 12:01:04 block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad block_height: 860339 - postponed_until: 2024-09-01 12:31:10 - url: https://arc-callback-2/callback token: token @@ -405,4 +389,3 @@ timestamp: 2024-09-01 12:01:04 block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad block_height: 860339 - postponed_until: 2024-09-01 12:31:10 diff --git a/internal/callbacker/store/postgresql/fixtures/pop_many/callbacker.callbacks.yaml b/internal/callbacker/store/postgresql/fixtures/get_and_delete/callbacker.callbacks.yaml similarity index 100% rename from internal/callbacker/store/postgresql/fixtures/pop_many/callbacker.callbacks.yaml rename to internal/callbacker/store/postgresql/fixtures/get_and_delete/callbacker.callbacks.yaml diff --git a/internal/callbacker/store/postgresql/fixtures/pop_failed_many/callbacker.callbacks.yaml b/internal/callbacker/store/postgresql/fixtures/pop_failed_many/callbacker.callbacks.yaml deleted file mode 100644 index 8510e0593..000000000 --- a/internal/callbacker/store/postgresql/fixtures/pop_failed_many/callbacker.callbacks.yaml +++ /dev/null @@ -1,408 +0,0 @@ -- url: https://arc-callback-1/callback - token: token - tx_id: 96cbf8ba96dc3bad6ecc19ce34d1edbf57b2bc6f76cc3d80efdca95599cf5c28 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:00 - -- url: https://arc-callback-1/callback - token: token - tx_id: 96cbf8ba96dc3bad6ecc19ce34d1edbf57b2bc6f76cc3d80efdca95599cf5c28 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:00 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-2/callback - token: token - tx_id: 96cbf8ba96dc3bad6ecc19ce34d1edbf57b2bc6f76cc3d80efdca95599cf5c28 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:00 - -- url: https://arc-callback-2/callback - token: token - tx_id: 96cbf8ba96dc3bad6ecc19ce34d1edbf57b2bc6f76cc3d80efdca95599cf5c28 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:00 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-1/callback - token: token - tx_id: 3413cc9b40d48661c7f36bee88ebb39fca1d593f9672f840afdf07b018e73bb7 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:01 - -- url: https://arc-callback-1/callback - token: token - tx_id: 3413cc9b40d48661c7f36bee88ebb39fca1d593f9672f840afdf07b018e73bb7 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:01 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-2/callback - token: token - tx_id: 3413cc9b40d48661c7f36bee88ebb39fca1d593f9672f840afdf07b018e73bb7 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:01 - -- url: https://arc-callback-2/callback - token: token - tx_id: 3413cc9b40d48661c7f36bee88ebb39fca1d593f9672f840afdf07b018e73bb7 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:01 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-1/callback - token: token - tx_id: 277fb619a6ee37757123301fce61884e741ab4e01a0dea7ec465ae74f43f82cc - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-1/callback - token: token - tx_id: 277fb619a6ee37757123301fce61884e741ab4e01a0dea7ec465ae74f43f82cc - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-2/callback - token: token - tx_id: 277fb619a6ee37757123301fce61884e741ab4e01a0dea7ec465ae74f43f82cc - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-2/callback - token: token - tx_id: 277fb619a6ee37757123301fce61884e741ab4e01a0dea7ec465ae74f43f82cc - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-1/callback - token: token - tx_id: 13441601b8f4bd6062ce113118e957c04442a3293360fffbe0ed8805c34c6343 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - -- url: https://arc-callback-1/callback - token: token - tx_id: 13441601b8f4bd6062ce113118e957c04442a3293360fffbe0ed8805c34c6343 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-2/callback - token: token - tx_id: 13441601b8f4bd6062ce113118e957c04442a3293360fffbe0ed8805c34c6343 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - -- url: https://arc-callback-2/callback - token: token - tx_id: 13441601b8f4bd6062ce113118e957c04442a3293360fffbe0ed8805c34c6343 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-1/callback - token: token - tx_id: 862f7781e2e65efddd3cb6bb3a924ea53edba299354991eb38bf47ec6e5c986c - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - -- url: https://arc-callback-1/callback - token: token - tx_id: 862f7781e2e65efddd3cb6bb3a924ea53edba299354991eb38bf47ec6e5c986c - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-2/callback - token: token - tx_id: 862f7781e2e65efddd3cb6bb3a924ea53edba299354991eb38bf47ec6e5c986c - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - -- url: https://arc-callback-2/callback - token: token - tx_id: 862f7781e2e65efddd3cb6bb3a924ea53edba299354991eb38bf47ec6e5c986c - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-1/callback - token: token - tx_id: 0e27376ded97b656ccc02ecd6948d2e775d8904e7093eb89e8d2bf1eb7a60ea9 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-1/callback - token: token - tx_id: 0e27376ded97b656ccc02ecd6948d2e775d8904e7093eb89e8d2bf1eb7a60ea9 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-2/callback - token: token - tx_id: 0e27376ded97b656ccc02ecd6948d2e775d8904e7093eb89e8d2bf1eb7a60ea9 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-2/callback - token: token - tx_id: 0e27376ded97b656ccc02ecd6948d2e775d8904e7093eb89e8d2bf1eb7a60ea9 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-1/callback - token: token - tx_id: 6bcfe17b6b41511ee891401998caefb7ffdcee87653e197dfb1add5860b6a070 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - -- url: https://arc-callback-1/callback - token: token - tx_id: 6bcfe17b6b41511ee891401998caefb7ffdcee87653e197dfb1add5860b6a070 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-2/callback - token: token - tx_id: 6bcfe17b6b41511ee891401998caefb7ffdcee87653e197dfb1add5860b6a070 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - -- url: https://arc-callback-2/callback - token: token - tx_id: 6bcfe17b6b41511ee891401998caefb7ffdcee87653e197dfb1add5860b6a070 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-1/callback - token: token - tx_id: c50eeb84c58780bb0eb39f430deb93e5f362fe16b73aa4f811c089e14a1815ae - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - -- url: https://arc-callback-1/callback - token: token - tx_id: c50eeb84c58780bb0eb39f430deb93e5f362fe16b73aa4f811c089e14a1815ae - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - postponed_until: 2024-09-01 12:31:10 - -- url: https://arc-callback-2/callback - token: token - tx_id: c50eeb84c58780bb0eb39f430deb93e5f362fe16b73aa4f811c089e14a1815ae - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-2/callback - token: token - tx_id: c50eeb84c58780bb0eb39f430deb93e5f362fe16b73aa4f811c089e14a1815ae - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-1/callback - token: token - tx_id: 6de67f42990ab43f7cc480d6339f13932fe67fa8971a59aa34b93b5dac734c3e - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - -- url: https://arc-callback-1/callback - token: token - tx_id: 6de67f42990ab43f7cc480d6339f13932fe67fa8971a59aa34b93b5dac734c3e - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-2/callback - token: token - tx_id: 6de67f42990ab43f7cc480d6339f13932fe67fa8971a59aa34b93b5dac734c3e - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:02 - postponed_until: 2024-09-01 12:11:10 - -- url: https://arc-callback-2/callback - token: token - tx_id: 6de67f42990ab43f7cc480d6339f13932fe67fa8971a59aa34b93b5dac734c3e - tx_status: "MINED" - timestamp: 2024-09-01 12:01:02 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-1/callback - token: token - tx_id: d23d87582b477d4cdcfd92537cc44689220c753a716f355c35faa4856562d331 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:03 - -- url: https://arc-callback-1/callback - token: token - tx_id: d23d87582b477d4cdcfd92537cc44689220c753a716f355c35faa4856562d331 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:03 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - postponed_until: 2024-09-01 12:31:10 - -- url: https://arc-callback-2/callback - token: token - tx_id: d23d87582b477d4cdcfd92537cc44689220c753a716f355c35faa4856562d331 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:03 - -- url: https://arc-callback-2/callback - token: token - tx_id: d23d87582b477d4cdcfd92537cc44689220c753a716f355c35faa4856562d331 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:03 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-1/callback - token: token - tx_id: 975d0526e7a2d47225f266cb1c4bc2abad7f2c4a976dd9bd1381792d647d5430 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:03 - -- url: https://arc-callback-1/callback - token: token - tx_id: 975d0526e7a2d47225f266cb1c4bc2abad7f2c4a976dd9bd1381792d647d5430 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:03 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-2/callback - token: token - tx_id: 975d0526e7a2d47225f266cb1c4bc2abad7f2c4a976dd9bd1381792d647d5430 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:03 - -- url: https://arc-callback-2/callback - token: token - tx_id: 975d0526e7a2d47225f266cb1c4bc2abad7f2c4a976dd9bd1381792d647d5430 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:03 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-1/callback - token: token - tx_id: 8e84ed2ef2264cfa1f2a00c218329d8862d09a06ae5f6ad62d2f1b8069b13a64 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:04 - -- url: https://arc-callback-1/callback - token: token - tx_id: 8e84ed2ef2264cfa1f2a00c218329d8862d09a06ae5f6ad62d2f1b8069b13a64 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:04 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-2/callback - token: token - tx_id: 8e84ed2ef2264cfa1f2a00c218329d8862d09a06ae5f6ad62d2f1b8069b13a64 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:04 - -- url: https://arc-callback-2/callback - token: token - tx_id: 8e84ed2ef2264cfa1f2a00c218329d8862d09a06ae5f6ad62d2f1b8069b13a64 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:04 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-1/callback - token: token - tx_id: 292ff388160b2814473af90c289ed007b2f1b38cb02ba64cafab1af3b15e41d0 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:04 - -- url: https://arc-callback-1/callback - token: token - tx_id: 292ff388160b2814473af90c289ed007b2f1b38cb02ba64cafab1af3b15e41d0 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:04 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-2/callback - token: token - tx_id: 292ff388160b2814473af90c289ed007b2f1b38cb02ba64cafab1af3b15e41d0 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:04 - -- url: https://arc-callback-2/callback - token: token - tx_id: 292ff388160b2814473af90c289ed007b2f1b38cb02ba64cafab1af3b15e41d0 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:04 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - -- url: https://arc-callback-1/callback - token: token - tx_id: 0011f2d1abb9eec2c3b6194c2aae6acfb819f2cbd9faa21a31646dc597fbec78 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:04 - -- url: https://arc-callback-1/callback - token: token - tx_id: 0011f2d1abb9eec2c3b6194c2aae6acfb819f2cbd9faa21a31646dc597fbec78 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:04 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - postponed_until: 2024-09-01 12:31:10 - -- url: https://arc-callback-2/callback - token: token - tx_id: 0011f2d1abb9eec2c3b6194c2aae6acfb819f2cbd9faa21a31646dc597fbec78 - tx_status: "SEEN_ON_NETWORK" - timestamp: 2024-09-01 12:00:04 - -- url: https://arc-callback-2/callback - token: token - tx_id: 0011f2d1abb9eec2c3b6194c2aae6acfb819f2cbd9faa21a31646dc597fbec78 - tx_status: "MINED" - timestamp: 2024-09-01 12:01:04 - block_hash: 0000000000000000086527da012efb2d45e00fba9f31e84c35dce998abb409ad - block_height: 860339 - postponed_until: 2024-09-01 12:31:10 diff --git a/internal/callbacker/store/postgresql/internal/tests/utils.go b/internal/callbacker/store/postgresql/internal/tests/utils.go index 8baa47b7b..828af1b31 100644 --- a/internal/callbacker/store/postgresql/internal/tests/utils.go +++ b/internal/callbacker/store/postgresql/internal/tests/utils.go @@ -2,7 +2,6 @@ package tests import ( "database/sql" - "fmt" "reflect" "strings" "testing" @@ -28,7 +27,6 @@ func ReadAllCallbacks(t *testing.T, db *sql.DB) []*store.CallbackData { ,block_height ,timestamp ,competing_txs - ,postponed_until FROM callbacker.callbacks`, ) @@ -46,9 +44,8 @@ func ReadAllCallbacks(t *testing.T, db *sql.DB) []*store.CallbackData { var bh sql.NullString var bheight sql.NullInt64 var competingTxs sql.NullString - var pUntil sql.NullTime - _ = r.Scan(&c.URL, &c.Token, &c.TxID, &c.TxStatus, &ei, &mp, &bh, &bheight, &c.Timestamp, &competingTxs, &pUntil) + _ = r.Scan(&c.URL, &c.Token, &c.TxID, &c.TxStatus, &ei, &mp, &bh, &bheight, &c.Timestamp, &competingTxs) if ei.Valid { c.ExtraInfo = &ei.String @@ -65,9 +62,7 @@ func ReadAllCallbacks(t *testing.T, db *sql.DB) []*store.CallbackData { if competingTxs.Valid { c.CompetingTxs = strings.Split(competingTxs.String, ",") } - if pUntil.Valid { - c.PostponedUntil = ptrTo(pUntil.Time.UTC()) - } + c.Timestamp = c.Timestamp.UTC() callbacks = append(callbacks, c) @@ -76,32 +71,6 @@ func ReadAllCallbacks(t *testing.T, db *sql.DB) []*store.CallbackData { return callbacks } -func CountCallbacks(t *testing.T, db *sql.DB) int { - t.Helper() - - var count int - row := db.QueryRow("SELECT COUNT(1) FROM callbacker.callbacks") - - if err := row.Scan(&count); err != nil { - t.Fatal(err) - } - - return count -} - -func CountCallbacksWhere(t *testing.T, db *sql.DB, predicate string) int { - t.Helper() - - var count int - row := db.QueryRow(fmt.Sprintf("SELECT COUNT(1) FROM callbacker.callbacks WHERE %s", predicate)) - - if err := row.Scan(&count); err != nil { - t.Fatal(err) - } - - return count -} - func ptrTo[T any](v T) *T { return &v } diff --git a/internal/callbacker/store/postgresql/migrations/000005_drop_postponed_until.down.sql b/internal/callbacker/store/postgresql/migrations/000005_drop_postponed_until.down.sql new file mode 100644 index 000000000..de596aee8 --- /dev/null +++ b/internal/callbacker/store/postgresql/migrations/000005_drop_postponed_until.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE callbacker.callbacks ADD COLUMN postponed_until TIMESTAMPTZ; + +CREATE INDEX ix_callbacks_postponed_until ON callbacker.callbacks (postponed_until); diff --git a/internal/callbacker/store/postgresql/migrations/000005_drop_postponed_until.up.sql b/internal/callbacker/store/postgresql/migrations/000005_drop_postponed_until.up.sql new file mode 100644 index 000000000..39e923dfd --- /dev/null +++ b/internal/callbacker/store/postgresql/migrations/000005_drop_postponed_until.up.sql @@ -0,0 +1,3 @@ +DROP INDEX callbacker.ix_callbacks_postponed_until; + +ALTER TABLE callbacker.callbacks DROP COLUMN postponed_until; diff --git a/internal/callbacker/store/postgresql/postgres.go b/internal/callbacker/store/postgresql/postgres.go index 0b57b9d07..ca61ab7e3 100644 --- a/internal/callbacker/store/postgresql/postgres.go +++ b/internal/callbacker/store/postgresql/postgres.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "errors" - "fmt" "strings" "time" @@ -18,8 +17,7 @@ const ( ) var ( - ErrFailedToOpenDB = errors.New("failed to open postgres DB") - ErrFailedToRollback = errors.New("failed to rollback") + ErrFailedToOpenDB = errors.New("failed to open postgres DB") ) type PostgreSQL struct { @@ -57,7 +55,6 @@ func (p *PostgreSQL) SetMany(ctx context.Context, data []*store.CallbackData) er blockHashes := make([]*string, len(data)) blockHeights := make([]sql.NullInt64, len(data)) competingTxs := make([]*string, len(data)) - delayUntils := make([]sql.NullTime, len(data)) allowBatches := make([]bool, len(data)) for i, d := range data { @@ -78,10 +75,6 @@ func (p *PostgreSQL) SetMany(ctx context.Context, data []*store.CallbackData) er if len(d.CompetingTxs) > 0 { competingTxs[i] = ptrTo(strings.Join(d.CompetingTxs, ",")) } - - if d.PostponedUntil != nil { - delayUntils[i] = sql.NullTime{Time: d.PostponedUntil.UTC(), Valid: true} - } } const query = `INSERT INTO callbacker.callbacks ( @@ -95,7 +88,6 @@ func (p *PostgreSQL) SetMany(ctx context.Context, data []*store.CallbackData) er ,block_height ,timestamp ,competing_txs - ,postponed_until ,allow_batch ) SELECT @@ -109,8 +101,7 @@ func (p *PostgreSQL) SetMany(ctx context.Context, data []*store.CallbackData) er ,UNNEST($8::BIGINT[]) ,UNNEST($9::TIMESTAMPTZ[]) ,UNNEST($10::TEXT[]) - ,UNNEST($11::TIMESTAMPTZ[]) - ,UNNEST($12::BOOLEAN[])` + ,UNNEST($11::BOOLEAN[])` _, err := p.db.ExecContext(ctx, query, pq.Array(urls), @@ -123,127 +114,19 @@ func (p *PostgreSQL) SetMany(ctx context.Context, data []*store.CallbackData) er pq.Array(blockHeights), pq.Array(timestamps), pq.Array(competingTxs), - pq.Array(delayUntils), pq.Array(allowBatches), ) return err } -func (p *PostgreSQL) PopMany(ctx context.Context, limit int) ([]*store.CallbackData, error) { - tx, err := p.db.Begin() - if err != nil { - return nil, err - } - defer func() { - if err != nil { - if rErr := tx.Rollback(); rErr != nil { - err = errors.Join(ErrFailedToRollback, err, rErr) - } - } - }() - - const q = `DELETE FROM callbacker.callbacks - WHERE id IN ( - SELECT id FROM callbacker.callbacks - WHERE postponed_until IS NULL - ORDER BY id - LIMIT $1 - FOR UPDATE - ) - RETURNING - url - ,token - ,tx_id - ,tx_status - ,extra_info - ,merkle_path - ,block_hash - ,block_height - ,competing_txs - ,timestamp - ,postponed_until - ,allow_batch` - - rows, err := tx.QueryContext(ctx, q, limit) - if err != nil { - return nil, err - } - defer rows.Close() - - var records []*store.CallbackData - records, err = scanCallbacks(rows, limit) - if err != nil { - return nil, err - } - - if err = tx.Commit(); err != nil { - return nil, err - } - - return records, nil -} - -func (p *PostgreSQL) PopFailedMany(ctx context.Context, t time.Time, limit int) ([]*store.CallbackData, error) { - tx, err := p.db.Begin() - if err != nil { - return nil, err - } - defer func() { - if err != nil { - if rErr := tx.Rollback(); rErr != nil { - err = errors.Join(err, fmt.Errorf("failed to rollback: %v", rErr)) - } - } - }() - - const q = `DELETE FROM callbacker.callbacks - WHERE id IN ( - SELECT id FROM callbacker.callbacks - WHERE postponed_until IS NOT NULL AND postponed_until<= $1 - ORDER BY id - LIMIT $2 - FOR UPDATE - ) - RETURNING - url - ,token - ,tx_id - ,tx_status - ,extra_info - ,merkle_path - ,block_hash - ,block_height - ,competing_txs - ,timestamp - ,postponed_until - ,allow_batch` - - rows, err := tx.QueryContext(ctx, q, t, limit) - if err != nil { - return nil, err - } - defer rows.Close() - - var records []*store.CallbackData - records, err = scanCallbacks(rows, limit) - if err != nil { - return nil, err - } - - if err = tx.Commit(); err != nil { - return nil, err - } - - return records, nil -} - +// GetAndDelete returns deletes a number of callbacks limited by `limit` ordered by timestamp in ascending order func (p *PostgreSQL) GetAndDelete(ctx context.Context, url string, limit int) ([]*store.CallbackData, error) { const q = `DELETE FROM callbacker.callbacks WHERE id IN ( SELECT id FROM callbacker.callbacks WHERE url = $1 - ORDER BY timestamp DESC + ORDER BY timestamp ASC LIMIT $2 FOR UPDATE ) @@ -258,7 +141,6 @@ func (p *PostgreSQL) GetAndDelete(ctx context.Context, url string, limit int) ([ ,block_height ,competing_txs ,timestamp - ,postponed_until ,allow_batch` rows, err := p.db.QueryContext(ctx, q, url, limit) @@ -276,9 +158,9 @@ func (p *PostgreSQL) GetAndDelete(ctx context.Context, url string, limit int) ([ return records, nil } -func (p *PostgreSQL) DeleteFailedOlderThan(ctx context.Context, t time.Time) error { +func (p *PostgreSQL) DeleteOlderThan(ctx context.Context, t time.Time) error { const q = `DELETE FROM callbacker.callbacks - WHERE postponed_until IS NOT NULL AND timestamp <= $1` + WHERE timestamp <= $1` _, err := p.db.ExecContext(ctx, q, t) return err @@ -303,6 +185,21 @@ func (p *PostgreSQL) SetURLMapping(ctx context.Context, m store.URLMapping) erro return nil } +func (p *PostgreSQL) GetUnmappedURL(ctx context.Context) (url string, err error) { + const q = `SELECT c.url FROM callbacker.callbacks c LEFT JOIN callbacker.url_mapping um ON um.url = c.url WHERE um.url IS NULL LIMIT 1;` + + err = p.db.QueryRowContext(ctx, q).Scan(&url) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + + return "", err + } + + return url, nil +} + func (p *PostgreSQL) DeleteURLMapping(ctx context.Context, instance string) error { const q = `DELETE FROM callbacker.url_mapping WHERE instance=$1` @@ -348,13 +245,12 @@ func scanCallbacks(rows *sql.Rows, expectedNumber int) ([]*store.CallbackData, e r := &store.CallbackData{} var ( - ts time.Time - ei sql.NullString - mp sql.NullString - bh sql.NullString - bHeight sql.NullInt64 - ctxs sql.NullString - postponedUntil sql.NullTime + ts time.Time + ei sql.NullString + mp sql.NullString + bh sql.NullString + bHeight sql.NullInt64 + ctxs sql.NullString ) err := rows.Scan( @@ -368,7 +264,6 @@ func scanCallbacks(rows *sql.Rows, expectedNumber int) ([]*store.CallbackData, e &bHeight, &ctxs, &ts, - &postponedUntil, &r.AllowBatch, ) @@ -393,9 +288,6 @@ func scanCallbacks(rows *sql.Rows, expectedNumber int) ([]*store.CallbackData, e if ctxs.String != "" { r.CompetingTxs = strings.Split(ctxs.String, ",") } - if postponedUntil.Valid { - r.PostponedUntil = ptrTo(postponedUntil.Time.UTC()) - } records = append(records, r) } diff --git a/internal/callbacker/store/postgresql/postgres_test.go b/internal/callbacker/store/postgresql/postgres_test.go index a0ca48eda..dbabe017a 100644 --- a/internal/callbacker/store/postgresql/postgres_test.go +++ b/internal/callbacker/store/postgresql/postgres_test.go @@ -3,11 +3,9 @@ package postgresql import ( "context" "database/sql" - "fmt" "log" "os" "reflect" - "sync" "testing" "time" @@ -127,23 +125,21 @@ func TestPostgresDBt(t *testing.T) { BlockHeight: ptrTo(uint64(4524235)), }, { - URL: "https://test-callback-3/", - TxID: testdata.TX2, - TxStatus: "MINED", - Timestamp: now, - BlockHash: &testdata.Block1, - BlockHeight: ptrTo(uint64(4524235)), - PostponedUntil: ptrTo(now.Add(10 * time.Minute)), + URL: "https://test-callback-3/", + TxID: testdata.TX2, + TxStatus: "MINED", + Timestamp: now, + BlockHash: &testdata.Block1, + BlockHeight: ptrTo(uint64(4524235)), }, { - URL: "https://test-callback-3/", - TxID: testdata.TX3, - TxStatus: "MINED", - Timestamp: now, - BlockHash: &testdata.Block1, - BlockHeight: ptrTo(uint64(4524235)), - PostponedUntil: ptrTo(now.Add(10 * time.Minute)), + URL: "https://test-callback-3/", + TxID: testdata.TX3, + TxStatus: "MINED", + Timestamp: now, + BlockHash: &testdata.Block1, + BlockHeight: ptrTo(uint64(4524235)), }, } @@ -177,7 +173,7 @@ func TestPostgresDBt(t *testing.T) { t.Run("get and delete", func(t *testing.T) { // given defer pruneTables(t, postgresDB.db) - testutils.LoadFixtures(t, postgresDB.db, "fixtures/pop_many") + testutils.LoadFixtures(t, postgresDB.db, "fixtures/get_and_delete") ctx := context.Background() var rowsBefore int @@ -202,136 +198,26 @@ func TestPostgresDBt(t *testing.T) { require.Equal(t, 18, rowsAfter) }) - t.Run("pop many", func(t *testing.T) { - // given - defer pruneTables(t, postgresDB.db) - testutils.LoadFixtures(t, postgresDB.db, "fixtures/pop_many") - - const concurentCalls = 5 - const popLimit = 10 - - // count current records - count := tutils.CountCallbacks(t, postgresDB.db) - require.GreaterOrEqual(t, count, concurentCalls*popLimit) - - ctx := context.Background() - start := make(chan struct{}) - rm := sync.Map{} - wg := sync.WaitGroup{} - - // when - wg.Add(concurentCalls) - for i := range concurentCalls { - go func() { - defer wg.Done() - <-start - - records, err := postgresDB.PopMany(ctx, popLimit) - require.NoError(t, err) - - rm.Store(i, records) - }() - } - - close(start) // signal all goroutines to start - wg.Wait() - - // then - count2 := tutils.CountCallbacks(t, postgresDB.db) - require.Equal(t, count-concurentCalls*popLimit, count2) - - for i := range concurentCalls { - records, ok := rm.Load(i) - require.True(t, ok) - - callbacks, ok := records.([]*store.CallbackData) - require.True(t, ok) - require.Equal(t, popLimit, len(callbacks)) - } - }) - - t.Run("pop failed many", func(t *testing.T) { + t.Run("delete older than", func(t *testing.T) { // given defer pruneTables(t, postgresDB.db) - testutils.LoadFixtures(t, postgresDB.db, "fixtures/pop_failed_many") - - const concurentCalls = 5 - const popLimit = 10 - - // count current records - countAll := tutils.CountCallbacks(t, postgresDB.db) - require.GreaterOrEqual(t, countAll, concurentCalls*popLimit) - countToPop := tutils.CountCallbacksWhere(t, postgresDB.db, fmt.Sprintf("postponed_until <= '%s'", now.Format(time.RFC3339))) - require.Greater(t, countToPop, popLimit) - + testutils.LoadFixtures(t, postgresDB.db, "fixtures/delete_older_than") ctx := context.Background() - start := make(chan struct{}) - rm := sync.Map{} - wg := sync.WaitGroup{} - // when - wg.Add(concurentCalls) - for i := range concurentCalls { - go func() { - defer wg.Done() - <-start - - records, err := postgresDB.PopFailedMany(ctx, now, popLimit) - require.NoError(t, err) - - rm.Store(i, records) - }() - } - - close(start) // signal all goroutines to start - wg.Wait() - - // then - count2 := tutils.CountCallbacks(t, postgresDB.db) - require.Equal(t, countAll-countToPop, count2) - - for i := range concurentCalls { - records, ok := rm.Load(i) - require.True(t, ok) - - callbacks, ok := records.([]*store.CallbackData) - require.True(t, ok) - require.LessOrEqual(t, len(callbacks), popLimit) - } - }) - - t.Run("delete failed older than", func(t *testing.T) { - // given - defer pruneTables(t, postgresDB.db) - testutils.LoadFixtures(t, postgresDB.db, "fixtures/delete_failed_older_than") - - const concurentCalls = 5 - - // count current records - countAll := tutils.CountCallbacks(t, postgresDB.db) - countToDelete := tutils.CountCallbacksWhere(t, postgresDB.db, fmt.Sprintf("timestamp <= '%s' AND postponed_until IS NOT NULL", now.Format(time.RFC3339))) - - ctx := context.Background() - start := make(chan struct{}) - wg := sync.WaitGroup{} + var rowsBefore int + err = postgresDB.db.QueryRowContext(ctx, "SELECT COUNT(1) FROM callbacker.callbacks").Scan(&rowsBefore) + require.NoError(t, err) + require.Equal(t, 56, rowsBefore) - // when - wg.Add(concurentCalls) - for range concurentCalls { - go func() { - defer wg.Done() - <-start - - err := postgresDB.DeleteFailedOlderThan(ctx, now) - require.NoError(t, err) - }() - } + err := postgresDB.DeleteOlderThan(ctx, now) + require.NoError(t, err) - close(start) // signal all goroutines to start - wg.Wait() + var rowsAfter int + err = postgresDB.db.QueryRowContext(ctx, "SELECT COUNT(1) FROM callbacker.callbacks").Scan(&rowsAfter) + require.NoError(t, err) // then - require.Equal(t, countAll-countToDelete, tutils.CountCallbacks(t, postgresDB.db)) + require.Equal(t, 3, rowsAfter) }) t.Run("set URL mapping", func(t *testing.T) { diff --git a/internal/callbacker/store/store.go b/internal/callbacker/store/store.go index b44656ada..fa9c5de59 100644 --- a/internal/callbacker/store/store.go +++ b/internal/callbacker/store/store.go @@ -11,15 +11,6 @@ var ( ErrURLMappingDeleteFailed = errors.New("failed to delete URL mapping entry") ) -type CallbackerStore interface { - Set(ctx context.Context, dto *CallbackData) error - SetMany(ctx context.Context, data []*CallbackData) error - PopMany(ctx context.Context, limit int) ([]*CallbackData, error) - PopFailedMany(ctx context.Context, t time.Time, limit int) ([]*CallbackData, error) // TODO: find better name - DeleteFailedOlderThan(ctx context.Context, t time.Time) error - Close() error -} - type CallbackData struct { URL string Token string @@ -36,14 +27,16 @@ type CallbackData struct { BlockHash *string BlockHeight *uint64 - PostponedUntil *time.Time - AllowBatch bool + AllowBatch bool } type ProcessorStore interface { SetURLMapping(ctx context.Context, m URLMapping) error GetURLMappings(ctx context.Context) (urlInstanceMappings map[string]string, err error) DeleteURLMapping(ctx context.Context, instance string) error + GetUnmappedURL(ctx context.Context) (url string, err error) + GetAndDelete(ctx context.Context, url string, limit int) ([]*CallbackData, error) + DeleteOlderThan(ctx context.Context, t time.Time) error } type URLMapping struct { diff --git a/internal/metamorph/processor.go b/internal/metamorph/processor.go index 909fbb288..ad4e1a92f 100644 --- a/internal/metamorph/processor.go +++ b/internal/metamorph/processor.go @@ -322,7 +322,7 @@ func (p *Processor) updateMined(ctx context.Context, txsBlocks []*blocktx_api.Tr }) if len(data.Callbacks) > 0 { - requests := toSendRequest(data) + requests := toSendRequest(data, p.now()) for _, request := range requests { err = p.mqClient.PublishMarshal(ctx, CallbackTopic, request) if err != nil { @@ -570,7 +570,7 @@ func (p *Processor) statusUpdateWithCallback(ctx context.Context, statusUpdates, } if sendCallback && len(data.Callbacks) > 0 { - requests := toSendRequest(data) + requests := toSendRequest(data, p.now()) for _, request := range requests { err = p.mqClient.PublishMarshal(ctx, CallbackTopic, request) if err != nil { diff --git a/internal/metamorph/processor_helpers.go b/internal/metamorph/processor_helpers.go index f3e68149a..f6309741f 100644 --- a/internal/metamorph/processor_helpers.go +++ b/internal/metamorph/processor_helpers.go @@ -3,8 +3,11 @@ package metamorph import ( "encoding/json" "errors" - "github.com/libsv/go-p2p/chaincfg/chainhash" "log/slog" + "time" + + "github.com/libsv/go-p2p/chaincfg/chainhash" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/bitcoin-sv/arc/internal/cache" "github.com/bitcoin-sv/arc/internal/callbacker/callbacker_api" @@ -184,7 +187,7 @@ func filterUpdates(all []store.UpdateStatus, processed []*store.Data) []store.Up return unprocessed } -func toSendRequest(d *store.Data) []*callbacker_api.SendRequest { +func toSendRequest(d *store.Data, timestamp time.Time) []*callbacker_api.SendRequest { if len(d.Callbacks) == 0 { return nil } @@ -209,6 +212,8 @@ func toSendRequest(d *store.Data) []*callbacker_api.SendRequest { BlockHash: getCallbackBlockHash(d), BlockHeight: d.BlockHeight, + + Timestamp: timestamppb.New(timestamp), } requests = append(requests, request) } diff --git a/pkg/message_queue/nats/client/nats_jetstream/nats_jetstream_client.go b/pkg/message_queue/nats/client/nats_jetstream/nats_jetstream_client.go index e78718315..9796c6dd8 100644 --- a/pkg/message_queue/nats/client/nats_jetstream/nats_jetstream_client.go +++ b/pkg/message_queue/nats/client/nats_jetstream/nats_jetstream_client.go @@ -317,7 +317,7 @@ func (cl *Client) Shutdown() { if err != nil { cl.logger.Error("failed to delete consumer", slog.String("consumer", consumer), slog.String("stream", stream), slog.String("err", err.Error())) } else { - cl.logger.Error("deleted consumer", slog.String("consumer", consumer), slog.String("stream", stream)) + cl.logger.Info("deleted consumer", slog.String("consumer", consumer), slog.String("stream", stream)) } } } diff --git a/test/config/config.yaml b/test/config/config.yaml index 053260985..4b16f6a0e 100644 --- a/test/config/config.yaml +++ b/test/config/config.yaml @@ -159,10 +159,12 @@ callbacker: listenAddr: 0.0.0.0:8021 dialAddr: callbacker:8021 health: - serverDialAddr: localhost:8022 - delay: 0s - pause: 0s + serverDialAddr: localhost:8025 + pause: 10ms batchSendInterval: 5s + pruneOlderThan: 336h + pruneInterval: 24h + expiration: 24h db: mode: postgres postgres: @@ -174,8 +176,3 @@ callbacker: maxIdleConns: 10 maxOpenConns: 80 sslMode: disable - pruneInterval: 24h - pruneOlderThan: 336h - failedCallbackCheckInterval: 1m - delayDuration: 5s - expiration: 24h From 238ca2d53bcb9ad42ecf0b392bb119f8e1b23bc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Thu, 23 Jan 2025 10:59:00 +0100 Subject: [PATCH 13/18] feat(ARCO-291): make lint --- internal/callbacker/processor_test.go | 2 +- internal/callbacker/server_test.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/callbacker/processor_test.go b/internal/callbacker/processor_test.go index 98a46c171..71f5bb7cb 100644 --- a/internal/callbacker/processor_test.go +++ b/internal/callbacker/processor_test.go @@ -325,7 +325,7 @@ func TestStartCallbackStoreCleanup(t *testing.T) { DeleteOlderThanFunc: func(_ context.Context, _ time.Time) error { return tc.deleteFailedOlderThanErr }, - DeleteURLMappingFunc: func(ctx context.Context, instance string) error { + DeleteURLMappingFunc: func(_ context.Context, _ string) error { return nil }, } diff --git a/internal/callbacker/server_test.go b/internal/callbacker/server_test.go index 795f44388..6d27d78c7 100644 --- a/internal/callbacker/server_test.go +++ b/internal/callbacker/server_test.go @@ -91,6 +91,5 @@ func TestSendCallback(t *testing.T) { assert.IsType(t, &emptypb.Empty{}, resp) time.Sleep(100 * time.Millisecond) require.Equal(t, 2, len(mockDispatcher.DispatchCalls()), "Expected two dispatch calls") - }) } From f5d925535e33648337ca5199febd548867064d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Fri, 31 Jan 2025 17:23:24 +0100 Subject: [PATCH 14/18] feat(ARCO-291): Feedback --- internal/callbacker/processor.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/callbacker/processor.go b/internal/callbacker/processor.go index 3cb263c3d..b1d690b95 100644 --- a/internal/callbacker/processor.go +++ b/internal/callbacker/processor.go @@ -203,10 +203,6 @@ func (p *Processor) DispatchPersistedCallbacks() { continue } - if url == "" { - continue - } - err = p.store.SetURLMapping(ctx, store.URLMapping{ URL: url, Instance: p.hostName, From bf57f3a99c0b9f4a9ff7aa1cb8e18f9ec00b9632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Fri, 31 Jan 2025 17:24:42 +0100 Subject: [PATCH 15/18] feat(ARCO-291): Do not allow buffer size larger than 10k --- internal/callbacker/send_manager/send_manager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/callbacker/send_manager/send_manager.go b/internal/callbacker/send_manager/send_manager.go index 6831e7305..36c84f114 100644 --- a/internal/callbacker/send_manager/send_manager.go +++ b/internal/callbacker/send_manager/send_manager.go @@ -73,6 +73,10 @@ func WithNow(nowFunc func() time.Time) func(*SendManager) { func WithBufferSize(size int) func(*SendManager) { return func(m *SendManager) { + if size >= entriesBufferSize { + m.bufferSize = entriesBufferSize + return + } m.bufferSize = size } } From ef2ebeae6945249ab332d04e3c278bbb259c6ef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Wed, 5 Feb 2025 17:09:10 +0100 Subject: [PATCH 16/18] feat(ARCO-291): Use slice instead of linked list --- .../callbacker/send_manager/send_manager.go | 137 ++++++------------ 1 file changed, 46 insertions(+), 91 deletions(-) diff --git a/internal/callbacker/send_manager/send_manager.go b/internal/callbacker/send_manager/send_manager.go index 36c84f114..8a3f1351a 100644 --- a/internal/callbacker/send_manager/send_manager.go +++ b/internal/callbacker/send_manager/send_manager.go @@ -1,10 +1,10 @@ package send_manager import ( - "container/list" "context" "errors" "log/slog" + "sort" "sync" "time" @@ -44,10 +44,11 @@ type SendManager struct { batchSendInterval time.Duration batchSize int - bufferSize int - callbackQueue *list.List + bufferSize int + now func() time.Time - now func() time.Time + mu sync.Mutex + callbackQueue []*callbacker.CallbackEntry } const ( @@ -131,7 +132,7 @@ func New(url string, sender callbacker.SenderI, store SendManagerStore, logger * batchSendInterval: batchSendIntervalDefault, batchSize: batchSizeDefault, - callbackQueue: list.New(), + callbackQueue: make([]*callbacker.CallbackEntry, 0, entriesBufferSize), bufferSize: entriesBufferSize, now: time.Now, @@ -149,47 +150,19 @@ func New(url string, sender callbacker.SenderI, store SendManagerStore, logger * } func (m *SendManager) Enqueue(entry callbacker.CallbackEntry) { - if m.callbackQueue.Len() >= m.bufferSize { + m.mu.Lock() + defer m.mu.Unlock() + + if len(m.callbackQueue) >= m.bufferSize { m.storeToDB(entry) return } - m.callbackQueue.PushBack(entry) -} - -// sortByTimestampAsc sorts the callback queue by timestamp in ascending order -func (m *SendManager) sortByTimestampAsc() error { - current := m.callbackQueue.Front() - if m.callbackQueue.Front() == nil { - return nil - } - for current != nil { - index := current.Next() - for index != nil { - currentTime, ok := current.Value.(callbacker.CallbackEntry) - if !ok { - return ErrElementIsNotCallbackEntry - } - - indexTime, ok := index.Value.(callbacker.CallbackEntry) - if !ok { - return ErrElementIsNotCallbackEntry - } - if currentTime.Data.Timestamp.After(indexTime.Data.Timestamp) { - temp := current.Value - current.Value = index.Value - index.Value = temp - } - index = index.Next() - } - current = current.Next() - } - - return nil + m.callbackQueue = append(m.callbackQueue, &entry) } func (m *SendManager) CallbacksQueued() int { - return m.callbackQueue.Len() + return len(m.callbackQueue) } func (m *SendManager) Start() { @@ -199,26 +172,24 @@ func (m *SendManager) Start() { batchSendTicker := time.NewTicker(m.batchSendInterval) m.entriesWg.Add(1) - var callbackBatch []*list.Element + var callbackBatch []*callbacker.CallbackEntry go func() { var err error defer func() { // read all from callback queue and store in database - data := make([]*store.CallbackData, m.callbackQueue.Len()+len(callbackBatch)) - + data := make([]*store.CallbackData, len(m.callbackQueue)+len(callbackBatch)) + m.mu.Lock() for _, callbackElement := range callbackBatch { - entry, ok := callbackElement.Value.(callbacker.CallbackEntry) - if !ok { - continue - } - m.callbackQueue.PushBack(entry) + m.callbackQueue = append(m.callbackQueue, callbackElement) } - - for i, entry := range m.dequeueAll() { - data[i] = toStoreDto(m.url, entry) + for i, entry := range m.callbackQueue { + data[i] = toStoreDto(m.url, *entry) } + m.callbackQueue = m.callbackQueue[:0] + m.mu.Unlock() + if len(data) > 0 { err = m.store.SetMany(context.Background(), data) if err != nil { @@ -236,7 +207,11 @@ func (m *SendManager) Start() { case <-m.ctx.Done(): return case <-sortQueueTicker.C: - err = m.sortByTimestampAsc() + m.mu.Lock() + sort.Slice(m.callbackQueue, func(i, j int) bool { + return m.callbackQueue[j].Data.Timestamp.After(m.callbackQueue[i].Data.Timestamp) + }) + m.mu.Unlock() if err != nil { m.logger.Error("Failed to sort by timestamp", slog.String("err", err.Error())) } @@ -244,7 +219,7 @@ func (m *SendManager) Start() { case <-backfillQueueTicker.C: m.fillUpQueue() - m.logger.Debug("Callback queue backfilled", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) + m.logger.Debug("Callback queue filled up", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) case <-batchSendTicker.C: if len(callbackBatch) == 0 { continue @@ -259,20 +234,18 @@ func (m *SendManager) Start() { callbackBatch = callbackBatch[:0] m.logger.Debug("Batched callbacks sent on interval", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) case <-queueTicker.C: - front := m.callbackQueue.Front() - if front == nil { + if len(m.callbackQueue) == 0 { continue } - - callbackEntry, ok := front.Value.(callbacker.CallbackEntry) - if !ok { + callbackEntry := m.callbackQueue[0] + if callbackEntry == nil { continue } // If item is expired - dequeue without storing if m.now().Sub(callbackEntry.Data.Timestamp) > m.expiration { m.logger.Warn("Callback expired", slog.Time("timestamp", callbackEntry.Data.Timestamp), slog.String("hash", callbackEntry.Data.TxID), slog.String("status", callbackEntry.Data.TxStatus)) - m.callbackQueue.Remove(front) + m.callbackQueue = m.callbackQueue[1:] continue } @@ -280,9 +253,9 @@ func (m *SendManager) Start() { lastIterationWasBatch = true if len(callbackBatch) < m.batchSize { - callbackBatch = append(callbackBatch, front) + callbackBatch = append(callbackBatch, callbackEntry) queueTicker.Reset(m.queueProcessInterval) - m.callbackQueue.Remove(front) + m.callbackQueue = m.callbackQueue[1:] continue } @@ -313,7 +286,7 @@ func (m *SendManager) Start() { success, retry := m.sender.Send(m.url, callbackEntry.Token, callbackEntry.Data) if !retry || success { - m.callbackQueue.Remove(front) + m.callbackQueue = m.callbackQueue[1:] m.logger.Debug("Single callback sent", slog.Int("callback elements", len(callbackBatch)), slog.Int("queue length", m.CallbacksQueued()), slog.String("url", m.url)) continue } @@ -323,21 +296,22 @@ func (m *SendManager) Start() { }() } -func (m *SendManager) sendElementBatch(callbackElements []*list.Element) error { - var callbackElement *list.Element +func (m *SendManager) sendElementBatch(callbackElements []*callbacker.CallbackEntry) error { callbackBatch := make([]callbacker.CallbackEntry, 0, len(callbackElements)) - for _, element := range callbackElements { - callback, ok := element.Value.(callbacker.CallbackEntry) - if !ok { - continue - } - callbackBatch = append(callbackBatch, callback) + for _, callback := range callbackElements { + callbackBatch = append(callbackBatch, *callback) } success, retry := m.sendBatch(callbackBatch) if !retry || success { - for _, callbackElement = range callbackElements { - m.callbackQueue.Remove(callbackElement) + m.mu.Lock() + for _, callbackElement := range callbackElements { + for i, cb := range m.callbackQueue { + if cb == callbackElement { + m.callbackQueue = append(m.callbackQueue[:i], m.callbackQueue[i+1:]...) + } + } } + m.mu.Unlock() return nil } @@ -357,7 +331,7 @@ func (m *SendManager) sendBatch(batch []callbacker.CallbackEntry) (success, retr // fillUpQueue calculates the capacity left in the queue and fills it up func (m *SendManager) fillUpQueue() { - capacityLeft := m.bufferSize - m.callbackQueue.Len() + capacityLeft := m.bufferSize - len(m.callbackQueue) if capacityLeft == 0 { return } @@ -417,25 +391,6 @@ func toEntry(callbackData *store.CallbackData) callbacker.CallbackEntry { } } -func (m *SendManager) dequeueAll() []callbacker.CallbackEntry { - callbacks := make([]callbacker.CallbackEntry, 0, m.callbackQueue.Len()) - - var next *list.Element - for front := m.callbackQueue.Front(); front != nil; front = next { - next = front.Next() - entry, ok := front.Value.(callbacker.CallbackEntry) - if !ok { - m.callbackQueue.Remove(front) - continue - } - callbacks = append(callbacks, entry) - - m.callbackQueue.Remove(front) - } - - return callbacks -} - // GracefulStop On service termination, any unsent callbacks are persisted in the store, ensuring no loss of data during shutdown. func (m *SendManager) GracefulStop() { if m.cancelAll != nil { From 9e087d734eaa6b459e6183e8342b919c2152fa56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Wed, 5 Feb 2025 17:13:07 +0100 Subject: [PATCH 17/18] feat(ARCO-291): make lint --- internal/callbacker/send_manager/send_manager.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/callbacker/send_manager/send_manager.go b/internal/callbacker/send_manager/send_manager.go index 8a3f1351a..b17075a65 100644 --- a/internal/callbacker/send_manager/send_manager.go +++ b/internal/callbacker/send_manager/send_manager.go @@ -179,10 +179,9 @@ func (m *SendManager) Start() { defer func() { // read all from callback queue and store in database data := make([]*store.CallbackData, len(m.callbackQueue)+len(callbackBatch)) + m.mu.Lock() - for _, callbackElement := range callbackBatch { - m.callbackQueue = append(m.callbackQueue, callbackElement) - } + m.callbackQueue = append(m.callbackQueue, callbackBatch...) for i, entry := range m.callbackQueue { data[i] = toStoreDto(m.url, *entry) } From b3dfacccc374b78ca8b847c432a45abb12f76250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Thu, 6 Feb 2025 14:34:32 +0100 Subject: [PATCH 18/18] feat(ARCO-291): add details to logs --- internal/callbacker/sender.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/internal/callbacker/sender.go b/internal/callbacker/sender.go index 941d5c6bf..d0644f20e 100644 --- a/internal/callbacker/sender.go +++ b/internal/callbacker/sender.go @@ -118,6 +118,7 @@ func (p *CallbackSender) Send(url, token string, dto *Callback) (success, retry slog.String("token", token), slog.String("hash", dto.TxID), slog.String("status", dto.TxStatus), + slog.String("timestamp", dto.Timestamp.String()), slog.String("err", err.Error())) return false, false } @@ -130,7 +131,9 @@ func (p *CallbackSender) Send(url, token string, dto *Callback) (success, retry slog.String("token", token), slog.String("hash", dto.TxID), slog.String("status", dto.TxStatus), - slog.Int("retries", retries)) + slog.String("timestamp", dto.Timestamp.String()), + slog.Int("retries", retries), + ) p.updateSuccessStats(dto.TxStatus) return success, retry @@ -141,7 +144,9 @@ func (p *CallbackSender) Send(url, token string, dto *Callback) (success, retry slog.String("token", token), slog.String("hash", dto.TxID), slog.String("status", dto.TxStatus), - slog.Int("retries", retries)) + slog.String("timestamp", dto.Timestamp.String()), + slog.Int("retries", retries), + ) p.stats.callbackFailedCount.Inc() return success, retry @@ -167,15 +172,17 @@ func (p *CallbackSender) SendBatch(url, token string, dtos []*Callback) (success success, retry, retries = p.sendCallbackWithRetries(url, token, payload) p.stats.callbackBatchCount.Inc() if success { - for _, c := range dtos { - p.logger.Info("Callback sent", + for _, dto := range dtos { + p.logger.Info("Callback sent in batch", slog.String("url", url), slog.String("token", token), - slog.String("hash", c.TxID), - slog.String("status", c.TxStatus), - slog.Int("retries", retries)) + slog.String("hash", dto.TxID), + slog.String("status", dto.TxStatus), + slog.String("timestamp", dto.Timestamp.String()), + slog.Int("retries", retries), + ) - p.updateSuccessStats(c.TxStatus) + p.updateSuccessStats(dto.TxStatus) } return success, retry }