From f8a73a8d1a438f6132697ead58af54cf6fa00fd1 Mon Sep 17 00:00:00 2001 From: Kirill Sysoev Date: Sat, 13 Apr 2024 11:36:00 +0800 Subject: [PATCH] Updates circuit breaker documentation and adds test cases for circuit breaker logic --- middleware/request/circuit_breaker.go | 21 +- middleware/request/circuit_breaker_test.go | 278 +++++++++++++++++++++ 2 files changed, 288 insertions(+), 11 deletions(-) create mode 100644 middleware/request/circuit_breaker_test.go diff --git a/middleware/request/circuit_breaker.go b/middleware/request/circuit_breaker.go index b814ac0..2514ccc 100644 --- a/middleware/request/circuit_breaker.go +++ b/middleware/request/circuit_breaker.go @@ -20,17 +20,16 @@ const ( Open ) -// NewCircuitBreakerMiddleware creates a new circuit breaker middleware with the specified threshold and period. -// The circuit breaker middleware wraps a given `wasabi.RequestHandler` and provides circuit breaking functionality. -// The circuit breaker tracks the number of consecutive errors and opens the circuit when the error count exceeds the threshold. -// During the open state, all requests are rejected with an `ErrCircuitBreakerOpen` error. -// After a specified period of time, the circuit breaker transitions to the semi-open state, allowing a single request to be processed. -// If the request succeeds, the circuit breaker resets the error count and transitions back to the closed state. -// If the request fails, the circuit breaker remains in the open state. -// The circuit breaker uses synchronization primitives to ensure thread safety. -// The `treshold` parameter specifies the maximum number of consecutive errors allowed before opening the circuit. -// The `period` parameter specifies the duration of time after which the circuit breaker transitions to the semi-open state. -// The returned function is a middleware that can be used with the `wasabi` framework. +// NewCircuitBreakerMiddleware creates a new circuit breaker middleware with the specified parameters. +// It returns a function that wraps the provided `wasabi.RequestHandler` and implements the circuit breaker logic. +// The circuit breaker monitors the number of errors and successful requests within a given time period. +// If the number of errors exceeds the threshold, the circuit breaker switches to the "Open" state and rejects subsequent requests. +// After a specified number of successful requests, the circuit breaker switches back to the "Closed" state. +// The circuit breaker uses a lock to ensure thread safety. +// The `treshold` parameter specifies the maximum number of errors allowed within the time period. +// The `period` parameter specifies the duration of the time period. +// The `recoverAfter` parameter specifies the number of successful requests required to recover from the "Open" state. +// The returned function can be used as middleware in a Wasabi server. func NewCircuitBreakerMiddleware(treshold uint, period time.Duration, recoverAfter uint) func(next wasabi.RequestHandler) wasabi.RequestHandler { var errorCounter, successCounter uint diff --git a/middleware/request/circuit_breaker_test.go b/middleware/request/circuit_breaker_test.go new file mode 100644 index 0000000..79fcb33 --- /dev/null +++ b/middleware/request/circuit_breaker_test.go @@ -0,0 +1,278 @@ +package request + +import ( + "fmt" + "testing" + "time" + + "github.com/ksysoev/wasabi" + "github.com/ksysoev/wasabi/dispatch" + "github.com/ksysoev/wasabi/mocks" +) + +func TestNewCircuitBreakerMiddleware_ClosedState(t *testing.T) { + treshold := uint(3) + period := time.Second + recoverAfter := uint(1) + + // Create a mock request handler + mockHandler := dispatch.RequestHandlerFunc(func(conn wasabi.Connection, req wasabi.Request) error { return nil }) + mockRequest := mocks.NewMockRequest(t) + mockConn := mocks.NewMockConnection(t) + + // Create the circuit breaker middleware + middleware := NewCircuitBreakerMiddleware(treshold, period, recoverAfter)(mockHandler) + + // Test the Closed state + for i := uint(0); i < treshold+1; i++ { + err := middleware.Handle(mockConn, mockRequest) + if err != nil { + t.Errorf("Expected no error, but got %v", err) + } + } +} + +func TestNewCircuitBreakerMiddleware_OpenState(t *testing.T) { + treshold := uint(1) + period := time.Second + recoverAfter := uint(1) + + testError := fmt.Errorf("test error") + + // Create a mock request handler + mockHandler := dispatch.RequestHandlerFunc(func(conn wasabi.Connection, req wasabi.Request) error { + time.Sleep(5 * time.Millisecond) + return testError + }) + + mockRequest := mocks.NewMockRequest(t) + mockConn := mocks.NewMockConnection(t) + + // Create the circuit breaker middleware + middleware := NewCircuitBreakerMiddleware(treshold, period, recoverAfter)(mockHandler) + + // Bring the circuit breaker to the Open state + err := middleware.Handle(mockConn, mockRequest) + if err != testError { + t.Errorf("Expected error %v, but got %v", testError, err) + } + + // Test the Open state + results := make(chan error) + + for i := 0; i < 2; i++ { + go func() { + results <- middleware.Handle(mockConn, mockRequest) + }() + } + + OpenErrorCount := 0 + TestErrorCount := 0 + + for i := 0; i < 2; i++ { + select { + case err := <-results: + if err != ErrCircuitBreakerOpen && err != testError { + t.Errorf("Expected error %v, but got %v", ErrCircuitBreakerOpen, err) + continue + } + + if err == ErrCircuitBreakerOpen { + OpenErrorCount++ + } else if err == testError { + TestErrorCount++ + } + + case <-time.After(100 * time.Millisecond): + t.Fatal("Expected error, but got none") + } + } + + if OpenErrorCount != 1 { + t.Errorf("Expected 1 ErrCircuitBreakerOpen error, but got %d", OpenErrorCount) + } + + if TestErrorCount != 1 { + t.Errorf("Expected 1 test error, but got %d", TestErrorCount) + } +} + +func TestNewCircuitBreakerMiddleware_SemiOpenState(t *testing.T) { + treshold := uint(1) + period := time.Second + recoverAfter := uint(1) + + testError := fmt.Errorf("test error") + + errorToReturn := testError + + // Create a mock request handler + mockHandler := dispatch.RequestHandlerFunc(func(conn wasabi.Connection, req wasabi.Request) error { + time.Sleep(5 * time.Millisecond) + return errorToReturn + }) + + mockRequest := mocks.NewMockRequest(t) + mockConn := mocks.NewMockConnection(t) + + // Create the circuit breaker middleware + middleware := NewCircuitBreakerMiddleware(treshold, period, recoverAfter)(mockHandler) + + // Bring the circuit breaker to the Open state + err := middleware.Handle(mockConn, mockRequest) + if err != testError { + t.Errorf("Expected error %v, but got %v", testError, err) + } + + // Test the Open state + errorToReturn = nil + OpenErrorCount := 0 + SuccessCount := 0 + results := make(chan error) + + for i := 0; i < 2; i++ { + go func() { + results <- middleware.Handle(mockConn, mockRequest) + }() + } + + for i := 0; i < 2; i++ { + select { + case err := <-results: + if err != ErrCircuitBreakerOpen && err != nil { + t.Errorf("Expected error %v, but got %v", ErrCircuitBreakerOpen, err) + continue + } + + if err == ErrCircuitBreakerOpen { + OpenErrorCount++ + } else if err == nil { + SuccessCount++ + } + + case <-time.After(100 * time.Millisecond): + t.Fatal("Expected error, but got none") + } + } + + if OpenErrorCount != 1 { + t.Errorf("Expected 1 ErrCircuitBreakerOpen error, but got %d", OpenErrorCount) + } + + if SuccessCount != 1 { + t.Errorf("Expected 1 test error, but got %d", SuccessCount) + } + + // Confirm that the circuit breaker is now in the Closed state + + for i := 0; i < 2; i++ { + go func() { + results <- middleware.Handle(mockConn, mockRequest) + }() + } + + OpenErrorCount = 0 + SuccessCount = 0 + + for i := 0; i < 2; i++ { + select { + case err := <-results: + if err != ErrCircuitBreakerOpen && err != nil { + t.Errorf("Expected error %v, but got %v", ErrCircuitBreakerOpen, err) + continue + } + + if err == ErrCircuitBreakerOpen { + OpenErrorCount++ + } else if err == nil { + SuccessCount++ + } + + case <-time.After(100 * time.Millisecond): + t.Fatal("Expected error, but got none") + } + } + + if OpenErrorCount != 0 { + t.Errorf("Expected 0 ErrCircuitBreakerOpen error, but got %d", OpenErrorCount) + } + + if SuccessCount != 2 { + t.Errorf("Expected 2 test error, but got %d", SuccessCount) + } +} + +func TestNewCircuitBreakerMiddleware_ResetMeasureInterval(t *testing.T) { + treshold := uint(2) + period := 20 * time.Millisecond + recoverAfter := uint(1) + + testError := fmt.Errorf("test error") + + errorToReturn := testError + + // Create a mock request handler + mockHandler := dispatch.RequestHandlerFunc(func(conn wasabi.Connection, req wasabi.Request) error { + time.Sleep(5 * time.Millisecond) + return errorToReturn + }) + + mockRequest := mocks.NewMockRequest(t) + mockConn := mocks.NewMockConnection(t) + + // Create the circuit breaker middleware + middleware := NewCircuitBreakerMiddleware(treshold, period, recoverAfter)(mockHandler) + + // Bring the circuit breaker to the Open state + + if err := middleware.Handle(mockConn, mockRequest); err != testError { + t.Errorf("Expected error %v, but got %v", testError, err) + } + + time.Sleep(period) + + if err := middleware.Handle(mockConn, mockRequest); err != testError { + t.Errorf("Expected error %v, but got %v", testError, err) + } + + // Confirm that the circuit breaker is now in the Closed state + + errorToReturn = nil + results := make(chan error) + + for i := 0; i < 2; i++ { + go func() { + results <- middleware.Handle(mockConn, mockRequest) + }() + } + + OpenErrorCount := 0 + SuccessCount := 0 + + for i := 0; i < 2; i++ { + select { + case err := <-results: + if err != ErrCircuitBreakerOpen && err != nil { + t.Errorf("Expected error %v, but got %v", ErrCircuitBreakerOpen, err) + continue + } + + if err == ErrCircuitBreakerOpen { + OpenErrorCount++ + } else if err == nil { + SuccessCount++ + } + + case <-time.After(100 * time.Millisecond): + t.Fatal("Expected error, but got none") + } + } + + if OpenErrorCount != 0 { + t.Errorf("Expected 0 ErrCircuitBreakerOpen error, but got %d", OpenErrorCount) + } + + if SuccessCount != 2 { + t.Errorf("Expected 2 test error, but got %d", SuccessCount) + } +}