Skip to content

Commit

Permalink
Updates circuit breaker documentation and adds test cases for circuit…
Browse files Browse the repository at this point in the history
… breaker logic
  • Loading branch information
ksysoev committed Apr 13, 2024
1 parent 1e9a8b6 commit f8a73a8
Show file tree
Hide file tree
Showing 2 changed files with 288 additions and 11 deletions.
21 changes: 10 additions & 11 deletions middleware/request/circuit_breaker.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,16 @@ const (
Open
)

// NewCircuitBreakerMiddleware creates a new circuit breaker middleware with the specified threshold and period.
// The circuit breaker middleware wraps a given `wasabi.RequestHandler` and provides circuit breaking functionality.
// The circuit breaker tracks the number of consecutive errors and opens the circuit when the error count exceeds the threshold.
// During the open state, all requests are rejected with an `ErrCircuitBreakerOpen` error.
// After a specified period of time, the circuit breaker transitions to the semi-open state, allowing a single request to be processed.
// If the request succeeds, the circuit breaker resets the error count and transitions back to the closed state.
// If the request fails, the circuit breaker remains in the open state.
// The circuit breaker uses synchronization primitives to ensure thread safety.
// The `treshold` parameter specifies the maximum number of consecutive errors allowed before opening the circuit.
// The `period` parameter specifies the duration of time after which the circuit breaker transitions to the semi-open state.
// The returned function is a middleware that can be used with the `wasabi` framework.
// NewCircuitBreakerMiddleware creates a new circuit breaker middleware with the specified parameters.
// It returns a function that wraps the provided `wasabi.RequestHandler` and implements the circuit breaker logic.
// The circuit breaker monitors the number of errors and successful requests within a given time period.
// If the number of errors exceeds the threshold, the circuit breaker switches to the "Open" state and rejects subsequent requests.
// After a specified number of successful requests, the circuit breaker switches back to the "Closed" state.
// The circuit breaker uses a lock to ensure thread safety.
// The `treshold` parameter specifies the maximum number of errors allowed within the time period.
// The `period` parameter specifies the duration of the time period.
// The `recoverAfter` parameter specifies the number of successful requests required to recover from the "Open" state.
// The returned function can be used as middleware in a Wasabi server.
func NewCircuitBreakerMiddleware(treshold uint, period time.Duration, recoverAfter uint) func(next wasabi.RequestHandler) wasabi.RequestHandler {
var errorCounter, successCounter uint

Expand Down
278 changes: 278 additions & 0 deletions middleware/request/circuit_breaker_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,278 @@
package request

import (
"fmt"
"testing"
"time"

"github.com/ksysoev/wasabi"
"github.com/ksysoev/wasabi/dispatch"
"github.com/ksysoev/wasabi/mocks"
)

func TestNewCircuitBreakerMiddleware_ClosedState(t *testing.T) {
treshold := uint(3)
period := time.Second
recoverAfter := uint(1)

// Create a mock request handler
mockHandler := dispatch.RequestHandlerFunc(func(conn wasabi.Connection, req wasabi.Request) error { return nil })
mockRequest := mocks.NewMockRequest(t)
mockConn := mocks.NewMockConnection(t)

// Create the circuit breaker middleware
middleware := NewCircuitBreakerMiddleware(treshold, period, recoverAfter)(mockHandler)

// Test the Closed state
for i := uint(0); i < treshold+1; i++ {
err := middleware.Handle(mockConn, mockRequest)
if err != nil {
t.Errorf("Expected no error, but got %v", err)
}
}
}

func TestNewCircuitBreakerMiddleware_OpenState(t *testing.T) {
treshold := uint(1)
period := time.Second
recoverAfter := uint(1)

testError := fmt.Errorf("test error")

// Create a mock request handler
mockHandler := dispatch.RequestHandlerFunc(func(conn wasabi.Connection, req wasabi.Request) error {
time.Sleep(5 * time.Millisecond)
return testError
})

mockRequest := mocks.NewMockRequest(t)
mockConn := mocks.NewMockConnection(t)

// Create the circuit breaker middleware
middleware := NewCircuitBreakerMiddleware(treshold, period, recoverAfter)(mockHandler)

// Bring the circuit breaker to the Open state
err := middleware.Handle(mockConn, mockRequest)
if err != testError {
t.Errorf("Expected error %v, but got %v", testError, err)
}

// Test the Open state
results := make(chan error)

for i := 0; i < 2; i++ {
go func() {
results <- middleware.Handle(mockConn, mockRequest)
}()
}

OpenErrorCount := 0
TestErrorCount := 0

for i := 0; i < 2; i++ {
select {
case err := <-results:
if err != ErrCircuitBreakerOpen && err != testError {
t.Errorf("Expected error %v, but got %v", ErrCircuitBreakerOpen, err)
continue
}

if err == ErrCircuitBreakerOpen {
OpenErrorCount++
} else if err == testError {
TestErrorCount++
}

case <-time.After(100 * time.Millisecond):
t.Fatal("Expected error, but got none")
}
}

if OpenErrorCount != 1 {
t.Errorf("Expected 1 ErrCircuitBreakerOpen error, but got %d", OpenErrorCount)
}

if TestErrorCount != 1 {
t.Errorf("Expected 1 test error, but got %d", TestErrorCount)
}
}

func TestNewCircuitBreakerMiddleware_SemiOpenState(t *testing.T) {
treshold := uint(1)
period := time.Second
recoverAfter := uint(1)

testError := fmt.Errorf("test error")

errorToReturn := testError

// Create a mock request handler
mockHandler := dispatch.RequestHandlerFunc(func(conn wasabi.Connection, req wasabi.Request) error {
time.Sleep(5 * time.Millisecond)
return errorToReturn
})

mockRequest := mocks.NewMockRequest(t)
mockConn := mocks.NewMockConnection(t)

// Create the circuit breaker middleware
middleware := NewCircuitBreakerMiddleware(treshold, period, recoverAfter)(mockHandler)

// Bring the circuit breaker to the Open state
err := middleware.Handle(mockConn, mockRequest)
if err != testError {
t.Errorf("Expected error %v, but got %v", testError, err)
}

// Test the Open state
errorToReturn = nil
OpenErrorCount := 0
SuccessCount := 0
results := make(chan error)

for i := 0; i < 2; i++ {
go func() {
results <- middleware.Handle(mockConn, mockRequest)
}()
}

for i := 0; i < 2; i++ {
select {
case err := <-results:
if err != ErrCircuitBreakerOpen && err != nil {
t.Errorf("Expected error %v, but got %v", ErrCircuitBreakerOpen, err)
continue
}

if err == ErrCircuitBreakerOpen {
OpenErrorCount++
} else if err == nil {
SuccessCount++
}

case <-time.After(100 * time.Millisecond):
t.Fatal("Expected error, but got none")
}
}

if OpenErrorCount != 1 {
t.Errorf("Expected 1 ErrCircuitBreakerOpen error, but got %d", OpenErrorCount)
}

if SuccessCount != 1 {
t.Errorf("Expected 1 test error, but got %d", SuccessCount)
}

// Confirm that the circuit breaker is now in the Closed state

for i := 0; i < 2; i++ {
go func() {
results <- middleware.Handle(mockConn, mockRequest)
}()
}

OpenErrorCount = 0
SuccessCount = 0

for i := 0; i < 2; i++ {
select {
case err := <-results:
if err != ErrCircuitBreakerOpen && err != nil {
t.Errorf("Expected error %v, but got %v", ErrCircuitBreakerOpen, err)
continue
}

if err == ErrCircuitBreakerOpen {
OpenErrorCount++
} else if err == nil {
SuccessCount++
}

case <-time.After(100 * time.Millisecond):
t.Fatal("Expected error, but got none")
}
}

if OpenErrorCount != 0 {
t.Errorf("Expected 0 ErrCircuitBreakerOpen error, but got %d", OpenErrorCount)
}

if SuccessCount != 2 {
t.Errorf("Expected 2 test error, but got %d", SuccessCount)
}
}

func TestNewCircuitBreakerMiddleware_ResetMeasureInterval(t *testing.T) {
treshold := uint(2)
period := 20 * time.Millisecond
recoverAfter := uint(1)

testError := fmt.Errorf("test error")

errorToReturn := testError

// Create a mock request handler
mockHandler := dispatch.RequestHandlerFunc(func(conn wasabi.Connection, req wasabi.Request) error {
time.Sleep(5 * time.Millisecond)
return errorToReturn
})

mockRequest := mocks.NewMockRequest(t)
mockConn := mocks.NewMockConnection(t)

// Create the circuit breaker middleware
middleware := NewCircuitBreakerMiddleware(treshold, period, recoverAfter)(mockHandler)

// Bring the circuit breaker to the Open state

if err := middleware.Handle(mockConn, mockRequest); err != testError {
t.Errorf("Expected error %v, but got %v", testError, err)
}

time.Sleep(period)

if err := middleware.Handle(mockConn, mockRequest); err != testError {
t.Errorf("Expected error %v, but got %v", testError, err)
}

// Confirm that the circuit breaker is now in the Closed state

errorToReturn = nil
results := make(chan error)

for i := 0; i < 2; i++ {
go func() {
results <- middleware.Handle(mockConn, mockRequest)
}()
}

OpenErrorCount := 0
SuccessCount := 0

for i := 0; i < 2; i++ {
select {
case err := <-results:
if err != ErrCircuitBreakerOpen && err != nil {
t.Errorf("Expected error %v, but got %v", ErrCircuitBreakerOpen, err)
continue
}

if err == ErrCircuitBreakerOpen {
OpenErrorCount++
} else if err == nil {
SuccessCount++
}

case <-time.After(100 * time.Millisecond):
t.Fatal("Expected error, but got none")
}
}

if OpenErrorCount != 0 {
t.Errorf("Expected 0 ErrCircuitBreakerOpen error, but got %d", OpenErrorCount)
}

if SuccessCount != 2 {
t.Errorf("Expected 2 test error, but got %d", SuccessCount)
}
}

0 comments on commit f8a73a8

Please sign in to comment.