Skip to content

Commit f8351f6

Browse files
YoshiyukiMineocall-stackKalpit Pant
authored
Title: Implement Distributed Circuit Breaker (#70) (#73)
* Title: Implement Distributed Circuit Breaker (#70) * feature/redis-circuit-breaker * feature/redis-circuit-breaker * Refactor * save state * Saving half-open state also * Saving half-open state also * Added test case * Saving state transition * Pass context * Moved redis circuit breaker to v2 * Revert go.mod and go.sum * Acked review comments * Refactor * Refactor --------- Co-authored-by: Kalpit Pant <kalpit@setu.co> * Rename * Rename * Refactor * Rename * Rename * Use generic cache store (#74) Co-authored-by: Kalpit Pant <kalpit@setu.co> * Update distributed_gobreaker.go * Update distributed_gobreaker.go * Update distributed_gobreaker.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker_test.go * Update distributed_gobreaker_test.go --------- Co-authored-by: kp <kalpitpant32@gmail.com> Co-authored-by: Kalpit Pant <kalpit@setu.co>
1 parent d78b227 commit f8351f6

File tree

4 files changed

+631
-0
lines changed

4 files changed

+631
-0
lines changed

v2/distributed_gobreaker.go

+265
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
package gobreaker
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"errors"
7+
"time"
8+
)
9+
10+
var (
11+
// ErrNoSharedStore is returned when there is no shared store.
12+
ErrNoSharedStore = errors.New("no shared store")
13+
// ErrNoSharedState is returned when there is no shared state.
14+
ErrNoSharedState = errors.New("no shared state")
15+
)
16+
17+
// SharedState represents the shared state of DistributedCircuitBreaker.
18+
type SharedState struct {
19+
State State `json:"state"`
20+
Generation uint64 `json:"generation"`
21+
Counts Counts `json:"counts"`
22+
Expiry time.Time `json:"expiry"`
23+
}
24+
25+
// SharedDataStore stores the shared state of DistributedCircuitBreaker.
26+
type SharedDataStore interface {
27+
GetData(ctx context.Context, name string) ([]byte, error)
28+
SetData(ctx context.Context, name string, data []byte) error
29+
}
30+
31+
// DistributedCircuitBreaker extends CircuitBreaker with SharedDataStore.
32+
type DistributedCircuitBreaker[T any] struct {
33+
*CircuitBreaker[T]
34+
store SharedDataStore
35+
}
36+
37+
// NewDistributedCircuitBreaker returns a new DistributedCircuitBreaker.
38+
func NewDistributedCircuitBreaker[T any](ctx context.Context, store SharedDataStore, settings Settings) (*DistributedCircuitBreaker[T], error) {
39+
if store == nil {
40+
return nil, ErrNoSharedStore
41+
}
42+
43+
dcb := &DistributedCircuitBreaker[T]{
44+
CircuitBreaker: NewCircuitBreaker[T](settings),
45+
store: store,
46+
}
47+
state := SharedState{
48+
State: dcb.state,
49+
Generation: dcb.generation,
50+
Counts: dcb.counts,
51+
Expiry: dcb.expiry,
52+
}
53+
err := dcb.setSharedState(ctx, state)
54+
return dcb, err
55+
}
56+
57+
func (dcb *DistributedCircuitBreaker[T]) sharedStateKey() string {
58+
return "gobreaker:" + dcb.name
59+
}
60+
61+
func (dcb *DistributedCircuitBreaker[T]) getSharedState(ctx context.Context) (SharedState, error) {
62+
var state SharedState
63+
if dcb.store == nil {
64+
return state, ErrNoSharedStore
65+
}
66+
67+
data, err := dcb.store.GetData(ctx, dcb.sharedStateKey())
68+
if len(data) == 0 {
69+
return state, ErrNoSharedState
70+
} else if err != nil {
71+
return state, err
72+
}
73+
74+
err = json.Unmarshal(data, &state)
75+
return state, err
76+
}
77+
78+
func (dcb *DistributedCircuitBreaker[T]) setSharedState(ctx context.Context, state SharedState) error {
79+
if dcb.store == nil {
80+
return ErrNoSharedStore
81+
}
82+
83+
data, err := json.Marshal(state)
84+
if err != nil {
85+
return err
86+
}
87+
88+
return dcb.store.SetData(ctx, dcb.sharedStateKey(), data)
89+
}
90+
91+
// State returns the State of DistributedCircuitBreaker.
92+
func (dcb *DistributedCircuitBreaker[T]) State(ctx context.Context) (State, error) {
93+
state, err := dcb.getSharedState(ctx)
94+
if err != nil {
95+
return state.State, err
96+
}
97+
98+
now := time.Now()
99+
currentState, _ := dcb.currentState(state, now)
100+
101+
// update the state if it has changed
102+
if currentState != state.State {
103+
state.State = currentState
104+
if err := dcb.setSharedState(ctx, state); err != nil {
105+
return state.State, err
106+
}
107+
}
108+
109+
return state.State, nil
110+
}
111+
112+
// Execute runs the given request if the DistributedCircuitBreaker accepts it.
113+
func (dcb *DistributedCircuitBreaker[T]) Execute(ctx context.Context, req func() (T, error)) (t T, err error) {
114+
generation, err := dcb.beforeRequest(ctx)
115+
if err != nil {
116+
var defaultValue T
117+
return defaultValue, err
118+
}
119+
120+
defer func() {
121+
e := recover()
122+
if e != nil {
123+
ae := dcb.afterRequest(ctx, generation, false)
124+
if err == nil {
125+
err = ae
126+
}
127+
panic(e)
128+
}
129+
}()
130+
131+
result, err := req()
132+
ae := dcb.afterRequest(ctx, generation, dcb.isSuccessful(err))
133+
if err == nil {
134+
err = ae
135+
}
136+
return result, err
137+
}
138+
139+
func (dcb *DistributedCircuitBreaker[T]) beforeRequest(ctx context.Context) (uint64, error) {
140+
state, err := dcb.getSharedState(ctx)
141+
if err != nil {
142+
return 0, err
143+
}
144+
145+
now := time.Now()
146+
currentState, generation := dcb.currentState(state, now)
147+
148+
if currentState != state.State {
149+
dcb.setState(&state, currentState, now)
150+
err = dcb.setSharedState(ctx, state)
151+
if err != nil {
152+
return 0, err
153+
}
154+
}
155+
156+
if currentState == StateOpen {
157+
return generation, ErrOpenState
158+
} else if currentState == StateHalfOpen && state.Counts.Requests >= dcb.maxRequests {
159+
return generation, ErrTooManyRequests
160+
}
161+
162+
state.Counts.onRequest()
163+
err = dcb.setSharedState(ctx, state)
164+
if err != nil {
165+
return 0, err
166+
}
167+
168+
return generation, nil
169+
}
170+
171+
func (dcb *DistributedCircuitBreaker[T]) afterRequest(ctx context.Context, before uint64, success bool) error {
172+
state, err := dcb.getSharedState(ctx)
173+
if err != nil {
174+
return err
175+
}
176+
177+
now := time.Now()
178+
currentState, generation := dcb.currentState(state, now)
179+
if generation != before {
180+
return nil
181+
}
182+
183+
if success {
184+
dcb.onSuccess(&state, currentState, now)
185+
} else {
186+
dcb.onFailure(&state, currentState, now)
187+
}
188+
return dcb.setSharedState(ctx, state)
189+
}
190+
191+
func (dcb *DistributedCircuitBreaker[T]) onSuccess(state *SharedState, currentState State, now time.Time) {
192+
if state.State == StateOpen {
193+
state.State = currentState
194+
}
195+
196+
switch currentState {
197+
case StateClosed:
198+
state.Counts.onSuccess()
199+
case StateHalfOpen:
200+
state.Counts.onSuccess()
201+
if state.Counts.ConsecutiveSuccesses >= dcb.maxRequests {
202+
dcb.setState(state, StateClosed, now)
203+
}
204+
}
205+
}
206+
207+
func (dcb *DistributedCircuitBreaker[T]) onFailure(state *SharedState, currentState State, now time.Time) {
208+
switch currentState {
209+
case StateClosed:
210+
state.Counts.onFailure()
211+
if dcb.readyToTrip(state.Counts) {
212+
dcb.setState(state, StateOpen, now)
213+
}
214+
case StateHalfOpen:
215+
dcb.setState(state, StateOpen, now)
216+
}
217+
}
218+
219+
func (dcb *DistributedCircuitBreaker[T]) currentState(state SharedState, now time.Time) (State, uint64) {
220+
switch state.State {
221+
case StateClosed:
222+
if !state.Expiry.IsZero() && state.Expiry.Before(now) {
223+
dcb.toNewGeneration(&state, now)
224+
}
225+
case StateOpen:
226+
if state.Expiry.Before(now) {
227+
dcb.setState(&state, StateHalfOpen, now)
228+
}
229+
}
230+
return state.State, state.Generation
231+
}
232+
233+
func (dcb *DistributedCircuitBreaker[T]) setState(state *SharedState, newState State, now time.Time) {
234+
if state.State == newState {
235+
return
236+
}
237+
238+
prev := state.State
239+
state.State = newState
240+
241+
dcb.toNewGeneration(state, now)
242+
243+
if dcb.onStateChange != nil {
244+
dcb.onStateChange(dcb.name, prev, newState)
245+
}
246+
}
247+
248+
func (dcb *DistributedCircuitBreaker[T]) toNewGeneration(state *SharedState, now time.Time) {
249+
state.Generation++
250+
state.Counts.clear()
251+
252+
var zero time.Time
253+
switch state.State {
254+
case StateClosed:
255+
if dcb.interval == 0 {
256+
state.Expiry = zero
257+
} else {
258+
state.Expiry = now.Add(dcb.interval)
259+
}
260+
case StateOpen:
261+
state.Expiry = now.Add(dcb.timeout)
262+
default: // StateHalfOpen
263+
state.Expiry = zero
264+
}
265+
}

0 commit comments

Comments
 (0)