Merge pull request #87 from banzaicloud/native-reconciler-retryonerror

support retry on resource reconcile error in native reconciler
cisco-open · Nov 19, 2021 · 04dbff6 · 04dbff6
2 parents c8de0b1 + 5859139
commit 04dbff6
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 3 deletions.
diff --git a/pkg/reconciler/native.go b/pkg/reconciler/native.go
@@ -31,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
@@ -155,6 +156,8 @@ type NativeReconciler struct {
 	setControllerRef       bool
 	reconciledObjectStates map[reconciledObjectState][]runtime.Object
 	waitBackoff            *wait.Backoff
+	retryBackoff           wait.Backoff
+	retriableErrorFunc     func(error) bool
 	objectModifiers        []resources.ObjectModifierWithParentFunc
 }
 
@@ -190,6 +193,18 @@ func NativeReconcilerWithModifier(modifierFunc resources.ObjectModifierWithParen
 	}
 }
 
+func NativeReconcilerWithRetryBackoff(backoff wait.Backoff) NativeReconcilerOpt {
+	return func(r *NativeReconciler) {
+		r.retryBackoff = backoff
+	}
+}
+
+func NativeReconcilerWithRetriableErrorFunc(retriableErrorFunc func(error) bool) NativeReconcilerOpt {
+	return func(r *NativeReconciler) {
+		r.retriableErrorFunc = retriableErrorFunc
+	}
+}
+
 func NewNativeReconcilerWithDefaults(
 	component string,
 	client client.Client,
@@ -236,6 +251,10 @@ func NewNativeReconciler(
 		reconciledComponent:       reconciledComponent,
 		configTranslate:           resourceTranslate,
 		componentName:             componentName,
+
+		// do not retry on errors by default
+		retriableErrorFunc: func(error) bool { return false },
+		retryBackoff:       retry.DefaultRetry,
 	}
 
 	reconciler.initReconciledObjectStates()
@@ -319,7 +338,11 @@ LOOP:
 			}
 
 			var result *reconcile.Result
-			result, err = rec.ReconcileResource(o, state)
+			err = retry.OnError(rec.retryBackoff, rec.retriableErrorFunc, func() error {
+				var err error
+				result, err = rec.ReconcileResource(o, state)
+				return err
+			})
 			if err == nil {
 				resourceID, err := rec.generateResourceID(o)
 				if err != nil {

diff --git a/pkg/wait/wait.go b/pkg/wait/wait.go
@@ -29,7 +29,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 )
 
-type Backoff wait.Backoff
+type Backoff = wait.Backoff
 
 type ResourceConditionChecks struct {
 	client  client.Client
@@ -69,7 +69,6 @@ func (c *ResourceConditionChecks) WaitForCustomConditionChecks(id string, checkF
 
 		return true, nil
 	})
-
 	if err != nil {
 		return err
 	}