Merge pull request #11 from 3scale-ops/feat/dynamic-watches

feat/dynamic-watches

README.md

@@ -4,6 +4,7 @@ Basereconciler is an attempt to create a reconciler that can be imported and use
 At the moment basereconciler can perform the following tasks:
 
 * **Get the custom resource and perform some common tasks on it**:
+  * Management of initialization logic: custom initialization functions can be passed to perform initialization tasks on the custom resource. Initialization can be done persisting changes in the API server (use reconciler.WithInitializationFunc) or without persisting them (reconciler.WithInMemoryInitializationFunc).
   * Management of resource finalizer: some custom resources required more complex finalization logic. For this to happen a finalizer must be in place. Basereconciler can keep this finalizer in place and remove it when necessary during resource finalization.
   * Management of finalization logic: it checks if the resource is being finalized and executed the finalization logic passed to it if that is the case. When all finalization logic is completed it removes the finalizer on the custom resource.
 * **Reconcile resources owned by the custom resource**: basereconciler can keep the owned resources of a custom resource in it's desired state. It works for any resource type, and only requires that the user configures how each specific resource type has to be configured. The resource reconciler only works in "update mode" right now, so any operation to transition a given resource from its live state to its desired state will be an Update. We might add a "patch mode" in the future.
@@ -85,6 +86,11 @@ func (r *GuestbookReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	// a finalization function that will casuse a log line to show when the resource is being deleted.
 	guestbook := &webappv1.Guestbook{}
 	result := r.ManageResourceLifecycle(ctx, req, guestbook,
+		reconciler.WithInitializationFunc(
+			func(context.Context, client.Client) error {
+				logger.Info("initializing resource")
+				return nil
+			}),
 		reconciler.WithFinalizer("guestbook-finalizer"),
 		reconciler.WithFinalizationFunc(
 			func(context.Context, client.Client) error {
@@ -140,13 +146,12 @@ func (r *GuestbookReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 }
 
 func (r *GuestbookReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&webappv1.Guestbook{}).
-		// add the watches for the specific resource types that the
-		// custom resource owns to watch for changes on those
-		Owns(&appsv1.Deployment{}).
-		Owns(&corev1.Service{}).
-		Complete(r)
+	// SetupWithDynamicTypeWatches will configure the controller to dynamically
+	// watch for any resource type that the controller owns.
+	return reconciler.SetupWithDynamicTypeWatches(r,
+		ctrl.NewControllerManagedBy(mgr).
+			For(&webappv1.Guestbook{}),
+	)
 }
 ```
 

config/global.go

@@ -17,10 +17,12 @@ type ReconcileConfigForGVK struct {
 var config = struct {
 	annotationsDomain              string
 	resourcePruner                 bool
+	dynamicWatches                 bool
 	defaultResourceReconcileConfig map[string]ReconcileConfigForGVK
 }{
 	annotationsDomain: "basereconciler.3cale.net",
 	resourcePruner:    true,
+	dynamicWatches:    true,
 	defaultResourceReconcileConfig: map[string]ReconcileConfigForGVK{
 		"*": {
 			EnsureProperties: []string{
@@ -56,6 +58,17 @@ func DisableResourcePruner() { config.resourcePruner = false }
 // IsResourcePrunerEnabled returs a boolean indicating wheter the resource pruner is enabled or not.
 func IsResourcePrunerEnabled() bool { return config.resourcePruner }
 
+// EnableDynamicWatches enables controller dynamic watches. Dynamic watches keep track of the resource
+// types that the controller owns and dynamically adds watches to the controller for those.
+func EnableDynamicWatches() { config.dynamicWatches = true }
+
+// DisableDynamicWatches disables controller dynamic watches. Dynamic watches keep track of the resource
+// types that the controller owns and dynamically adds watches to the controller for those.
+func DisableDynamicWatches() { config.dynamicWatches = false }
+
+// AreDynamicWatchesEnabled returs a boolean indicating wheter the dynamic watches are enabled or not.
+func AreDynamicWatchesEnabled() bool { return config.dynamicWatches }
+
 // GetDefaultReconcileConfigForGVK returns the default configuration that instructs basereconciler how to reconcile
 // a given kubernetes GVK (GroupVersionKind). This default config will be used if the "resource.Template" object (see
 // the resource package) does not specify a configuration itself.

mutators/mutators.go

@@ -50,7 +50,7 @@ func SetDeploymentReplicas(enforce bool) resource.TemplateMutationFunction {
 
 // SetServiceLiveValues retrieves some live values of the Service spec from the Kubernetes
 // API to avoid overwriting them. These values are typically set the by the kube-controller-manager
-// (in some rare occasions the user might explicitely set them) and should not be modified by the
+// (in some rare occasions the user might explicitly set them) and should not be modified by the
 // reconciler. The fields that this function keeps in sync with the live state are:
 //   - spec.clusterIP
 //   - spec.ClisterIPs

reconciler/pruner.go

@@ -3,16 +3,13 @@ package reconciler
 import (
 	"context"
 	"fmt"
-	"reflect"
 	"strconv"
-	"sync"
 
 	"github.com/3scale-ops/basereconciler/config"
 	"github.com/3scale-ops/basereconciler/util"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 )
@@ -72,18 +69,3 @@ func isPrunerEnabled(owner client.Object) bool {
 	}
 	return prune && config.IsResourcePrunerEnabled()
 }
-
-type typeTracker struct {
-	seenTypes []schema.GroupVersionKind
-	mu        sync.Mutex
-}
-
-func (tt *typeTracker) trackType(gvk schema.GroupVersionKind) {
-	if !util.ContainsBy(tt.seenTypes, func(x schema.GroupVersionKind) bool {
-		return reflect.DeepEqual(x, gvk)
-	}) {
-		tt.mu.Lock()
-		defer tt.mu.Unlock()
-		tt.seenTypes = append(tt.seenTypes, gvk)
-	}
-}

reconciler/reconciler.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/3scale-ops/basereconciler/config"
 	"github.com/3scale-ops/basereconciler/resource"
 	"github.com/3scale-ops/basereconciler/util"
 	"github.com/go-logr/logr"
@@ -252,7 +253,7 @@ func (r *Reconciler) isInitialized(ctx context.Context, obj client.Object, fns [
 // inMemoryInitialization can be used to perform initializarion on the resource that is not
 // persisted in the API storage. This can be used to perform initialization on the resource without
 // writing it to the API to avoid surfacing it uo to the user. This approach is a bit more
-// gitops firendly as it avoids modifying the resource, but it doesn't provide any information
+// gitops friendly as it avoids modifying the resource, but it doesn't provide any information
 // to the user on the initialization being used for reconciliation.
 func (r *Reconciler) inMemoryInitialization(ctx context.Context, obj client.Object, fns []inMemoryinitializationFunction) error {
 	for _, fn := range fns {
@@ -285,9 +286,10 @@ func (r *Reconciler) finalize(ctx context.Context, fns []finalizationFunction, l
 //   - Each template is added to the list of managed resources if resource.CreateOrUpdate returns with no error
 //   - If the resource pruner is enabled any resource owned by the custom resource not present in the list of managed
 //     resources is deleted. The resource pruner must be enabled in the global config (see package config) and also not
-//     explicitely disabled in the resource by the '<annotations-domain>/prune: true/false' annotation.
+//     explicitly disabled in the resource by the '<annotations-domain>/prune: true/false' annotation.
 func (r *Reconciler) ReconcileOwnedResources(ctx context.Context, owner client.Object, list []resource.TemplateInterface) Result {
 	managedResources := []corev1.ObjectReference{}
+	requeue := false
 
 	for _, template := range list {
 		ref, err := resource.CreateOrUpdate(ctx, r.Client, r.Scheme, owner, template)
@@ -296,7 +298,13 @@ func (r *Reconciler) ReconcileOwnedResources(ctx context.Context, owner client.O
 		}
 		if ref != nil {
 			managedResources = append(managedResources, *ref)
-			r.typeTracker.trackType(schema.FromAPIVersionAndKind(ref.APIVersion, ref.Kind))
+			gvk := schema.FromAPIVersionAndKind(ref.APIVersion, ref.Kind)
+			if changed := r.typeTracker.trackType(gvk); changed && config.AreDynamicWatchesEnabled() {
+				r.watchOwned(gvk, owner)
+				// requeue so we make sure we haven't lost any events related to the owned resource
+				// while the watch was not still up
+				requeue = true
+			}
 		}
 	}
 
@@ -307,7 +315,11 @@ func (r *Reconciler) ReconcileOwnedResources(ctx context.Context, owner client.O
 		}
 	}
 
-	return Result{Action: ContinueAction}
+	if requeue {
+		return Result{Action: ReturnAndRequeueAction}
+	} else {
+		return Result{Action: ContinueAction}
+	}
 }
 
 // FilteredEventHandler returns an EventHandler for the specific client.ObjectList

reconciler/reconciler_test.go

@@ -15,12 +15,16 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
 func TestResult_ShouldReturn(t *testing.T) {
@@ -396,11 +400,23 @@ func TestReconciler_ManageResourceLifecycle(t *testing.T) {
 	}
 }
 
+type testController struct {
+	reconcile.Reconciler
+}
+
+func (c *testController) Watch(src source.Source, eventhandler handler.EventHandler, predicates ...predicate.Predicate) error {
+	return nil
+}
+func (c *testController) Start(ctx context.Context) error { return nil }
+func (c *testController) GetLogger() logr.Logger          { return logr.Discard() }
+
 func TestReconciler_ReconcileOwnedResources(t *testing.T) {
+
 	type fields struct {
-		Client client.Client
-		Log    logr.Logger
-		Scheme *runtime.Scheme
+		Client    client.Client
+		Log       logr.Logger
+		Scheme    *runtime.Scheme
+		SeenTypes []schema.GroupVersionKind
 	}
 	type args struct {
 		owner client.Object
@@ -415,16 +431,51 @@ func TestReconciler_ReconcileOwnedResources(t *testing.T) {
 		{
 			name: "Creates owned resources",
 			fields: fields{
-				Client: fake.NewClientBuilder().Build(),
+				Client:    fake.NewClientBuilder().Build(),
+				Log:       logr.Discard(),
+				Scheme:    scheme.Scheme,
+				SeenTypes: []schema.GroupVersionKind{},
+			},
+			args: args{
+				owner: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "owner", Namespace: "ns"}},
+				list: []resource.TemplateInterface{
+					resource.NewTemplateFromObjectFunction[*corev1.Service](
+						func() *corev1.Service {
+							return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "ns"}}
+						}),
+					resource.NewTemplateFromObjectFunction[*corev1.ConfigMap](
+						func() *corev1.ConfigMap {
+							return &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "ns"}}
+						}),
+				},
+			},
+			want: Result{
+				Action:       ReturnAndRequeueAction,
+				RequeueAfter: 0,
+				Error:        nil,
+			},
+		},
+		{
+			name: "Updates owned resources and does not add new watches",
+			fields: fields{
+				Client: fake.NewClientBuilder().WithObjects(
+					&corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "owner", Namespace: "ns"}},
+					&corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "ns"}},
+					&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "ns"}},
+				).Build(),
 				Log:    logr.Discard(),
 				Scheme: scheme.Scheme,
+				SeenTypes: []schema.GroupVersionKind{
+					{Group: "", Version: "v1", Kind: "Service"},
+					{Group: "", Version: "v1", Kind: "ConfigMap"},
+				},
 			},
 			args: args{
 				owner: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "owner", Namespace: "ns"}},
 				list: []resource.TemplateInterface{
 					resource.NewTemplateFromObjectFunction[*corev1.Service](
 						func() *corev1.Service {
-							return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "ns"}}
+							return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "ns", Labels: map[string]string{"key": "value"}}}
 						}),
 					resource.NewTemplateFromObjectFunction[*corev1.ConfigMap](
 						func() *corev1.ConfigMap {
@@ -444,6 +495,10 @@ func TestReconciler_ReconcileOwnedResources(t *testing.T) {
 			r := &Reconciler{
 				Client: tt.fields.Client,
 				Scheme: tt.fields.Scheme,
+				typeTracker: typeTracker{
+					seenTypes: tt.fields.SeenTypes,
+					ctrl:      &testController{},
+				},
 			}
 			got := r.ReconcileOwnedResources(context.TODO(), tt.args.owner, tt.args.list)
 			if diff := cmp.Diff(got, tt.want); len(diff) > 0 {

reconciler/tracker.go

@@ -0,0 +1,85 @@
+package reconciler
+
+import (
+	"reflect"
+	"sync"
+
+	"github.com/3scale-ops/basereconciler/util"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+// ReconcilerWithTypeTracker is a reconciler with a TypeTracker
+// The type tracker is used by the "resource pruner" and "dynamic watches"
+// features
+type ReconcilerWithTypeTracker interface {
+	reconcile.Reconciler
+	BuildTypeTracker(ctrl controller.Controller)
+}
+
+// SetupWithDynamicTypeWatches is a helper to build a controller that can watch resource
+// types dynamically. It is typically used within the "SetupWithManager" function.
+// Example usage:
+//
+//	func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
+//		return reconciler.SetupWithDynamicTypeWatches(r,
+//			ctrl.NewControllerManagedBy(mgr).
+//				For(&v1alpha1.Test{}).
+//	            // add any other watches here
+//				Watches(...}.Watches(...),
+//		)
+//	}
+func SetupWithDynamicTypeWatches(r ReconcilerWithTypeTracker, bldr *builder.Builder) error {
+	ctrl, err := bldr.Build(r)
+	if err != nil {
+		return err
+	}
+	r.BuildTypeTracker(ctrl)
+	return nil
+}
+
+type typeTracker struct {
+	seenTypes []schema.GroupVersionKind
+	ctrl      controller.Controller
+	mu        sync.Mutex
+}
+
+func (tt *typeTracker) trackType(gvk schema.GroupVersionKind) bool {
+	if !util.ContainsBy(tt.seenTypes, func(x schema.GroupVersionKind) bool {
+		return reflect.DeepEqual(x, gvk)
+	}) {
+		tt.mu.Lock()
+		defer tt.mu.Unlock()
+		tt.seenTypes = append(tt.seenTypes, gvk)
+		return true
+	}
+	return false
+}
+
+func (r *Reconciler) watchOwned(gvk schema.GroupVersionKind, owner client.Object) error {
+	o, err := util.NewObjectFromGVK(gvk, r.Scheme)
+	if err != nil {
+		return err
+	}
+	r.typeTracker.mu.Lock()
+	defer r.typeTracker.mu.Unlock()
+	err = r.typeTracker.ctrl.Watch(&source.Kind{Type: o}, &handler.EnqueueRequestForOwner{OwnerType: owner, IsController: true})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// BuildTypeTracker passes the controller to the reconciler so watches
+// can be added dynamically
+func (r *Reconciler) BuildTypeTracker(ctrl controller.Controller) {
+	r.typeTracker = typeTracker{
+		seenTypes: []schema.GroupVersionKind{},
+		ctrl:      ctrl,
+	}
+}

resource/create_or_update.go

@@ -30,7 +30,7 @@ import (
 //   - owner: the object that owns the resource. Used to set the OwnerReference in the resource
 //   - template: the struct that describes how the resource needs to be reconciled. It must implement
 //     the TemplateInterface interface. When template.GetEnsureProperties is not set or an empty list, this
-//     function will lookup for configuration in the global configuration (see pacakge config).
+//     function will lookup for configuration in the global configuration (see package config).
 func CreateOrUpdate(ctx context.Context, cl client.Client, scheme *runtime.Scheme,
 	owner client.Object, template TemplateInterface) (*corev1.ObjectReference, error) {
 

resource/doc.go

@@ -0,0 +1,3 @@
+// Package resource contains types and methods to reconcile controller owned resources
+// It is generalized to work with any GroupVersionKind.
+package resource

resource/template.go

@@ -37,7 +37,7 @@ type Template[T client.Object] struct {
 	// TemplateBuilder has been invoked, to perform mutations on the object that require
 	// access to a kubernetes API server.
 	TemplateMutations []TemplateMutationFunction
-	// IsEnabled specifies whether the resourse described by this Template should
+	// IsEnabled specifies whether the resource described by this Template should
 	// exist or not.
 	IsEnabled bool
 	// EnsureProperties are the properties from the desired object that should be enforced

test/doc.go

@@ -0,0 +1,2 @@
+// Package test contains a test controller with its test suite
+package test