feat: add v2prov kubeconfig label patching

This change adds a controller that watches for v2prov kubeconfig
secrets. It will then add the owned label (if it doesn't exist) as this
is required by CAPI 1.5.0 and higher.

It has been added as a feature that needs to be enabled so that we can
disable it in the future when the changes in Rancher have merged and are
generally available.

Additionally, this adds a new test suite that ensure that the Rancher label patcher
work and that a v2prov cluster can be provisined with CAPI 1.5.x.

In the future we can use this test to ensure we don't break v2prov
instead of the existing test. Although the current test is done against
the HEAD version of Rancher which is valuable.

Signed-off-by: Richard Case <richard.case@suse.com>

charts/rancher-turtles/templates/deployment.yaml

@@ -26,6 +26,7 @@ spec:
       containers:
       - args:
         - --leader-elect
+        - --feature-gates=rancher-kube-secret-patch={{ index .Values "rancherTurtles" "features" "rancher-kubeconfigs" "label"}}
         {{- range .Values.rancherTurtles.managerArguments }}
         - {{ . }}
         {{- end }}  

charts/rancher-turtles/templates/rancher-turtles-components.yaml

@@ -87,6 +87,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  - secrets
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:

charts/rancher-turtles/values.yaml

@@ -11,6 +11,8 @@ rancherTurtles:
     rancher-webhook:
       cleanup: true
       kubectlImage: rancher/kubectl
+    rancher-kubeconfigs:
+      label: true
 cluster-api-operator:
   enabled: true
   cert-manager:

config/rbac/role.yaml

@@ -18,6 +18,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  - secrets
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:

feature/feature.go

@@ -0,0 +1,20 @@
+package feature
+
+import (
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/component-base/featuregate"
+)
+
+const (
+	// RancherKubeSecretPatch is used to enable patching of the Rancher v2prov created kubeconfig
+	// secrets so that they can be used with CAPI 1.5.x.
+	RancherKubeSecretPatch featuregate.Feature = "rancher-kube-secret-patch" //nolint:gosec
+)
+
+func init() {
+	utilruntime.Must(MutableGates.Add(defaultGates))
+}
+
+var defaultGates = map[featuregate.Feature]featuregate.FeatureSpec{
+	RancherKubeSecretPatch: {Default: false, PreRelease: featuregate.Beta},
+}

feature/gates.go

@@ -0,0 +1,19 @@
+package feature
+
+import (
+	"k8s.io/component-base/featuregate"
+
+	"sigs.k8s.io/cluster-api/feature"
+)
+
+var (
+	// MutableGates is a mutable version of DefaultFeatureGate.
+	// Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this.
+	// Tests that need to modify featuregate gates for the duration of their test should use
+	// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, <value>)().
+	MutableGates featuregate.MutableFeatureGate = feature.MutableGates
+
+	// Gates is a shared global FeatureGate.
+	// Top-level commands/options setup that needs to modify this featuregate gate should use DefaultMutableFeatureGate.
+	Gates featuregate.FeatureGate = MutableGates
+)

internal/controllers/patch_kcfg_controller.go

@@ -0,0 +1,168 @@
+/*
+Copyright 2023 SUSE.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/retry"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controllers/external"
+	"sigs.k8s.io/cluster-api/util/predicates"
+
+	provisioningv1 "github.com/rancher-sandbox/rancher-turtles/internal/rancher/provisioning/v1"
+	turtlespredicates "github.com/rancher-sandbox/rancher-turtles/util/predicates"
+)
+
+// RancherKubeconfigSecretReconciler is a controller that will reconcile secrets created by Rancher as
+// part of provisioning v2. Its job is to add the label required by Cluster API v1.5.0 and higher.
+type RancherKubeconfigSecretReconciler struct {
+	Client           client.Client
+	recorder         record.EventRecorder
+	WatchFilterValue string
+	Scheme           *runtime.Scheme
+
+	controller      controller.Controller
+	externalTracker external.ObjectTracker
+}
+
+// SetupWithManager will setup the controller.
+func (r *RancherKubeconfigSecretReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
+	log := log.FromContext(ctx)
+
+	capiPredicates := predicates.All(log,
+		turtlespredicates.V2ProvClusterOwned(log),
+		turtlespredicates.NameHasSuffix(log, "-kubeconfig"),
+	)
+
+	c, err := ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Secret{}).
+		WithOptions(options).
+		WithEventFilter(capiPredicates).
+		Build(r)
+	if err != nil {
+		return fmt.Errorf("creating new controller: %w", err)
+	}
+
+	r.recorder = mgr.GetEventRecorderFor("rancher-turtles-v2prov")
+	r.controller = c
+	r.externalTracker = external.ObjectTracker{
+		Controller: c,
+	}
+
+	return nil
+}
+
+// +kubebuilder:rbac:groups="",resources=secrets;events,verbs=get;list;watch;create;update;patch
+// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;create;update
+// +kubebuilder:rbac:groups=provisioning.cattle.io,resources=clusters;clusters/status,verbs=get;list;watch
+
+// Reconcile will patch v2prov created kubeconfig secrets to add the required owner label if its missing.
+func (r *RancherKubeconfigSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, reterr error) {
+	log := log.FromContext(ctx)
+	log.Info("Reconciling v2prov cluster")
+
+	secret := &corev1.Secret{}
+	if err := r.Client.Get(ctx, req.NamespacedName, secret); err != nil {
+		if apierrors.IsNotFound(err) {
+			return ctrl.Result{Requeue: true}, nil
+		}
+
+		return ctrl.Result{Requeue: true}, err
+	}
+
+	_, ok := secret.Labels[clusterv1.ClusterNameLabel]
+	if ok {
+		log.V(4).Info("kubeconfig secret %s/%s already has the capi cluster label", secret.Name, secret.Name)
+
+		return ctrl.Result{}, nil
+	}
+
+	clusterName, err := r.getClusterName(ctx, secret)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("getting cluster name from secret: %w", err)
+	}
+
+	if clusterName == "" {
+		log.Info("Could not determine cluster name from kubeconfig secret")
+
+		return ctrl.Result{}, nil
+	}
+
+	if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		secretCopy := secret.DeepCopy()
+		if secretCopy.Labels == nil {
+			secretCopy.Labels = map[string]string{}
+		}
+		secretCopy.Labels[clusterv1.ClusterNameLabel] = clusterName
+
+		patchBase := client.MergeFromWithOptions(secret, client.MergeFromWithOptimisticLock{})
+
+		if err := r.Client.Patch(ctx, secretCopy, patchBase); err != nil {
+			return fmt.Errorf("failed to patch secret: %w", err)
+		}
+
+		log.V(4).Info("patched kubeconfig secret", "name", secret.Name, "namespace", secret.Namespace, "cluster", clusterName)
+
+		return nil
+	}); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *RancherKubeconfigSecretReconciler) getClusterName(ctx context.Context, secret *corev1.Secret) (string, error) {
+	v2ProvClusterName := ""
+
+	for _, ref := range secret.OwnerReferences {
+		if ref.APIVersion == provisioningv1.GroupVersion.Identifier() {
+			if ref.Kind == "Cluster" {
+				v2ProvClusterName = ref.Name
+
+				break
+			}
+		}
+	}
+
+	if v2ProvClusterName == "" {
+		return "", nil
+	}
+
+	v2ProvCluster := &provisioningv1.Cluster{}
+
+	if err := r.Client.Get(ctx, types.NamespacedName{Name: v2ProvClusterName, Namespace: secret.Namespace}, v2ProvCluster); err != nil {
+		return "", fmt.Errorf("getting rancher cluster: %w", err)
+	}
+
+	if v2ProvCluster.Spec.RKEConfig == nil {
+		return "", nil
+	}
+
+	return v2ProvCluster.Name, nil
+}