🌱 Support for management v3 downgrade to provisioning v1 with automig…

…ration (#581) * Opt-in for cluster downgrade scenario with managementv3-cluster-migration Signed-off-by: Danil-Grigorev <danil.grigorev@suse.com> * Perform e2e test for downgrade to provv1 with automigrate enabled - Alternative: caa656f Signed-off-by: Danil-Grigorev <danil.grigorev@suse.com> * Update v1 controller to cleanup finalizer on CAPI clusters Signed-off-by: Danil-Grigorev <danil.grigorev@suse.com> --------- Signed-off-by: Danil-Grigorev <danil.grigorev@suse.com>
rancher · Jul 15, 2024 · 0fbf6d0 · 0fbf6d0
1 parent 2444fd5
commit 0fbf6d0
Show file tree

Hide file tree

Showing 8 changed files with 166 additions and 201 deletions.
diff --git a/charts/rancher-turtles/templates/post-upgrade-job.yaml b/charts/rancher-turtles/templates/post-upgrade-job.yaml
@@ -1,4 +1,4 @@
-{{- if and (eq (index .Values "rancherTurtles" "features" "managementv3-cluster" "enabled") true) (eq (index .Values "rancherTurtles" "features" "managementv3-cluster-migration" "enabled") true) }}
+{{- if eq (index .Values "rancherTurtles" "features" "managementv3-cluster-migration" "enabled") true }}
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -24,6 +24,13 @@ rules:
   verbs:
   - list
   - delete
+- apiGroups:
+  - management.cattle.io
+  resources:
+  - clusters
+  verbs:
+  - list
+  - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -58,7 +65,11 @@ spec:
           image: {{ index .Values "rancherTurtles" "features" "rancher-webhook" "kubectlImage" }}
           args:
           - delete
+          {{- if eq (index .Values "rancherTurtles" "features" "managementv3-cluster" "enabled") true }}
           - clusters.provisioning.cattle.io
+          {{- else }}
+          - clusters.management.cattle.io
+          {{- end }}
           - --selector=cluster-api.cattle.io/owned
           - -A
           - --ignore-not-found=true

diff --git a/internal/controllers/import_controller.go b/internal/controllers/import_controller.go
@@ -30,10 +30,12 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
@@ -43,6 +45,7 @@ import (
 	"sigs.k8s.io/cluster-api/util/predicates"
 
 	"github.com/rancher/turtles/feature"
+	managementv3 "github.com/rancher/turtles/internal/rancher/management/v3"
 	provisioningv1 "github.com/rancher/turtles/internal/rancher/provisioning/v1"
 	"github.com/rancher/turtles/util"
 	turtlesannotations "github.com/rancher/turtles/util/annotations"
@@ -143,16 +146,23 @@ func (r *CAPIImportReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 	log = log.WithValues("cluster", capiCluster.Name)
 
+	// Collect errors as an aggregate to return together after all patches have been performed.
+	var errs []error
+
+	if !capiCluster.ObjectMeta.DeletionTimestamp.IsZero() && controllerutil.RemoveFinalizer(capiCluster, managementv3.CapiClusterFinalizer) {
+		if err := r.Client.Patch(ctx, capiCluster, patchBase); err != nil {
+			log.Error(err, "failed to remove CAPI cluster finalizer "+managementv3.CapiClusterFinalizer)
+			errs = append(errs, err)
+		}
+	}
+
 	// Wait for controlplane to be ready. This should never be false as the predicates
 	// do the filtering.
 	if !capiCluster.Status.ControlPlaneReady && !conditions.IsTrue(capiCluster, clusterv1.ControlPlaneReadyCondition) {
 		log.Info("clusters control plane is not ready, requeue")
 		return ctrl.Result{RequeueAfter: defaultRequeueDuration}, nil
 	}
 
-	// Collect errors as an aggregate to return together after all patches have been performed.
-	var errs []error
-
 	result, err := r.reconcile(ctx, capiCluster)
 	if err != nil {
 		errs = append(errs, fmt.Errorf("error reconciling cluster: %w", err))
@@ -350,3 +360,42 @@ func (r *CAPIImportReconciler) reconcileDelete(ctx context.Context, capiCluster
 
 	return ctrl.Result{}, nil
 }
+
+// CAPIDowngradeReconciler is a reconciler for downgraded managementv3 clusters.
+type CAPIDowngradeReconciler struct {
+	RancherClient client.Client
+	Scheme        *runtime.Scheme
+}
+
+// SetupWithManager sets up reconciler with manager.
+func (r *CAPIDowngradeReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager, options controller.Options) error {
+	if err := ctrl.NewControllerManagedBy(mgr).
+		For(&managementv3.Cluster{}).
+		WithOptions(options).
+		WithEventFilter(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			_, exist := object.GetLabels()[ownedLabelName]
+			return exist
+		})).
+		Complete(reconcile.AsReconciler(r.RancherClient, r)); err != nil {
+		return fmt.Errorf("creating new downgrade controller: %w", err)
+	}
+
+	return nil
+}
+
+// Reconcile performs check for downgraded clusters and removes finalizer on the clusters still owned by the previous management v3 controller.
+func (r *CAPIDowngradeReconciler) Reconcile(ctx context.Context, cluster *managementv3.Cluster) (res ctrl.Result, err error) {
+	log := log.FromContext(ctx)
+
+	patchBase := client.MergeFromWithOptions(cluster.DeepCopy(), client.MergeFromWithOptimisticLock{})
+
+	if !controllerutil.RemoveFinalizer(cluster, managementv3.CapiClusterFinalizer) {
+		return
+	}
+
+	if err = r.RancherClient.Patch(ctx, cluster, patchBase); err != nil {
+		log.Error(err, "Unable to remove turtles finalizer from cluster"+cluster.Name)
+	}
+
+	return
+}
diff --git a/internal/controllers/import_controller_test.go b/internal/controllers/import_controller_test.go
@@ -146,6 +146,25 @@ var _ = Describe("reconcile CAPI Cluster", func() {
 		}).Should(Succeed())
 	})
 
+	It("should remove a CAPI cluster with turtles finalizer", func() {
+		capiCluster.Finalizers = []string{managementv3.CapiClusterFinalizer}
+		Expect(cl.Create(ctx, capiCluster)).To(Succeed())
+		capiCluster.Status.ControlPlaneReady = true
+		Expect(cl.Status().Update(ctx, capiCluster)).To(Succeed())
+		Expect(cl.Delete(ctx, capiCluster)).To(Succeed())
+
+		Eventually(func(g Gomega) {
+			_, err := r.Reconcile(ctx, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: capiCluster.Namespace,
+					Name:      capiCluster.Name,
+				},
+			})
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(cl.Get(ctx, client.ObjectKeyFromObject(capiCluster), capiCluster)).To(HaveOccurred())
+		}).Should(Succeed())
+	})
+
 	It("should reconcile a CAPI cluster when rancher cluster doesn't exist", func() {
 		capiCluster.Labels = map[string]string{
 			importLabelName: "true",

diff --git a/internal/controllers/import_controller_v3.go b/internal/controllers/import_controller_v3.go
@@ -171,17 +171,15 @@ func (r *CAPIImportManagementV3Reconciler) Reconcile(ctx context.Context, req ct
 	// Collect errors as an aggregate to return together after all patches have been performed.
 	var errs []error
 
+	patchBase := client.MergeFromWithOptions(capiCluster.DeepCopy(), client.MergeFromWithOptimisticLock{})
+
 	result, err := r.reconcile(ctx, capiCluster)
 	if err != nil {
 		errs = append(errs, fmt.Errorf("error reconciling cluster: %w", err))
 	}
 
 	if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
-		capiClusterCopy := capiCluster.DeepCopy()
-
-		patchBase := client.MergeFromWithOptions(capiCluster, client.MergeFromWithOptimisticLock{})
-
-		if err := r.Client.Patch(ctx, capiClusterCopy, patchBase); err != nil {
+		if err := r.Client.Patch(ctx, capiCluster, patchBase); err != nil {
 			errs = append(errs, fmt.Errorf("failed to patch cluster: %w", err))
 		}
 		return nil
@@ -456,6 +454,8 @@ func (r *CAPIImportManagementV3Reconciler) reconcileDelete(ctx context.Context,
 		capiCluster.Name,
 		turtlesannotations.ClusterImportedAnnotation))
 
+	patchBase := client.MergeFromWithOptions(capiCluster.DeepCopy(), client.MergeFromWithOptimisticLock{})
+
 	annotations := capiCluster.GetAnnotations()
 	if annotations == nil {
 		annotations = map[string]string{}
@@ -465,7 +465,7 @@ func (r *CAPIImportManagementV3Reconciler) reconcileDelete(ctx context.Context,
 	capiCluster.SetAnnotations(annotations)
 	controllerutil.RemoveFinalizer(capiCluster, managementv3.CapiClusterFinalizer)
 
-	if err := r.Client.Update(ctx, capiCluster); err != nil {
+	if err := r.Client.Patch(ctx, capiCluster, patchBase); err != nil {
 		return fmt.Errorf("error removing finalizer: %w", err)
 	}
 

diff --git a/main.go b/main.go
@@ -226,6 +226,15 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) {
 			setupLog.Error(err, "unable to create capi controller")
 			os.Exit(1)
 		}
+
+		if err := (&controllers.CAPIDowngradeReconciler{
+			RancherClient: rancherClient,
+		}).SetupWithManager(ctx, mgr, controller.Options{
+			MaxConcurrentReconciles: concurrencyNumber,
+		}); err != nil {
+			setupLog.Error(err, "unable to create rancher management v3 downgrade controller")
+			os.Exit(1)
+		}
 	}
 
 	if feature.Gates.Enabled(feature.RancherKubeSecretPatch) {

diff --git a/test/e2e/specs/migrate_gitops_provv1_mgmtv3.go b/test/e2e/specs/migrate_gitops_provv1_mgmtv3.go
@@ -362,6 +362,7 @@ func MigrateToV3UsingGitOpsSpec(ctx context.Context, inputGetter func() MigrateT
 			Image:                        fmt.Sprintf("ghcr.io/rancher/turtles-e2e-%s", runtime.GOARCH),
 			Tag:                          "v0.0.1",
 			WaitDeploymentsReadyInterval: input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-controllers"),
+			SkipCleanup:                  true,
 			AdditionalValues:             map[string]string{},
 		}
 
@@ -387,6 +388,25 @@ func MigrateToV3UsingGitOpsSpec(ctx context.Context, inputGetter func() MigrateT
 
 		By("Rancher should be available using new cluster import")
 		validateRancherCluster()
+
+		By("Running downgrade on provisioningv1 cluster later")
+		upgradeInput.AdditionalValues["rancherTurtles.features.managementv3-cluster.enabled"] = "false"
+		upgradeInput.SkipCleanup = false
+		testenv.UpgradeRancherTurtles(ctx, upgradeInput)
+
+		By("Waiting for the new Rancher cluster record to be removed")
+		Eventually(komega.Get(rancherCluster), deleteClusterWait...).Should(MatchError(ContainSubstring("not found")), "Rancher cluster should be unimported (deleted)")
+
+		By("CAPI cluster should NOT have the 'imported' annotation")
+		Consistently(func() bool {
+			Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
+			annotations := capiCluster.GetAnnotations()
+			_, found := annotations["imported"]
+			return !found
+		}, 5*time.Second).Should(BeTrue(), "'imported' annotation is NOT expected on CAPI cluster")
+
+		By("Rancher should be available using old cluster import")
+		validateLegacyRancherCluster()
 	})
 
 	AfterEach(func() {