CLOUDP-224541: Added terminationProtection flag (#1356)

Added terminationProtection flag & refactored deletion logic

---------

Co-authored-by: Sergiusz Urbaniak <sergiusz.urbaniak@gmail.com>

config/crd/bases/atlas.mongodb.com_atlasdeployments.yaml

@@ -390,6 +390,13 @@ spec:
                       type: object
                     maxItems: 50
                     type: array
+                  terminationProtectionEnabled:
+                    default: false
+                    description: Flag that indicates whether termination protection
+                      is enabled on the cluster. If set to true, MongoDB Cloud won't
+                      delete the cluster. If set to false, MongoDB Cloud will delete
+                      the cluster.
+                    type: boolean
                   versionReleaseSystem:
                     type: string
                 type: object
@@ -594,7 +601,10 @@ spec:
                     type: array
                   terminationProtectionEnabled:
                     default: false
-                    description: TerminationProtectionEnabled flag
+                    description: Flag that indicates whether termination protection
+                      is enabled on the cluster. If set to true, MongoDB Cloud won't
+                      delete the cluster. If set to false, MongoDB Cloud will delete
+                      the cluster.
                     type: boolean
                 required:
                 - name

config/crd/bases/atlas.mongodb.com_atlasprojects.yaml

@@ -157,7 +157,7 @@ spec:
                               types.
                             type: boolean
                           flowName:
-                            description: Flowdock flow namse in lower-case letters.
+                            description: Flowdock flow name in lower-case letters.
                             type: string
                           flowdockApiTokenRef:
                             description: The Flowdock personal API token. Populated
@@ -185,7 +185,7 @@ spec:
                               are sent. Populated for the SMS notifications type.
                             type: string
                           opsGenieApiKeyRef:
-                            description: Opsgenie API Key. Populated for the OPS_GENIE
+                            description: OpsGenie API Key. Populated for the OPS_GENIE
                               notifications type. If the key later becomes invalid,
                               Atlas sends an email to the project owner and eventually
                               removes the token.

pkg/api/v1/atlasdeployment_types.go

@@ -127,6 +127,9 @@ type AdvancedDeploymentSpec struct {
 	CustomZoneMapping []CustomZoneMapping `json:"customZoneMapping,omitempty"`
 	// +optional
 	ManagedNamespaces []ManagedNamespace `json:"managedNamespaces,omitempty"`
+	// Flag that indicates whether termination protection is enabled on the cluster. If set to true, MongoDB Cloud won't delete the cluster. If set to false, MongoDB Cloud will delete the cluster.
+	// +kubebuilder:default:=false
+	TerminationProtectionEnabled bool `json:"terminationProtectionEnabled,omitempty"`
 }
 
 // ToAtlas converts the AdvancedDeploymentSpec to native Atlas client ToAtlas format.
@@ -169,7 +172,7 @@ type ServerlessSpec struct {
 	// Serverless Backup Options
 	BackupOptions ServerlessBackupOptions `json:"backupOptions,omitempty"`
 
-	// TerminationProtectionEnabled flag
+	// Flag that indicates whether termination protection is enabled on the cluster. If set to true, MongoDB Cloud won't delete the cluster. If set to false, MongoDB Cloud will delete the cluster.
 	// +kubebuilder:default:=false
 	TerminationProtectionEnabled bool `json:"terminationProtectionEnabled,omitempty"`
 }

pkg/api/v1/atlasdeployment_types_test.go

@@ -33,7 +33,7 @@ func init() {
 	excludedClusterFieldsTheirs["replicationFactor"] = true
 
 	// Termination protection
-	excludedClusterFieldsTheirs["terminationProtectionEnabled"] = true
+	// excludedClusterFieldsTheirs["terminationProtectionEnabled"] = true
 
 	// Root cert type
 	excludedClusterFieldsTheirs["rootCertType"] = true

pkg/controller/atlasdatabaseuser/atlasdatabaseuser_controller.go

@@ -244,7 +244,7 @@ func (r *AtlasDatabaseUserReconciler) handleDeletion(
 		}
 	}
 
-	if customresource.IsResourceProtected(dbUser, r.ObjectDeletionProtection) {
+	if customresource.IsResourcePolicyKeepOrDefault(dbUser, r.ObjectDeletionProtection) {
 		log.Info("Not removing Atlas database user from Atlas as per configuration")
 
 		err := customresource.ManageFinalizer(ctx, r.Client, dbUser, customresource.UnsetFinalizer)

pkg/controller/atlasdatafederation/datafederation_controller.go

@@ -156,7 +156,7 @@ func (r *AtlasDataFederationReconciler) Reconcile(context context.Context, req c
 
 	if !dataFederation.GetDeletionTimestamp().IsZero() {
 		if customresource.HaveFinalizer(dataFederation, customresource.FinalizerLabel) {
-			if customresource.IsResourceProtected(dataFederation, r.ObjectDeletionProtection) {
+			if customresource.IsResourcePolicyKeepOrDefault(dataFederation, r.ObjectDeletionProtection) {
 				log.Info("Not removing AtlasDataFederation from Atlas as per configuration")
 			} else {
 				if err = r.deleteDataFederationFromAtlas(context, atlasClient, dataFederation, project, log); err != nil {

pkg/controller/atlasdeployment/atlasdeployment_controller.go

@@ -272,40 +272,51 @@ func (r *AtlasDeploymentReconciler) handleDeletion(
 				return true, workflow.Terminate(workflow.Internal, err.Error())
 			}
 		}
+
+		return false, workflow.OK()
 	}
 
-	if !deployment.GetDeletionTimestamp().IsZero() {
-		if customresource.HaveFinalizer(deployment, customresource.FinalizerLabel) {
-			if err := r.cleanupBindings(workflowCtx.Context, deployment); err != nil {
-				result := workflow.Terminate(workflow.Internal, err.Error())
-				log.Errorw("failed to cleanup deployment bindings (backups)", "error", err)
-				return true, result
-			}
-			isProtected := customresource.IsResourceProtected(deployment, r.ObjectDeletionProtection)
-			if isProtected {
-				log.Info("Not removing Atlas deployment from Atlas as per configuration")
-			} else {
-				if customresource.ResourceShouldBeLeftInAtlas(deployment) {
-					log.Infof("Not removing Atlas Deployment from Atlas as the '%s' annotation is set", customresource.ResourcePolicyAnnotation)
-				} else {
-					if err := r.deleteDeploymentFromAtlas(workflowCtx, log, project, deployment); err != nil {
-						log.Errorf("failed to remove deployment from Atlas: %s", err)
-						result := workflow.Terminate(workflow.Internal, err.Error())
-						workflowCtx.SetConditionFromResult(status.DeploymentReadyType, result)
-						return true, result
-					}
-				}
-			}
-			err := customresource.ManageFinalizer(workflowCtx.Context, r.Client, deployment, customresource.UnsetFinalizer)
-			if err != nil {
-				result := workflow.Terminate(workflow.Internal, err.Error())
-				log.Errorw("failed to remove finalizer", "error", err)
-				return true, result
-			}
-		}
+	if !customresource.HaveFinalizer(deployment, customresource.FinalizerLabel) {
 		return true, prevResult
 	}
-	return false, workflow.OK()
+
+	if err := r.cleanupBindings(workflowCtx.Context, deployment); err != nil {
+		result := workflow.Terminate(workflow.Internal, err.Error())
+		log.Errorw("failed to cleanup deployment bindings (backups)", "error", err)
+		return true, result
+	}
+
+	switch {
+	case customresource.IsResourcePolicyKeepOrDefault(deployment, r.ObjectDeletionProtection):
+		log.Info("Not removing Atlas deployment from Atlas as per configuration")
+	case customresource.IsResourcePolicyKeep(deployment):
+		log.Infof("Not removing Atlas deployment from Atlas as the '%s' annotation is set", customresource.ResourcePolicyAnnotation)
+	case isTerminationProtectionEnabled(deployment):
+		msg := fmt.Sprintf("Termination protection for %s deployment enabled. Deployment in Atlas won't be removed", deployment.GetName())
+		log.Info(msg)
+		r.EventRecorder.Event(deployment, "Warning", "AtlasDeploymentTermination", msg)
+	default:
+		if err := r.deleteDeploymentFromAtlas(workflowCtx, log, project, deployment); err != nil {
+			log.Errorf("failed to remove deployment from Atlas: %s", err)
+			result := workflow.Terminate(workflow.Internal, err.Error())
+			workflowCtx.SetConditionFromResult(status.DeploymentReadyType, result)
+			return true, result
+		}
+	}
+
+	if err := customresource.ManageFinalizer(workflowCtx.Context, r.Client, deployment, customresource.UnsetFinalizer); err != nil {
+		result := workflow.Terminate(workflow.Internal, err.Error())
+		log.Errorw("failed to remove finalizer", "error", err)
+		return true, result
+	}
+
+	return true, prevResult
+}
+
+func isTerminationProtectionEnabled(deployment *mdbv1.AtlasDeployment) bool {
+	return (deployment.Spec.DeploymentSpec != nil &&
+		deployment.Spec.DeploymentSpec.TerminationProtectionEnabled) || (deployment.Spec.ServerlessSpec != nil &&
+		deployment.Spec.ServerlessSpec.TerminationProtectionEnabled)
 }
 
 func (r *AtlasDeploymentReconciler) cleanupBindings(context context.Context, deployment *mdbv1.AtlasDeployment) error {

pkg/controller/atlasproject/atlasproject_controller.go

@@ -241,7 +241,7 @@ func (r *AtlasProjectReconciler) ensureDeletionFinalizer(workflowCtx *workflow.C
 
 	if !project.GetDeletionTimestamp().IsZero() {
 		if customresource.HaveFinalizer(project, customresource.FinalizerLabel) {
-			if customresource.IsResourceProtected(project, r.ObjectDeletionProtection) {
+			if customresource.IsResourcePolicyKeepOrDefault(project, r.ObjectDeletionProtection) {
 				log.Info("Not removing Project from Atlas as per configuration")
 				result = workflow.OK()
 			} else {

pkg/controller/atlasproject/team_reconciler.go

@@ -123,7 +123,7 @@ func (r *AtlasProjectReconciler) teamReconcile(
 		if !team.GetDeletionTimestamp().IsZero() {
 			if customresource.HaveFinalizer(team, customresource.FinalizerLabel) {
 				log.Warnf("team %s is assigned to a project. Remove it from all projects before delete", team.Name)
-			} else if customresource.IsResourceProtected(team, r.ObjectDeletionProtection) {
+			} else if customresource.IsResourcePolicyKeepOrDefault(team, r.ObjectDeletionProtection) {
 				log.Info("Not removing Team from Atlas as per configuration")
 				return workflow.OK().ReconcileResult(), nil
 			} else {

pkg/controller/customresource/customresource.go

@@ -83,8 +83,16 @@ func MarkReconciliationStarted(client client.Client, resource mdbv1.AtlasCustomR
 	return ctx
 }
 
-// ResourceShouldBeLeftInAtlas returns 'true' if the resource should not be removed from Atlas on K8s resource removal.
-func ResourceShouldBeLeftInAtlas(resource mdbv1.AtlasCustomResource) bool {
+func IsResourcePolicyKeepOrDefault(resource mdbv1.AtlasCustomResource, protectionFlag bool) bool {
+	if policy, ok := resource.GetAnnotations()[ResourcePolicyAnnotation]; ok {
+		return policy == ResourcePolicyKeep
+	}
+
+	return protectionFlag
+}
+
+// IsResourcePolicyKeep returns 'true' if the resource should not be removed from Atlas on K8s resource removal.
+func IsResourcePolicyKeep(resource mdbv1.AtlasCustomResource) bool {
 	if v, ok := resource.GetAnnotations()[ResourcePolicyAnnotation]; ok {
 		return v == ResourcePolicyKeep
 	}

pkg/controller/customresource/customresource_test.go

@@ -15,19 +15,19 @@ import (
 
 func TestResourceShouldBeLeftInAtlas(t *testing.T) {
 	t.Run("Empty annotations", func(t *testing.T) {
-		assert.False(t, ResourceShouldBeLeftInAtlas(&v1.AtlasDatabaseUser{}))
+		assert.False(t, IsResourcePolicyKeep(&v1.AtlasDatabaseUser{}))
 	})
 
 	t.Run("Other annotations", func(t *testing.T) {
-		assert.False(t, ResourceShouldBeLeftInAtlas(&v1.AtlasDatabaseUser{
+		assert.False(t, IsResourcePolicyKeep(&v1.AtlasDatabaseUser{
 			ObjectMeta: metav1.ObjectMeta{
 				Annotations: map[string]string{"foo": "bar"},
 			},
 		}))
 	})
 
 	t.Run("Annotation present, resources should be removed", func(t *testing.T) {
-		assert.False(t, ResourceShouldBeLeftInAtlas(&v1.AtlasDatabaseUser{
+		assert.False(t, IsResourcePolicyKeep(&v1.AtlasDatabaseUser{
 			ObjectMeta: metav1.ObjectMeta{
 				// Any other value except for "keep" is considered as "purge"
 				Annotations: map[string]string{ResourcePolicyAnnotation: "foobar"},
@@ -36,7 +36,7 @@ func TestResourceShouldBeLeftInAtlas(t *testing.T) {
 	})
 
 	t.Run("Annotation present, resources should be kept", func(t *testing.T) {
-		assert.True(t, ResourceShouldBeLeftInAtlas(&v1.AtlasDatabaseUser{
+		assert.True(t, IsResourcePolicyKeep(&v1.AtlasDatabaseUser{
 			ObjectMeta: metav1.ObjectMeta{
 				Annotations: map[string]string{ResourcePolicyAnnotation: ResourcePolicyKeep},
 			},

pkg/controller/customresource/protection.go

@@ -39,14 +39,6 @@ func IsOwner(resource mdbv1.AtlasCustomResource, protectionFlag bool, operatorCh
 	return !existInAtlas, nil
 }
 
-func IsResourceProtected(resource mdbv1.AtlasCustomResource, protectionFlag bool) bool {
-	if policy, ok := resource.GetAnnotations()[ResourcePolicyAnnotation]; ok {
-		return policy == ResourcePolicyKeep
-	}
-
-	return protectionFlag
-}
-
 func ApplyLastConfigApplied(ctx context.Context, resource mdbv1.AtlasCustomResource, k8sClient client.Client) error {
 	uObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(resource)
 	if err != nil {

pkg/controller/customresource/protection_test.go

@@ -130,7 +130,7 @@ func TestIsResourceProtected(t *testing.T) {
 	}
 	for _, tc := range tests {
 		t.Run(tc.title, func(t *testing.T) {
-			assert.Equal(t, tc.expectedProtected, customresource.IsResourceProtected(tc.resource, tc.protectionFlag))
+			assert.Equal(t, tc.expectedProtected, customresource.IsResourcePolicyKeepOrDefault(tc.resource, tc.protectionFlag))
 		})
 	}
 }

test/int/datafederation_test.go

@@ -84,7 +84,7 @@ var _ = Describe("AtlasDataFederation", Label("AtlasDataFederation"), func() {
 				By("Removing Atlas DataFederation " + createdDataFederation.Name)
 				Expect(k8sClient.Delete(context.Background(), createdDataFederation)).To(Succeed())
 				deploymentName := createdDataFederation.Name
-				if customresource.ResourceShouldBeLeftInAtlas(createdDataFederation) || customresource.ReconciliationShouldBeSkipped(createdDataFederation) {
+				if customresource.IsResourcePolicyKeep(createdDataFederation) || customresource.ReconciliationShouldBeSkipped(createdDataFederation) {
 					By("Removing Atlas DataFederation " + createdDataFederation.Name + " from Atlas manually")
 					Expect(deleteAtlasDataFederation(createdProject.Status.ID, deploymentName)).To(Succeed())
 				}

test/int/deployment_test.go

@@ -214,6 +214,79 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
 		lastGeneration++
 	}
 
+	Describe("Deployment with Termination Protection should remain in Atlas after the CR is deleted", Label("dedicated-termination-protection", "slow"), func() {
+		It("Should succeed", func() {
+			createdDeployment = mdbv1.DefaultAWSDeployment(namespace.Name, createdProject.Name)
+
+			By(fmt.Sprintf("Creating the Deployment %s", kube.ObjectKeyFromObject(createdDeployment)), func() {
+				createdDeployment.Spec.DeploymentSpec.TerminationProtectionEnabled = true
+
+				performCreate(createdDeployment, 30*time.Minute)
+
+				doDeploymentStatusChecks()
+				checkAtlasState()
+			})
+
+			By("Removing deployment", func() {
+				Expect(k8sClient.Delete(context.Background(), createdDeployment)).To(Succeed())
+			})
+
+			By("Verifying the deployment is still in Atlas", func() {
+				Eventually(func(g Gomega) {
+					ctx, cancelF := context.WithTimeout(context.Background(), 20*time.Second)
+					defer cancelF()
+					aCluster, _, err := atlasClient.ClustersApi.GetCluster(ctx, createdProject.ID(),
+						createdDeployment.GetDeploymentName()).Execute()
+					g.Expect(err).NotTo(HaveOccurred())
+					Expect(aCluster.GetName()).Should(BeEquivalentTo(createdDeployment.GetDeploymentName()))
+				}).WithTimeout(30 * time.Second).WithPolling(5 * time.Second)
+			})
+
+			By("Disabling Termination protection", func() {
+				ctx, cancelF := context.WithTimeout(context.Background(), 20*time.Second)
+				defer cancelF()
+				aCluster, _, err := atlasClient.ClustersApi.GetCluster(ctx, createdProject.ID(),
+					createdDeployment.GetDeploymentName()).Execute()
+				Expect(err).NotTo(HaveOccurred())
+				aCluster.TerminationProtectionEnabled = pointer.MakePtr(false)
+				aCluster.ConnectionStrings = nil
+				_, _, err = atlasClient.ClustersApi.UpdateCluster(ctx, createdProject.ID(), createdDeployment.GetDeploymentName(), aCluster).Execute()
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			By("Waiting for Termination protection to be disabled", func() {
+				Eventually(func(g Gomega) {
+					ctx, cancelF := context.WithTimeout(context.Background(), 20*time.Second)
+					defer cancelF()
+					aCluster, _, err := atlasClient.ClustersApi.GetCluster(ctx, createdProject.ID(),
+						createdDeployment.GetDeploymentName()).Execute()
+					g.Expect(err).NotTo(HaveOccurred())
+					g.Expect(aCluster.TerminationProtectionEnabled).NotTo(BeNil())
+					g.Expect(*aCluster.TerminationProtectionEnabled).To(BeFalse())
+				}).WithTimeout(2 * time.Minute).WithPolling(10 * time.Second)
+			})
+
+			By("Manually deleting the cluster", func() {
+				ctx, cancelF := context.WithTimeout(context.Background(), 20*time.Second)
+				defer cancelF()
+				_, err := atlasClient.ClustersApi.DeleteCluster(ctx, createdProject.ID(),
+					createdDeployment.GetDeploymentName()).Execute()
+				Expect(err).NotTo(HaveOccurred())
+				createdDeployment = nil
+			})
+
+			By("Waiting for Deployment termination", func() {
+				Eventually(func(g Gomega) {
+					ctx, cancelF := context.WithTimeout(context.Background(), 20*time.Second)
+					defer cancelF()
+					_, resp, _ := atlasClient.ClustersApi.GetCluster(ctx, createdProject.ID(),
+						createdDeployment.GetDeploymentName()).Execute()
+					g.Expect(resp.Status).To(Equal(http.StatusNotFound))
+				}).WithTimeout(10 * time.Minute).WithPolling(20 * time.Second)
+			})
+		})
+	})
+
 	Describe("Deployment CR should exist if it is tried to delete and the token is not valid", func() {
 		It("Should Succeed", func() {
 			expectedDeployment := mdbv1.DefaultAWSDeployment(namespace.Name, createdProject.Name)
@@ -1628,7 +1701,7 @@ func deleteDeploymentFromKubernetes(project *mdbv1.AtlasProject, deployment *mdb
 	By(fmt.Sprintf("Removing Atlas Deployment %q", deployment.Name), func() {
 		Expect(k8sClient.Delete(context.Background(), deployment)).To(Succeed())
 		deploymentName := deployment.GetDeploymentName()
-		if customresource.ResourceShouldBeLeftInAtlas(deployment) || customresource.ReconciliationShouldBeSkipped(deployment) {
+		if customresource.IsResourcePolicyKeep(deployment) || customresource.ReconciliationShouldBeSkipped(deployment) {
 			By("Removing Atlas Deployment " + deployment.Name + " from Atlas manually")
 			Expect(deleteAtlasDeployment(project.Status.ID, deploymentName)).To(Succeed())
 		}
@@ -1639,7 +1712,7 @@ func deleteDeploymentFromKubernetes(project *mdbv1.AtlasProject, deployment *mdb
 func deleteProjectFromKubernetes(project *mdbv1.AtlasProject) {
 	By(fmt.Sprintf("Removing Atlas Project %s", project.Status.ID), func() {
 		Expect(k8sClient.Delete(context.Background(), project)).To(Succeed())
-		Eventually(checkAtlasProjectRemoved(project.Status.ID), 60, interval).Should(BeTrue())
+		Eventually(checkAtlasProjectRemoved(project.Status.ID), 240, interval).Should(BeTrue())
 	})
 }