From 94bb692efa21da22d8888f281c0dd9886f9c4857 Mon Sep 17 00:00:00 2001 From: Jose Vazquez Date: Tue, 4 Jul 2023 14:27:51 +0200 Subject: [PATCH] CLOUDP-186825: Add deletion protection to deployments Signed-off-by: Jose Vazquez --- cmd/manager/main.go | 20 +- go.mod | 1 + go.sum | 1 + pkg/api/v1/atlasdeployment_types.go | 7 + pkg/controller/atlas/api_error.go | 7 + .../atlasdeployment_controller.go | 345 ++++++++--- .../atlasdeployment_controller_test.go | 537 ++++++++++++++++++ .../atlasdeployment/clusters_mock_test.go | 42 ++ .../atlasdeployment/serverless_mock_test.go | 34 ++ .../atlasproject/atlasproject_controller.go | 1 - test/e2e/annotations_test.go | 1 + test/int/databaseuser_protected_test.go | 5 + test/int/deployment_protected_test.go | 130 +++++ test/int/deployment_test.go | 18 + test/int/deployment_unprotected_test.go | 121 ++++ test/int/integration_suite_test.go | 16 +- 16 files changed, 1191 insertions(+), 95 deletions(-) create mode 100644 pkg/controller/atlasdeployment/atlasdeployment_controller_test.go create mode 100644 pkg/controller/atlasdeployment/clusters_mock_test.go create mode 100644 pkg/controller/atlasdeployment/serverless_mock_test.go create mode 100644 test/int/deployment_protected_test.go create mode 100644 test/int/deployment_unprotected_test.go diff --git a/cmd/manager/main.go b/cmd/manager/main.go index b6a68f587d..20c8f91ae2 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -54,8 +54,6 @@ import ( ) const ( - objectDeletionProtectionFlag = "object-deletion-protection" - subobjectDeletionProtectionFlag = "subobject-deletion-protection" objectDeletionProtectionDefault = false subobjectDeletionProtectionDefault = false @@ -136,14 +134,16 @@ func main() { } if err = (&atlasdeployment.AtlasDeploymentReconciler{ - Client: mgr.GetClient(), - Log: logger.Named("controllers").Named("AtlasDeployment").Sugar(), - Scheme: mgr.GetScheme(), - AtlasDomain: config.AtlasDomain, - GlobalAPISecret: config.GlobalAPISecret, - ResourceWatcher: watch.NewResourceWatcher(), - GlobalPredicates: globalPredicates, - EventRecorder: mgr.GetEventRecorderFor("AtlasDeployment"), + Client: mgr.GetClient(), + Log: logger.Named("controllers").Named("AtlasDeployment").Sugar(), + Scheme: mgr.GetScheme(), + AtlasDomain: config.AtlasDomain, + GlobalAPISecret: config.GlobalAPISecret, + ResourceWatcher: watch.NewResourceWatcher(), + GlobalPredicates: globalPredicates, + EventRecorder: mgr.GetEventRecorderFor("AtlasDeployment"), + ObjectDeletionProtection: config.ObjectDeletionProtection, + SubObjectDeletionProtection: config.SubObjectDeletionProtection, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "AtlasDeployment") os.Exit(1) diff --git a/go.mod b/go.mod index 724143ea66..921865ca76 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/s2a-go v0.1.4 // indirect diff --git a/go.sum b/go.sum index ca5c95194f..30e09a934d 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,7 @@ github.com/aws/aws-sdk-go v1.44.318 h1:Yl66rpbQHFUbxe9JBKLcvOvRivhVgP6+zH0b9KzAR github.com/aws/aws-sdk-go v1.44.318/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/pkg/api/v1/atlasdeployment_types.go b/pkg/api/v1/atlasdeployment_types.go index 856329681f..f50b323c34 100644 --- a/pkg/api/v1/atlasdeployment_types.go +++ b/pkg/api/v1/atlasdeployment_types.go @@ -210,6 +210,13 @@ type ServerlessSpec struct { PrivateEndpoints []ServerlessPrivateEndpoint `json:"privateEndpoints,omitempty"` } +// ToAtlas converts the ServerlessSpec to native Atlas client Cluster format. +func (s *ServerlessSpec) ToAtlas() (*mongodbatlas.Cluster, error) { + result := &mongodbatlas.Cluster{} + err := compat.JSONCopy(result, s) + return result, err +} + // BiConnector specifies BI Connector for Atlas configuration on this deployment. type BiConnector struct { Enabled *bool `json:"enabled,omitempty"` diff --git a/pkg/controller/atlas/api_error.go b/pkg/controller/atlas/api_error.go index 3557438688..692dae4bb8 100644 --- a/pkg/controller/atlas/api_error.go +++ b/pkg/controller/atlas/api_error.go @@ -17,6 +17,13 @@ const ( // Error indicates that the cluster doesn't exist ClusterNotFound = "CLUSTER_NOT_FOUND" + // ServerlessClusterNotFound indicates that the serverless cluster doesn't exist + ServerlessInstanceNotFound = "SERVERLESS_INSTANCE_NOT_FOUND" + + // ServerlessClusterFromClusterAPI indicates that we are trying to access + // a serverless instance from the cluster API, which is not allowed + ServerlessInstanceFromClusterAPI = "CANNOT_USE_SERVERLESS_INSTANCE_IN_CLUSTER_API" + // Resource not found ResourceNotFound = "RESOURCE_NOT_FOUND" ) diff --git a/pkg/controller/atlasdeployment/atlasdeployment_controller.go b/pkg/controller/atlasdeployment/atlasdeployment_controller.go index 6a130d1097..5aa18431b5 100644 --- a/pkg/controller/atlasdeployment/atlasdeployment_controller.go +++ b/pkg/controller/atlasdeployment/atlasdeployment_controller.go @@ -36,6 +36,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + mdbv1 "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1" "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/provider" "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/status" @@ -46,19 +49,22 @@ import ( "github.com/mongodb/mongodb-atlas-kubernetes/pkg/controller/validate" "github.com/mongodb/mongodb-atlas-kubernetes/pkg/controller/watch" "github.com/mongodb/mongodb-atlas-kubernetes/pkg/controller/workflow" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/compat" "github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/kube" ) // AtlasDeploymentReconciler reconciles an AtlasDeployment object type AtlasDeploymentReconciler struct { watch.ResourceWatcher - Client client.Client - Log *zap.SugaredLogger - Scheme *runtime.Scheme - AtlasDomain string - GlobalAPISecret client.ObjectKey - GlobalPredicates []predicate.Predicate - EventRecorder record.EventRecorder + Client client.Client + Log *zap.SugaredLogger + Scheme *runtime.Scheme + AtlasDomain string + GlobalAPISecret client.ObjectKey + GlobalPredicates []predicate.Predicate + EventRecorder record.EventRecorder + ObjectDeletionProtection bool + SubObjectDeletionProtection bool } // +kubebuilder:rbac:groups=atlas.mongodb.com,resources=atlasdeployments,verbs=get;list;watch;create;update;patch;delete @@ -87,6 +93,7 @@ func (r *AtlasDeploymentReconciler) Reconcile(context context.Context, req ctrl. if !result.IsOk() { return result.ReconcileResult(), nil } + prevResult := result if shouldSkip := customresource.ReconciliationShouldBeSkipped(deployment); shouldSkip { log.Infow(fmt.Sprintf("-> Skipping AtlasDeployment reconciliation as annotation %s=%s", customresource.ReconciliationPolicyAnnotation, customresource.ReconciliationPolicySkip), "spec", deployment.Spec) @@ -101,14 +108,14 @@ func (r *AtlasDeploymentReconciler) Reconcile(context context.Context, req ctrl. return workflow.OK().ReconcileResult(), nil } - ctx := customresource.MarkReconciliationStarted(r.Client, deployment, log) + workflowCtx := customresource.MarkReconciliationStarted(r.Client, deployment, log) log.Infow("-> Starting AtlasDeployment reconciliation", "spec", deployment.Spec, "status", deployment.Status) defer func() { - statushandler.Update(ctx, r.Client, r.EventRecorder, deployment) - r.EnsureMultiplesResourcesAreWatched(req.NamespacedName, log, ctx.ListResourcesToWatch()...) + statushandler.Update(workflowCtx, r.Client, r.EventRecorder, deployment) + r.EnsureMultiplesResourcesAreWatched(req.NamespacedName, log, workflowCtx.ListResourcesToWatch()...) }() - resourceVersionIsValid := customresource.ValidateResourceVersion(ctx, deployment, r.Log) + resourceVersionIsValid := customresource.ValidateResourceVersion(workflowCtx, deployment, r.Log) if !resourceVersionIsValid.IsOk() { r.Log.Debugf("deployment validation result: %v", resourceVersionIsValid) return resourceVersionIsValid.ReconcileResult(), nil @@ -116,71 +123,51 @@ func (r *AtlasDeploymentReconciler) Reconcile(context context.Context, req ctrl. if err := validate.DeploymentSpec(deployment.Spec); err != nil { result := workflow.Terminate(workflow.Internal, err.Error()) - ctx.SetConditionFromResult(status.ValidationSucceeded, result) + workflowCtx.SetConditionFromResult(status.ValidationSucceeded, result) return result.ReconcileResult(), nil } - ctx.SetConditionTrue(status.ValidationSucceeded) + workflowCtx.SetConditionTrue(status.ValidationSucceeded) project := &mdbv1.AtlasProject{} if result := r.readProjectResource(context, deployment, project); !result.IsOk() { - ctx.SetConditionFromResult(status.DeploymentReadyType, result) + workflowCtx.SetConditionFromResult(status.DeploymentReadyType, result) return result.ReconcileResult(), nil } connection, err := atlas.ReadConnection(log, r.Client, r.GlobalAPISecret, project.ConnectionSecretObjectKey()) if err != nil { result := workflow.Terminate(workflow.AtlasCredentialsNotProvided, err.Error()) - ctx.SetConditionFromResult(status.DeploymentReadyType, result) + workflowCtx.SetConditionFromResult(status.DeploymentReadyType, result) return result.ReconcileResult(), nil } - ctx.Connection = connection + workflowCtx.Connection = connection atlasClient, err := atlas.Client(r.AtlasDomain, connection, log) if err != nil { result := workflow.Terminate(workflow.Internal, err.Error()) - ctx.SetConditionFromResult(status.DeploymentReadyType, result) + workflowCtx.SetConditionFromResult(status.DeploymentReadyType, result) return result.ReconcileResult(), nil } - ctx.Client = atlasClient + workflowCtx.Client = atlasClient // Allow users to specify M0/M2/M5 deployments without providing TENANT for Normal and Serverless deployments r.verifyNonTenantCase(deployment) - if deployment.GetDeletionTimestamp().IsZero() { - if !customresource.HaveFinalizer(deployment, customresource.FinalizerLabel) { - err = r.Client.Get(context, kube.ObjectKeyFromObject(deployment), deployment) - if err != nil { - result = workflow.Terminate(workflow.Internal, err.Error()) - return result.ReconcileResult(), nil - } - customresource.SetFinalizer(deployment, customresource.FinalizerLabel) - if err = r.Client.Update(context, deployment); err != nil { - result = workflow.Terminate(workflow.Internal, err.Error()) - log.Errorw("failed to add finalizer", "error", err) - return result.ReconcileResult(), nil - } - } + if result := r.checkDeploymentIsManaged(workflowCtx, context, log, project, deployment); !result.IsOk() { + return result.ReconcileResult(), nil } - if !deployment.GetDeletionTimestamp().IsZero() { - if customresource.HaveFinalizer(deployment, customresource.FinalizerLabel) { - if customresource.ResourceShouldBeLeftInAtlas(deployment) { - log.Infof("Not removing Atlas Deployment from Atlas as the '%s' annotation is set", customresource.ResourcePolicyAnnotation) - } else { - if err = r.deleteDeploymentFromAtlas(context, project, deployment, atlasClient, log); err != nil { - log.Errorf("failed to remove deployment from Atlas: %s", err) - result = workflow.Terminate(workflow.Internal, err.Error()) - ctx.SetConditionFromResult(status.DeploymentReadyType, result) - return result.ReconcileResult(), nil - } - } - err = r.removeDeletionFinalizer(context, deployment) - if err != nil { - result = workflow.Terminate(workflow.Internal, err.Error()) - log.Errorw("failed to remove finalizer", "error", err) - return result.ReconcileResult(), nil - } - } + deletionRequest, result := r.handleDeletion(workflowCtx, context, log, prevResult, project, deployment) + if deletionRequest { + return result.ReconcileResult(), nil + } + + err = customresource.ApplyLastConfigApplied(context, deployment, r.Client) + if err != nil { + result = workflow.Terminate(workflow.Internal, err.Error()) + workflowCtx.SetConditionFromResult(status.DeploymentReadyType, result) + log.Error(result.GetMessage()) + return result.ReconcileResult(), nil } @@ -194,14 +181,14 @@ func (r *AtlasDeploymentReconciler) Reconcile(context context.Context, req ctrl. } handleDeployment := r.selectDeploymentHandler(deployment) - if result, _ := handleDeployment(ctx, project, deployment, req); !result.IsOk() { - ctx.SetConditionFromResult(status.DeploymentReadyType, result) + if result, _ := handleDeployment(workflowCtx, project, deployment, req); !result.IsOk() { + workflowCtx.SetConditionFromResult(status.DeploymentReadyType, result) return result.ReconcileResult(), nil } if !deployment.IsServerless() { - if result := r.handleAdvancedOptions(ctx, project, deployment); !result.IsOk() { - ctx.SetConditionFromResult(status.DeploymentReadyType, result) + if result := r.handleAdvancedOptions(workflowCtx, project, deployment); !result.IsOk() { + workflowCtx.SetConditionFromResult(status.DeploymentReadyType, result) return result.ReconcileResult(), nil } } @@ -231,6 +218,103 @@ func (r *AtlasDeploymentReconciler) verifyNonTenantCase(deployment *mdbv1.AtlasD modifyProviderSettings(pSettings, deploymentType) } +func (r *AtlasDeploymentReconciler) checkDeploymentIsManaged( + workflowCtx *workflow.Context, + context context.Context, + log *zap.SugaredLogger, + project *mdbv1.AtlasProject, + deployment *mdbv1.AtlasDeployment, +) workflow.Result { + dply := deployment + if deployment.IsLegacyDeployment() { + dply = deployment.DeepCopy() + if err := ConvertLegacyDeployment(&dply.Spec); err != nil { + result := workflow.Terminate(workflow.Internal, err.Error()) + log.Errorw("failed to temporary convert legacy deployment", "error", err) + return result + } + dply.Spec.DeploymentSpec = nil + } + + owner, err := customresource.IsOwner( + dply, + r.ObjectDeletionProtection, + customresource.IsResourceManagedByOperator, + managedByAtlas(context, workflowCtx.Client, project.ID(), log), + ) + + if err != nil { + result := workflow.Terminate(workflow.Internal, fmt.Sprintf("unable to resolve ownership for deletion protection: %s", err)) + workflowCtx.SetConditionFromResult(status.DatabaseUserReadyType, result) + log.Error(result.GetMessage()) + + return result + } + + if !owner { + result := workflow.Terminate( + workflow.AtlasDeletionProtection, + "unable to reconcile Deployment due to deletion protection being enabled. see https://dochub.mongodb.org/core/ako-deletion-protection for further information", + ) + workflowCtx.SetConditionFromResult(status.DatabaseUserReadyType, result) + log.Error(result.GetMessage()) + + return result + } + + return workflow.OK() +} + +func (r *AtlasDeploymentReconciler) handleDeletion( + workflowCtx *workflow.Context, + context context.Context, + log *zap.SugaredLogger, + prevResult workflow.Result, + project *mdbv1.AtlasProject, + deployment *mdbv1.AtlasDeployment, +) (bool, workflow.Result) { + if deployment.GetDeletionTimestamp().IsZero() { + if !customresource.HaveFinalizer(deployment, customresource.FinalizerLabel) { + err := r.Client.Get(context, kube.ObjectKeyFromObject(deployment), deployment) + if err != nil { + return true, workflow.Terminate(workflow.Internal, err.Error()) + } + customresource.SetFinalizer(deployment, customresource.FinalizerLabel) + if err = r.Client.Update(context, deployment); err != nil { + return true, workflow.Terminate(workflow.Internal, err.Error()) + } + } + } + + if !deployment.GetDeletionTimestamp().IsZero() { + if customresource.HaveFinalizer(deployment, customresource.FinalizerLabel) { + isProtected := customresource.IsResourceProtected(deployment, r.ObjectDeletionProtection) + if isProtected { + log.Info("Not removing Atlas deployment from Atlas as per configuration") + } else { + if customresource.ResourceShouldBeLeftInAtlas(deployment) { + log.Infof("Not removing Atlas Deployment from Atlas as the '%s' annotation is set", customresource.ResourcePolicyAnnotation) + } else { + if err := r.deleteDeploymentFromAtlas(workflowCtx, context, log, project, deployment); err != nil { + log.Errorf("failed to remove deployment from Atlas: %s", err) + result := workflow.Terminate(workflow.Internal, err.Error()) + workflowCtx.SetConditionFromResult(status.DeploymentReadyType, result) + return true, result + } + } + } + err := customresource.ManageFinalizer(context, r.Client, deployment, customresource.UnsetFinalizer) + if err != nil { + result := workflow.Terminate(workflow.Internal, err.Error()) + log.Errorw("failed to remove finalizer", "error", err) + return true, result + } + } + return true, prevResult + } + return false, workflow.OK() +} + func modifyProviderSettings(pSettings *mdbv1.ProviderSettingsSpec, deploymentType string) { if pSettings == nil || string(pSettings.ProviderName) == deploymentType { return @@ -257,10 +341,10 @@ func (r *AtlasDeploymentReconciler) selectDeploymentHandler(deployment *mdbv1.At } // handleAdvancedDeployment ensures the state of the deployment using the Advanced Deployment API -func (r *AtlasDeploymentReconciler) handleAdvancedDeployment(ctx *workflow.Context, project *mdbv1.AtlasProject, deployment *mdbv1.AtlasDeployment, req reconcile.Request) (workflow.Result, error) { - c, result := r.ensureAdvancedDeploymentState(ctx, project, deployment) +func (r *AtlasDeploymentReconciler) handleAdvancedDeployment(workflowCtx *workflow.Context, project *mdbv1.AtlasProject, deployment *mdbv1.AtlasDeployment, req reconcile.Request) (workflow.Result, error) { + c, result := r.ensureAdvancedDeploymentState(workflowCtx, project, deployment) if c != nil && c.StateName != "" { - ctx.EnsureStatusOption(status.AtlasDeploymentStateNameOption(c.StateName)) + workflowCtx.EnsureStatusOption(status.AtlasDeploymentStateNameOption(c.StateName)) } if !result.IsOk() { @@ -278,7 +362,7 @@ func (r *AtlasDeploymentReconciler) handleAdvancedDeployment(ctx *workflow.Conte ) } - ctx.EnsureStatusOption(status.AtlasDeploymentReplicaSet(replicaSetStatus)) + workflowCtx.EnsureStatusOption(status.AtlasDeploymentReplicaSet(replicaSetStatus)) backupEnabled := false if c.BackupEnabled != nil { @@ -287,32 +371,32 @@ func (r *AtlasDeploymentReconciler) handleAdvancedDeployment(ctx *workflow.Conte if err := r.ensureBackupScheduleAndPolicy( context.Background(), - ctx, project.ID(), + workflowCtx, project.ID(), deployment, backupEnabled, ); err != nil { result := workflow.Terminate(workflow.Internal, err.Error()) - ctx.SetConditionFromResult(status.DeploymentReadyType, result) + workflowCtx.SetConditionFromResult(status.DeploymentReadyType, result) return result, nil } - if csResult := r.ensureConnectionSecrets(ctx, project, c.Name, c.ConnectionStrings, deployment); !csResult.IsOk() { + if csResult := r.ensureConnectionSecrets(workflowCtx, project, c.Name, c.ConnectionStrings, deployment); !csResult.IsOk() { return csResult, nil } - ctx. + workflowCtx. SetConditionTrue(status.DeploymentReadyType). EnsureStatusOption(status.AtlasDeploymentMongoDBVersionOption(c.MongoDBVersion)). EnsureStatusOption(status.AtlasDeploymentConnectionStringsOption(c.ConnectionStrings)) - ctx.SetConditionTrue(status.ReadyType) + workflowCtx.SetConditionTrue(status.ReadyType) return result, nil } // handleServerlessInstance ensures the state of the serverless instance using the serverless API -func (r *AtlasDeploymentReconciler) handleServerlessInstance(ctx *workflow.Context, project *mdbv1.AtlasProject, deployment *mdbv1.AtlasDeployment, req reconcile.Request) (workflow.Result, error) { - c, result := ensureServerlessInstanceState(ctx, project, deployment.Spec.ServerlessSpec) - return r.ensureConnectionSecretsAndSetStatusOptions(ctx, project, deployment, result, c) +func (r *AtlasDeploymentReconciler) handleServerlessInstance(workflowCtx *workflow.Context, project *mdbv1.AtlasProject, deployment *mdbv1.AtlasDeployment, req reconcile.Request) (workflow.Result, error) { + c, result := ensureServerlessInstanceState(workflowCtx, project, deployment.Spec.ServerlessSpec) + return r.ensureConnectionSecretsAndSetStatusOptions(workflowCtx, project, deployment, result, c) } // ensureConnectionSecretsAndSetStatusOptions creates the relevant connection secrets and sets @@ -407,10 +491,10 @@ func (r *AtlasDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { // Delete implements a handler for the Delete event. func (r *AtlasDeploymentReconciler) deleteConnectionStrings( - ctx context.Context, + context context.Context, + log *zap.SugaredLogger, project *mdbv1.AtlasProject, deployment *mdbv1.AtlasDeployment, - log *zap.SugaredLogger, ) error { // We always remove the connection secrets even if the deployment is not removed from Atlas secrets, err := connectionsecret.ListByDeploymentName(r.Client, "", project.ID(), deployment.GetDeploymentName()) @@ -419,7 +503,7 @@ func (r *AtlasDeploymentReconciler) deleteConnectionStrings( } for i := range secrets { - if err := r.Client.Delete(ctx, &secrets[i]); err != nil { + if err := r.Client.Delete(context, &secrets[i]); err != nil { if k8serrors.IsNotFound(err) { continue } @@ -431,23 +515,24 @@ func (r *AtlasDeploymentReconciler) deleteConnectionStrings( } func (r *AtlasDeploymentReconciler) deleteDeploymentFromAtlas( - ctx context.Context, + workflowCtx *workflow.Context, + context context.Context, + log *zap.SugaredLogger, project *mdbv1.AtlasProject, deployment *mdbv1.AtlasDeployment, - atlasClient mongodbatlas.Client, - log *zap.SugaredLogger, ) error { log.Infow("-> Starting AtlasDeployment deletion", "spec", deployment.Spec) - err := r.deleteConnectionStrings(ctx, project, deployment, log) + err := r.deleteConnectionStrings(context, log, project, deployment) if err != nil { return err } + atlasClient := workflowCtx.Client if deployment.IsServerless() { - _, err = atlasClient.ServerlessInstances.Delete(ctx, project.Status.ID, deployment.GetDeploymentName()) + _, err = atlasClient.ServerlessInstances.Delete(context, project.Status.ID, deployment.GetDeploymentName()) } else { - _, err = atlasClient.AdvancedClusters.Delete(ctx, project.Status.ID, deployment.GetDeploymentName(), nil) + _, err = atlasClient.AdvancedClusters.Delete(context, project.Status.ID, deployment.GetDeploymentName(), nil) } var apiError *mongodbatlas.ErrorResponse @@ -477,4 +562,110 @@ func (r *AtlasDeploymentReconciler) removeDeletionFinalizer(context context.Cont return nil } -type deploymentHandlerFunc func(ctx *workflow.Context, project *mdbv1.AtlasProject, deployment *mdbv1.AtlasDeployment, req reconcile.Request) (workflow.Result, error) +type deploymentHandlerFunc func(workflowCtx *workflow.Context, project *mdbv1.AtlasProject, deployment *mdbv1.AtlasDeployment, req reconcile.Request) (workflow.Result, error) + +type atlasClusterType int + +const ( + Unset atlasClusterType = iota + Advanced + Serverless +) + +type atlasTypedCluster struct { + clusterType atlasClusterType + serverless *mongodbatlas.Cluster + advanced *mongodbatlas.AdvancedCluster +} + +func managedByAtlas(ctx context.Context, atlasClient mongodbatlas.Client, projectID string, log *zap.SugaredLogger) customresource.AtlasChecker { + return func(resource mdbv1.AtlasCustomResource) (bool, error) { + deployment, ok := resource.(*mdbv1.AtlasDeployment) + if !ok { + return false, errors.New("failed to match resource type as AtlasDeployment") + } + + typedAtlasCluster, err := findTypedAtlasCluster(ctx, atlasClient, projectID, deployment.GetDeploymentName()) + if typedAtlasCluster == nil || err != nil { + return false, err + } + + isSame, err := deploymentMatchesSpec(log, typedAtlasCluster, deployment) + if err != nil { + return true, err + } + + return !isSame, nil + } +} + +func findTypedAtlasCluster(ctx context.Context, atlasClient mongodbatlas.Client, projectID, deploymentName string) (*atlasTypedCluster, error) { + advancedCluster, _, err := atlasClient.AdvancedClusters.Get(ctx, projectID, deploymentName) + if err == nil { + return &atlasTypedCluster{clusterType: Advanced, advanced: advancedCluster}, nil + } + var apiError *mongodbatlas.ErrorResponse + if errors.As(err, &apiError) && + apiError.ErrorCode != atlas.ClusterNotFound && + apiError.ErrorCode != atlas.ServerlessInstanceFromClusterAPI { + return nil, err + } + // if not found, maybe it is a serverless instead + serverless, _, err := atlasClient.ServerlessInstances.Get(ctx, projectID, deploymentName) + if err == nil { + return &atlasTypedCluster{clusterType: Serverless, serverless: serverless}, nil + } + if errors.As(err, &apiError) && apiError.ErrorCode == atlas.ServerlessInstanceNotFound { + return nil, nil + } + return nil, err +} + +func deploymentMatchesSpec(log *zap.SugaredLogger, atlasSpec *atlasTypedCluster, deployment *mdbv1.AtlasDeployment) (bool, error) { + if deployment.IsServerless() { + if atlasSpec.clusterType != Serverless { + return false, nil + } + return serverlessDeploymentMatchesSpec(log, atlasSpec.serverless, deployment.Spec.ServerlessSpec) + } + if atlasSpec.clusterType != Advanced { + return false, nil + } + return advancedDeploymentMatchesSpec(log, atlasSpec.advanced, deployment.Spec.AdvancedDeploymentSpec) +} + +func serverlessDeploymentMatchesSpec(log *zap.SugaredLogger, atlasSpec *mongodbatlas.Cluster, operatorSpec *mdbv1.ServerlessSpec) (bool, error) { + clusterMerged := mongodbatlas.Cluster{} + if err := compat.JSONCopy(&clusterMerged, atlasSpec); err != nil { + return false, err + } + + if err := compat.JSONCopy(&clusterMerged, operatorSpec); err != nil { + return false, err + } + + d := cmp.Diff(atlasSpec, &clusterMerged, cmpopts.EquateEmpty()) + if d != "" { + log.Debugf("Serverless deployment differs from spec: %s", d) + } + + return d == "", nil +} + +func advancedDeploymentMatchesSpec(log *zap.SugaredLogger, atlasSpec *mongodbatlas.AdvancedCluster, operatorSpec *mdbv1.AdvancedDeploymentSpec) (bool, error) { + clusterMerged := mongodbatlas.AdvancedCluster{} + if err := compat.JSONCopy(&clusterMerged, atlasSpec); err != nil { + return false, err + } + + if err := compat.JSONCopy(&clusterMerged, operatorSpec); err != nil { + return false, err + } + + d := cmp.Diff(atlasSpec, &clusterMerged, cmpopts.EquateEmpty()) + if d != "" { + log.Debugf("Advanced deployment differs from spec: %s", d) + } + + return d == "", nil +} diff --git a/pkg/controller/atlasdeployment/atlasdeployment_controller_test.go b/pkg/controller/atlasdeployment/atlasdeployment_controller_test.go new file mode 100644 index 0000000000..1123c8dd4a --- /dev/null +++ b/pkg/controller/atlasdeployment/atlasdeployment_controller_test.go @@ -0,0 +1,537 @@ +/* +Copyright 2023 MongoDB. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package atlasdeployment + +import ( + "context" + "log" + "regexp" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mongodb.org/atlas/mongodbatlas" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1 "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/status" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/controller/atlas" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/controller/customresource" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/controller/workflow" +) + +const ( + fakeDomain = "atlas-unit-test.local" + fakeProject = "test-project" + fakeProjectID = "fake-test-project-id" + fakeDeployment = "fake-cluster" + fakeNamespace = "fake-namespace" +) + +func TestDeploymentManaged(t *testing.T) { + testCases := []struct { + title string + protected bool + managedTag bool + }{ + { + title: "unprotected means managed", + protected: false, + managedTag: false, + }, + { + title: "protected and tagged means managed", + protected: true, + managedTag: true, + }, + { + title: "protected not tagged and missing in Atlas means managed", + protected: true, + managedTag: false, + }, + } + for _, tc := range testCases { + t.Run(tc.title, func(t *testing.T) { + atlasClient := mongodbatlas.Client{ + AdvancedClusters: &advancedClustersClientMock{ + GetFn: func(groupID string, clusterName string) (*mongodbatlas.AdvancedCluster, *mongodbatlas.Response, error) { + return nil, nil, &mongodbatlas.ErrorResponse{ErrorCode: atlas.ClusterNotFound} + }, + }, + ServerlessInstances: &serverlessClientMock{ + GetFn: func(groupID string, name string) (*mongodbatlas.Cluster, *mongodbatlas.Response, error) { + return nil, nil, &mongodbatlas.ErrorResponse{ErrorCode: atlas.ServerlessInstanceNotFound} + }, + }, + } + project := testProject(fakeNamespace) + deployment := v1.NewDeployment(project.Namespace, fakeDeployment, fakeDeployment) + te := newTestDeploymentEnv(t, tc.protected, atlasClient, testK8sClient(), project, deployment) + if tc.managedTag { + customresource.SetAnnotation(te.deployment, customresource.AnnotationLastAppliedConfiguration, "") + } + + result := te.reconciler.checkDeploymentIsManaged(te.workflowCtx, te.context, te.log, te.project, te.deployment) + + assert.True(t, result.IsOk()) + }) + } +} + +func TestProtectedAdvancedDeploymentManagedInAtlas(t *testing.T) { + testCases := []struct { + title string + inAtlas *mongodbatlas.AdvancedCluster + expectedErr string + }{ + { + title: "advanced deployment not tagged and same in Atlas STILL means managed", + inAtlas: sameAdvancedDeployment(fakeDomain), + expectedErr: "", + }, + { + title: "advanced deployment not tagged and different in Atlas means unmanaged", + inAtlas: differentAdvancedDeployment(fakeDomain), + expectedErr: "unable to reconcile Deployment due to deletion protection being enabled. see https://dochub.mongodb.org/core/ako-deletion-protection for further information", + }, + } + for _, tc := range testCases { + t.Run(tc.title, func(t *testing.T) { + protected := true + project := testProject(fakeNamespace) + atlasClient := mongodbatlas.Client{ + AdvancedClusters: &advancedClustersClientMock{ + GetFn: func(groupID string, clusterName string) (*mongodbatlas.AdvancedCluster, *mongodbatlas.Response, error) { + return tc.inAtlas, nil, nil + }, + }, + } + deployment := v1.NewDeployment(project.Namespace, fakeDeployment, fakeDeployment) + te := newTestDeploymentEnv(t, protected, atlasClient, testK8sClient(), project, deployment) + + result := te.reconciler.checkDeploymentIsManaged(te.workflowCtx, te.context, te.log, te.project, te.deployment) + + if tc.expectedErr == "" { + assert.True(t, result.IsOk()) + } else { + assert.Regexp(t, regexp.MustCompile(tc.expectedErr), result.GetMessage()) + } + }) + } +} + +func TestProtectedServerlessManagedInAtlas(t *testing.T) { + testCases := []struct { + title string + inAtlas *mongodbatlas.Cluster + expectedErr string + }{ + { + title: "serverless deployment not tagged and same in Atlas STILL means managed", + inAtlas: sameServerlessDeployment(fakeDomain), + expectedErr: "", + }, + { + title: "serverless deployment not tagged and different in Atlas means unmanaged", + inAtlas: differentServerlessDeployment(fakeDomain), + expectedErr: "unable to reconcile Deployment due to deletion protection being enabled. see https://dochub.mongodb.org/core/ako-deletion-protection for further information", + }, + } + for _, tc := range testCases { + t.Run(tc.title, func(t *testing.T) { + protected := true + project := testProject(fakeNamespace) + atlasClient := mongodbatlas.Client{ + AdvancedClusters: &advancedClustersClientMock{ + GetFn: func(groupID string, clusterName string) (*mongodbatlas.AdvancedCluster, *mongodbatlas.Response, error) { + return nil, nil, &mongodbatlas.ErrorResponse{ErrorCode: atlas.ServerlessInstanceFromClusterAPI} + }, + }, + ServerlessInstances: &serverlessClientMock{ + GetFn: func(groupID string, name string) (*mongodbatlas.Cluster, *mongodbatlas.Response, error) { + return tc.inAtlas, nil, nil + }, + }, + } + deployment := v1.NewDefaultAWSServerlessInstance(project.Namespace, project.Name) + te := newTestDeploymentEnv(t, protected, atlasClient, testK8sClient(), project, deployment) + + result := te.reconciler.checkDeploymentIsManaged(te.workflowCtx, te.context, te.log, te.project, te.deployment) + + if tc.expectedErr == "" { + assert.True(t, result.IsOk()) + } else { + assert.Regexp(t, regexp.MustCompile(tc.expectedErr), result.GetMessage()) + } + }) + } +} + +func TestFinalizerNotFound(t *testing.T) { + protected := false + atlasClient := mongodbatlas.Client{} + project := testProject(fakeNamespace) + deployment := v1.NewDeployment(project.Namespace, fakeDeployment, fakeDeployment) + k8sclient := testK8sClient() + te := newTestDeploymentEnv(t, protected, atlasClient, k8sclient, project, deployment) + + deletionRequest, result := te.reconciler.handleDeletion( + te.workflowCtx, + te.context, + te.log, + te.prevResult, + te.project, + te.deployment, + ) + + require.True(t, deletionRequest) + assert.Regexp(t, regexp.MustCompile("not found"), result.GetMessage()) +} + +func TestFinalizerGetsSet(t *testing.T) { + testCases := []struct { + title string + haveFinalizer bool + }{ + { + title: "with a finalizer, it remains set", + haveFinalizer: true, + }, + { + title: "without a finalizer, it gets set", + haveFinalizer: false, + }, + } + for _, tc := range testCases { + t.Run(tc.title, func(t *testing.T) { + protected := false + atlasClient := mongodbatlas.Client{} + project := testProject(fakeNamespace) + deployment := v1.NewDeployment(project.Namespace, fakeDeployment, fakeDeployment) + if tc.haveFinalizer { + customresource.SetFinalizer(deployment, customresource.FinalizerLabel) + } + k8sclient := testK8sClient() + require.NoError(t, k8sclient.Create(context.Background(), deployment)) + te := newTestDeploymentEnv(t, protected, atlasClient, k8sclient, project, deployment) + + deletionRequest, _ := te.reconciler.handleDeletion( + te.workflowCtx, + te.context, + te.log, + te.prevResult, + te.project, + te.deployment, + ) + + require.False(t, deletionRequest) + finalDeployment := &v1.AtlasDeployment{} + require.NoError(t, te.reconciler.Client.Get(context.Background(), client.ObjectKeyFromObject(te.deployment), finalDeployment)) + assert.True(t, customresource.HaveFinalizer(finalDeployment, customresource.FinalizerLabel)) + }) + } +} + +func TestDeploymentDeletionProtection(t *testing.T) { + testCases := []struct { + title string + protected bool + expectRemoval bool + }{ + { + title: "Deployment with protection ON and no annotations is kept", + protected: true, + expectRemoval: false, + }, + { + title: "Deployment with protection OFF and no annotations is removed", + protected: false, + expectRemoval: true, + }, + } + for _, tc := range testCases { + t.Run(tc.title, func(t *testing.T) { + project := testProject(fakeNamespace) + called := false + atlasClient := mongodbatlas.Client{ + AdvancedClusters: &advancedClustersClientMock{ + DeleteFn: func(groupID string, clusterName string, options *mongodbatlas.DeleteAdvanceClusterOptions) (*mongodbatlas.Response, error) { + called = true + return nil, nil + }, + }, + } + deployment := v1.NewDeployment(project.Namespace, fakeDeployment, fakeDeployment) + deployment.SetDeletionTimestamp(&metav1.Time{Time: time.Now()}) + k8sclient := testK8sClient() + customresource.SetFinalizer(deployment, customresource.FinalizerLabel) + require.NoError(t, k8sclient.Create(context.Background(), deployment)) + te := newTestDeploymentEnv(t, tc.protected, atlasClient, k8sclient, project, deployment) + + deletionRequest, result := te.reconciler.handleDeletion( + te.workflowCtx, + te.context, + te.log, + te.prevResult, + te.project, + te.deployment, + ) + + require.True(t, deletionRequest) + require.True(t, result.IsOk()) + assert.Equal(t, tc.expectRemoval, called) + }) + } +} + +func TestKeepAnnotatedDeploymentAlwaysRemain(t *testing.T) { + testCases := []struct { + title string + protected bool + }{ + { + title: "Deployment with protection ON and 'keep' annotation is kept", + protected: true, + }, + { + title: "Deployment with protection OFF but 'keep' annotation is kept", + protected: false, + }, + } + for _, tc := range testCases { + t.Run(tc.title, func(t *testing.T) { + project := testProject(fakeNamespace) + called := false + atlasClient := mongodbatlas.Client{ + AdvancedClusters: &advancedClustersClientMock{ + DeleteFn: func(groupID string, clusterName string, options *mongodbatlas.DeleteAdvanceClusterOptions) (*mongodbatlas.Response, error) { + called = true + return nil, nil + }, + }, + } + deployment := v1.NewDeployment(project.Namespace, fakeDeployment, fakeDeployment) + deployment.SetDeletionTimestamp(&metav1.Time{Time: time.Now()}) + customresource.SetAnnotation(deployment, + customresource.ResourcePolicyAnnotation, + customresource.ResourcePolicyKeep, + ) + k8sclient := testK8sClient() + customresource.SetFinalizer(deployment, customresource.FinalizerLabel) + require.NoError(t, k8sclient.Create(context.Background(), deployment)) + te := newTestDeploymentEnv(t, tc.protected, atlasClient, k8sclient, project, deployment) + + deletionRequest, result := te.reconciler.handleDeletion( + te.workflowCtx, + te.context, + te.log, + te.prevResult, + te.project, + te.deployment, + ) + + require.True(t, deletionRequest) + require.True(t, result.IsOk()) + assert.Equal(t, false, called) + }) + } +} + +func TestDeleteAnnotatedDeploymentGetRemoved(t *testing.T) { + testCases := []struct { + title string + protected bool + }{ + { + title: "Deployment with protection ON but 'delete' annotation is removed", + protected: true, + }, + { + title: "Deployment with protection OFF and 'delete' annotation is removed", + protected: false, + }, + } + for _, tc := range testCases { + t.Run(tc.title, func(t *testing.T) { + project := testProject(fakeNamespace) + called := false + atlasClient := mongodbatlas.Client{ + AdvancedClusters: &advancedClustersClientMock{ + DeleteFn: func(groupID string, clusterName string, options *mongodbatlas.DeleteAdvanceClusterOptions) (*mongodbatlas.Response, error) { + called = true + return nil, nil + }, + }, + } + deployment := v1.NewDeployment(project.Namespace, fakeDeployment, fakeDeployment) + deployment.SetDeletionTimestamp(&metav1.Time{Time: time.Now()}) + customresource.SetAnnotation(deployment, + customresource.ResourcePolicyAnnotation, + customresource.ResourcePolicyDelete, + ) + k8sclient := testK8sClient() + customresource.SetFinalizer(deployment, customresource.FinalizerLabel) + require.NoError(t, k8sclient.Create(context.Background(), deployment)) + te := newTestDeploymentEnv(t, tc.protected, atlasClient, k8sclient, project, deployment) + + deletionRequest, result := te.reconciler.handleDeletion( + te.workflowCtx, + te.context, + te.log, + te.prevResult, + te.project, + te.deployment, + ) + + require.True(t, deletionRequest) + require.True(t, result.IsOk()) + assert.Equal(t, true, called) + }) + } +} + +func differentAdvancedDeployment(ns string) *mongodbatlas.AdvancedCluster { + project := testProject(ns) + deployment := v1.NewDeployment(project.Namespace, fakeDeployment, fakeDeployment) + deployment.Spec.DeploymentSpec.ProviderSettings.InstanceSizeName = "M2" + advancedSpec := asAdvanced(deployment).Spec.AdvancedDeploymentSpec + return intoAdvancedAtlasCluster(advancedSpec) +} + +func sameAdvancedDeployment(ns string) *mongodbatlas.AdvancedCluster { + project := testProject(ns) + deployment := asAdvanced(v1.NewDeployment(project.Namespace, fakeDeployment, fakeDeployment)) + advancedSpec := asAdvanced(deployment).Spec.AdvancedDeploymentSpec + return intoAdvancedAtlasCluster(advancedSpec) +} + +func differentServerlessDeployment(ns string) *mongodbatlas.Cluster { + project := testProject(ns) + deployment := v1.NewDefaultAWSServerlessInstance(project.Namespace, project.Name) + deployment.Spec.ServerlessSpec.ProviderSettings.RegionName = "US_EAST_2" + return intoServerlessAtlasCluster(deployment.Spec.ServerlessSpec) +} + +func sameServerlessDeployment(ns string) *mongodbatlas.Cluster { + project := testProject(ns) + deployment := v1.NewDefaultAWSServerlessInstance(project.Namespace, project.Name) + return intoServerlessAtlasCluster(deployment.Spec.ServerlessSpec) +} + +type testDeploymentEnv struct { + reconciler *AtlasDeploymentReconciler + workflowCtx *workflow.Context + context context.Context + log *zap.SugaredLogger + prevResult workflow.Result + project *v1.AtlasProject + deployment *v1.AtlasDeployment +} + +func newTestDeploymentEnv(t *testing.T, + protected bool, + atlasClient mongodbatlas.Client, + k8sclient client.Client, + project *v1.AtlasProject, + deployment *v1.AtlasDeployment, +) *testDeploymentEnv { + t.Helper() + + log := testLog(t) + r := testDeploymentReconciler(log, k8sclient, protected) + + prevResult := testPrevResult() + workflowCtx := customresource.MarkReconciliationStarted(r.Client, deployment, log) + workflowCtx.Client = atlasClient + return &testDeploymentEnv{ + reconciler: r, + workflowCtx: workflowCtx, + context: context.Background(), + log: r.Log.With("atlasdeployment", "test-namespace"), + prevResult: prevResult, + deployment: deployment, + project: project, + } +} + +func testK8sClient() client.Client { + sch := runtime.NewScheme() + sch.AddKnownTypes(corev1.SchemeGroupVersion, &corev1.SecretList{}) + sch.AddKnownTypes(v1.GroupVersion, &v1.AtlasDeployment{}) + return fake.NewClientBuilder().WithScheme(sch).Build() +} + +func testLog(t *testing.T) *zap.SugaredLogger { + t.Helper() + + return zaptest.NewLogger(t).Sugar() +} + +func testPrevResult() workflow.Result { + return workflow.Result{}.WithMessage("unchanged") +} + +func testDeploymentReconciler(log *zap.SugaredLogger, k8sclient client.Client, protected bool) *AtlasDeploymentReconciler { + return &AtlasDeploymentReconciler{ + Client: k8sclient, + Log: log, + ObjectDeletionProtection: protected, + } +} + +func testProject(ns string) *v1.AtlasProject { + return &v1.AtlasProject{ + ObjectMeta: metav1.ObjectMeta{ + Name: fakeProject, + Namespace: ns, + }, + Status: status.AtlasProjectStatus{ + ID: fakeProjectID, + }, + } +} + +func asAdvanced(deployment *v1.AtlasDeployment) *v1.AtlasDeployment { + if err := ConvertLegacyDeployment(&deployment.Spec); err != nil { + log.Fatalf("failed to convert legacy deployment: %v", err) + } + deployment.Spec.DeploymentSpec = nil + return deployment +} + +func intoAdvancedAtlasCluster(advancedSpec *v1.AdvancedDeploymentSpec) *mongodbatlas.AdvancedCluster { + ac, err := advancedSpec.ToAtlas() + if err != nil { + log.Fatalf("failed to convert advanced deployment to atlas: %v", err) + } + return ac +} + +func intoServerlessAtlasCluster(serverlessSpec *v1.ServerlessSpec) *mongodbatlas.Cluster { + ac, err := serverlessSpec.ToAtlas() + if err != nil { + log.Fatalf("failed to convert serverless deployment to atlas: %v", err) + } + return ac +} diff --git a/pkg/controller/atlasdeployment/clusters_mock_test.go b/pkg/controller/atlasdeployment/clusters_mock_test.go new file mode 100644 index 0000000000..32aa25825e --- /dev/null +++ b/pkg/controller/atlasdeployment/clusters_mock_test.go @@ -0,0 +1,42 @@ +package atlasdeployment + +import ( + "context" + + "go.mongodb.org/atlas/mongodbatlas" +) + +type advancedClustersClientMock struct { + GetFn func(groupID string, clusterName string) (*mongodbatlas.AdvancedCluster, *mongodbatlas.Response, error) + DeleteFn func(groupID string, clusterName string, options *mongodbatlas.DeleteAdvanceClusterOptions) (*mongodbatlas.Response, error) +} + +func (ac *advancedClustersClientMock) List(ctx context.Context, groupID string, options *mongodbatlas.ListOptions) (*mongodbatlas.AdvancedClustersResponse, *mongodbatlas.Response, error) { + panic("not implemented") // TODO: Implement +} + +func (ac *advancedClustersClientMock) Get(_ context.Context, groupID string, clusterName string) (*mongodbatlas.AdvancedCluster, *mongodbatlas.Response, error) { + if ac.GetFn == nil { + panic("GetFn not mocked for test") + } + return ac.GetFn(groupID, clusterName) +} + +func (ac *advancedClustersClientMock) Create(ctx context.Context, groupID string, cluster *mongodbatlas.AdvancedCluster) (*mongodbatlas.AdvancedCluster, *mongodbatlas.Response, error) { + panic("not implemented") // TODO: Implement +} + +func (ac *advancedClustersClientMock) Update(ctx context.Context, groupID string, clusterName string, cluster *mongodbatlas.AdvancedCluster) (*mongodbatlas.AdvancedCluster, *mongodbatlas.Response, error) { + panic("not implemented") // TODO: Implement +} + +func (ac *advancedClustersClientMock) Delete(ctx context.Context, groupID string, clusterName string, options *mongodbatlas.DeleteAdvanceClusterOptions) (*mongodbatlas.Response, error) { + if ac.DeleteFn == nil { + panic("GetFn not mocked for test") + } + return ac.DeleteFn(groupID, clusterName, options) +} + +func (ac *advancedClustersClientMock) TestFailover(ctx context.Context, groupID string, clusterName string) (*mongodbatlas.Response, error) { + panic("not implemented") // TODO: Implement +} diff --git a/pkg/controller/atlasdeployment/serverless_mock_test.go b/pkg/controller/atlasdeployment/serverless_mock_test.go new file mode 100644 index 0000000000..22b2b2fbda --- /dev/null +++ b/pkg/controller/atlasdeployment/serverless_mock_test.go @@ -0,0 +1,34 @@ +package atlasdeployment + +import ( + "context" + + "go.mongodb.org/atlas/mongodbatlas" +) + +type serverlessClientMock struct { + GetFn func(groupID string, name string) (*mongodbatlas.Cluster, *mongodbatlas.Response, error) +} + +func (sc *serverlessClientMock) List(_ context.Context, _ string, _ *mongodbatlas.ListOptions) (*mongodbatlas.ClustersResponse, *mongodbatlas.Response, error) { + panic("not implemented") // TODO: Implement +} + +func (sc *serverlessClientMock) Get(_ context.Context, groupID string, name string) (*mongodbatlas.Cluster, *mongodbatlas.Response, error) { + if sc.GetFn == nil { + panic("GetFn not mocked for test") + } + return sc.GetFn(groupID, name) +} + +func (sc *serverlessClientMock) Create(_ context.Context, _ string, _ *mongodbatlas.ServerlessCreateRequestParams) (*mongodbatlas.Cluster, *mongodbatlas.Response, error) { + panic("not implemented") // TODO: Implement +} + +func (sc *serverlessClientMock) Update(_ context.Context, _ string, _ string, _ *mongodbatlas.ServerlessUpdateRequestParams) (*mongodbatlas.Cluster, *mongodbatlas.Response, error) { + panic("not implemented") // TODO: Implement +} + +func (sc *serverlessClientMock) Delete(_ context.Context, _ string, _ string) (*mongodbatlas.Response, error) { + panic("not implemented") // TODO: Implement +} diff --git a/pkg/controller/atlasproject/atlasproject_controller.go b/pkg/controller/atlasproject/atlasproject_controller.go index 4ab5b5f668..f3cff82c48 100644 --- a/pkg/controller/atlasproject/atlasproject_controller.go +++ b/pkg/controller/atlasproject/atlasproject_controller.go @@ -227,7 +227,6 @@ func (r *AtlasProjectReconciler) ensureDeletionFinalizer(ctx context.Context, wo if !project.GetDeletionTimestamp().IsZero() { if customresource.HaveFinalizer(project, customresource.FinalizerLabel) { - log.Infow("RESOURCE PROTECTED", r.ObjectDeletionProtection, customresource.IsResourceProtected(project, r.ObjectDeletionProtection)) if customresource.IsResourceProtected(project, r.ObjectDeletionProtection) { log.Info("Not removing Atlas database user from Atlas as per configuration") } else { diff --git a/test/e2e/annotations_test.go b/test/e2e/annotations_test.go index 03b895bce8..84a71c92ea 100644 --- a/test/e2e/annotations_test.go +++ b/test/e2e/annotations_test.go @@ -31,6 +31,7 @@ var _ = Describe("Annotations base test.", Label("deployment-annotations-ns"), f testData = test mainCycle(test) }, + // TODO: fix test for deletion protection on, as it would fail to re-take the cluster after deletion Entry("Simple configuration with keep resource policy annotation on deployment", Label("ns-crd"), model.DataProvider( "operator-ns-crd", diff --git a/test/int/databaseuser_protected_test.go b/test/int/databaseuser_protected_test.go index 9b009747a3..8d027128b8 100644 --- a/test/int/databaseuser_protected_test.go +++ b/test/int/databaseuser_protected_test.go @@ -58,6 +58,11 @@ var _ = Describe("Atlas Database User", Label("int", "AtlasDatabaseUser", "prote By("Creating a deployment", func() { testDeployment = mdbv1.DefaultAWSDeployment(testNamespace.Name, projectName).Lightweight() + customresource.SetAnnotation( // this test deployment must be deleted + testDeployment, + customresource.ResourcePolicyAnnotation, + customresource.ResourcePolicyDelete, + ) Expect(k8sClient.Create(context.TODO(), testDeployment)).To(Succeed()) Eventually(func() bool { diff --git a/test/int/deployment_protected_test.go b/test/int/deployment_protected_test.go new file mode 100644 index 0000000000..5064b4ed76 --- /dev/null +++ b/test/int/deployment_protected_test.go @@ -0,0 +1,130 @@ +package int + +import ( + "context" + "fmt" + "time" + + mdbv1 "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/project" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/status" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/controller/customresource" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/kube" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/testutil" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("AtlasDeployment Deletion Protected", + Ordered, + Label("AtlasDeployment", "deletion-protection", "deployment-deletion-protected"), func() { + var testNamespace *corev1.Namespace + var stopManager context.CancelFunc + var connectionSecret corev1.Secret + var testProject *mdbv1.AtlasProject + + BeforeAll(func() { + By("Starting the operator with protection ON", func() { + testNamespace, stopManager = prepareControllers(true) + Expect(testNamespace).ToNot(BeNil()) + Expect(stopManager).ToNot(BeNil()) + }) + + By("Creating project connection secret", func() { + connectionSecret = buildConnectionSecret(fmt.Sprintf("%s-atlas-key", testNamespace.Name)) + Expect(k8sClient.Create(context.Background(), &connectionSecret)).To(Succeed()) + }) + + By("Creating a project with deletion annotation", func() { + testProject = mdbv1.DefaultProject(testNamespace.Name, connectionSecret.Name).WithIPAccessList(project.NewIPAccessList().WithCIDR("0.0.0.0/0")) + customresource.SetAnnotation( // this test project must be deleted + testProject, + customresource.ResourcePolicyAnnotation, + customresource.ResourcePolicyDelete, + ) + Expect(k8sClient.Create(context.TODO(), testProject, &client.CreateOptions{})).To(Succeed()) + + Eventually(func() bool { + return testutil.CheckCondition(k8sClient, testProject, status.TrueCondition(status.ReadyType)) + }).WithTimeout(3 * time.Minute).WithPolling(PollingInterval).Should(BeTrue()) + }) + }) + + AfterAll(func() { + By("Deleting project from k8s and atlas", func() { + Expect(k8sClient.Delete(context.TODO(), testProject, &client.DeleteOptions{})).To(Succeed()) + Eventually( + checkAtlasProjectRemoved(testProject.Status.ID), + ).WithTimeout(3 * time.Minute).WithPolling(PollingInterval).Should(BeTrue()) + }) + + By("Deleting project connection secret", func() { + Expect(k8sClient.Delete(context.Background(), &connectionSecret)).To(Succeed()) + }) + + By("Stopping the operator", func() { + stopManager() + err := k8sClient.Delete(context.Background(), testNamespace) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + It("removing advanced cluster from Kubernetes when protection is ON leaves it in Atlas", + Label("preserving-advanced-cluster"), + func() { + testDeployment := mdbv1.DefaultAWSDeployment(testNamespace.Name, testProject.Name).Lightweight() + preserveDeploymentFlow(testNamespace.Name, testProject, testDeployment) + }, + ) + + It("removing serverless instance from Kubernetes when protection is ON leaves it in Atlas", + Label("preserving-serverless-instance"), + func() { + testDeployment := mdbv1.NewDefaultAWSServerlessInstance(testNamespace.Name, testProject.Name) + preserveDeploymentFlow(testNamespace.Name, testProject, testDeployment) + }, + ) + }, +) + +func preserveDeploymentFlow(ns string, testProject *mdbv1.AtlasProject, testDeployment *mdbv1.AtlasDeployment) { + By("Creating deployment in Kubernetes", func() { + Expect(k8sClient.Create(context.TODO(), testDeployment, &client.CreateOptions{})).To(Succeed()) + }) + + By("Waiting the deployment to settle in Kubernetes", func() { + Eventually(func(g Gomega) bool { + return testutil.CheckCondition(k8sClient, testDeployment, status.TrueCondition(status.ReadyType), validateDeploymentUpdatingFunc(g)) + }).WithTimeout(30 * time.Minute).WithPolling(PollingInterval).Should(BeTrue()) + }) + + By("Deleting the deployment from Kubernetes", func() { + Expect(k8sClient.Delete(context.TODO(), testDeployment, &client.DeleteOptions{})).To(Succeed()) + Eventually(func() bool { + deployment := mdbv1.AtlasDeployment{} + err := k8sClient.Get(context.TODO(), kube.ObjectKey(ns, testDeployment.Name), &deployment, &client.GetOptions{}) + return k8serrors.IsNotFound(err) + }).WithTimeout(5 * time.Minute).WithPolling(PollingInterval).Should(BeTrue()) + }) + + By("Checking the Atlas deployment was NOT removed", func() { + if testDeployment.IsServerless() { + Expect(checkAtlasServerlessInstanceRemoved(testProject.Status.ID, testDeployment.Spec.ServerlessSpec.Name)()).To(BeFalse()) + return + } + Expect(checkAtlasDeploymentRemoved(testProject.Status.ID, testDeployment.Spec.DeploymentSpec.Name)()).To(BeFalse()) + }) + + By("Making sure deployment gets removed from Atlas manually", func() { + if testDeployment.IsServerless() { + Expect(deleteServerlessInstance(testProject.Status.ID, testDeployment.Spec.ServerlessSpec.Name)).ToNot(HaveOccurred()) + return + } + Expect(deleteAtlasDeployment(testProject.Status.ID, testDeployment.Spec.DeploymentSpec.Name)).ToNot(HaveOccurred()) + }) +} diff --git a/test/int/deployment_test.go b/test/int/deployment_test.go index 3fea7fd9be..024b0943fd 100644 --- a/test/int/deployment_test.go +++ b/test/int/deployment_test.go @@ -1291,11 +1291,29 @@ func checkAtlasDeploymentRemoved(projectID string, deploymentName string) func() } } +func checkAtlasServerlessInstanceRemoved(projectID string, deploymentName string) func() bool { + return func() bool { + _, r, err := atlasClient.ServerlessInstances.Get(context.Background(), projectID, deploymentName) + if err != nil { + if r != nil && r.StatusCode == http.StatusNotFound { + return true + } + } + + return false + } +} + func deleteAtlasDeployment(projectID string, deploymentName string) error { _, err := atlasClient.AdvancedClusters.Delete(context.Background(), projectID, deploymentName, nil) return err } +func deleteServerlessInstance(projectID string, deploymentName string) error { + _, err := atlasClient.ServerlessInstances.Delete(context.Background(), projectID, deploymentName) + return err +} + func int64ptr(i int64) *int64 { return &i } diff --git a/test/int/deployment_unprotected_test.go b/test/int/deployment_unprotected_test.go new file mode 100644 index 0000000000..ebb8179551 --- /dev/null +++ b/test/int/deployment_unprotected_test.go @@ -0,0 +1,121 @@ +package int + +import ( + "context" + "fmt" + "time" + + mdbv1 "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/project" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/status" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/kube" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/testutil" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("AtlasDeployment Deletion Unprotected", + Ordered, + Label("AtlasDeployment", "deletion-protection", "deployment-deletion-unprotected"), func() { + var testNamespace *corev1.Namespace + var stopManager context.CancelFunc + var connectionSecret corev1.Secret + var testProject *mdbv1.AtlasProject + + BeforeAll(func() { + By("Starting the operator with protection OFF", func() { + testNamespace, stopManager = prepareControllers(false) + Expect(testNamespace).ToNot(BeNil()) + Expect(stopManager).ToNot(BeNil()) + }) + + By("Creating project connection secret", func() { + connectionSecret = buildConnectionSecret(fmt.Sprintf("%s-atlas-key", testNamespace.Name)) + Expect(k8sClient.Create(context.Background(), &connectionSecret)).To(Succeed()) + }) + + By("Creating a project", func() { + testProject = mdbv1.DefaultProject(testNamespace.Name, connectionSecret.Name).WithIPAccessList(project.NewIPAccessList().WithCIDR("0.0.0.0/0")) + Expect(k8sClient.Create(context.TODO(), testProject, &client.CreateOptions{})).To(Succeed()) + + Eventually(func() bool { + return testutil.CheckCondition(k8sClient, testProject, status.TrueCondition(status.ReadyType)) + }).WithTimeout(3 * time.Minute).WithPolling(PollingInterval).Should(BeTrue()) + }) + }) + + AfterAll(func() { + By("Deleting project from k8s and atlas", func() { + Expect(k8sClient.Delete(context.TODO(), testProject, &client.DeleteOptions{})).To(Succeed()) + Eventually( + checkAtlasProjectRemoved(testProject.Status.ID), + ).WithTimeout(3 * time.Minute).WithPolling(PollingInterval).Should(BeTrue()) + }) + + By("Deleting project connection secret", func() { + Expect(k8sClient.Delete(context.Background(), &connectionSecret)).To(Succeed()) + }) + + By("Stopping the operator", func() { + stopManager() + err := k8sClient.Delete(context.Background(), testNamespace) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + It("removing advanced cluster from Kubernetes when protection is OFF wipes it from Atlas", + Label("wiping-advanced-cluster"), + func() { + testDeployment := mdbv1.DefaultAWSDeployment(testNamespace.Name, testProject.Name).Lightweight() + wipeDeploymentFlow(testNamespace.Name, testProject, testDeployment) + }, + ) + + It("removing serverless instance from Kubernetes when protection is OFF wipes it from Atlas", + Label("wiping-serverless-instance"), + func() { + testDeployment := mdbv1.NewDefaultAWSServerlessInstance(testNamespace.Name, testProject.Name) + wipeDeploymentFlow(testNamespace.Name, testProject, testDeployment) + }, + ) + }, +) + +func wipeDeploymentFlow(ns string, testProject *mdbv1.AtlasProject, testDeployment *mdbv1.AtlasDeployment) { + By("Creating a deployment in the cluster with annotation set to delete", func() { + testDeployment = mdbv1.DefaultAWSDeployment(ns, testProject.Name).Lightweight() + Expect(k8sClient.Create(context.TODO(), testDeployment, &client.CreateOptions{})).To(Succeed()) + }) + + By("Waiting the deployment to settle in kubernetes", func() { + Eventually(func(g Gomega) bool { + return testutil.CheckCondition(k8sClient, testDeployment, status.TrueCondition(status.ReadyType), validateDeploymentUpdatingFunc(g)) + }).WithTimeout(30 * time.Minute).WithPolling(PollingInterval).Should(BeTrue()) + }) + + By("Deleting the deployment from Kubernetes", func() { + Expect(k8sClient.Delete(context.TODO(), testDeployment, &client.DeleteOptions{})).To(Succeed()) + Eventually(func() bool { + deployment := mdbv1.AtlasDeployment{} + err := k8sClient.Get(context.TODO(), kube.ObjectKey(ns, testDeployment.Name), &deployment, &client.GetOptions{}) + return k8serrors.IsNotFound(err) + }).WithTimeout(2 * time.Minute).WithPolling(PollingInterval).Should(BeTrue()) + }) + + By("Checking whether the Atlas deployment got also removed", func() { + if testDeployment.IsServerless() { + Eventually( + checkAtlasServerlessInstanceRemoved(testProject.Status.ID, testDeployment.Spec.ServerlessSpec.Name), + ).WithTimeout(5 * time.Minute).WithPolling(PollingInterval).Should(BeTrue()) + return + } + Eventually( + checkAtlasDeploymentRemoved(testProject.Status.ID, testDeployment.Spec.DeploymentSpec.Name), + ).WithTimeout(5 * time.Minute).WithPolling(PollingInterval).Should(BeTrue()) + }) +} diff --git a/test/int/integration_suite_test.go b/test/int/integration_suite_test.go index 51a19254c2..797c98a3d8 100644 --- a/test/int/integration_suite_test.go +++ b/test/int/integration_suite_test.go @@ -232,13 +232,15 @@ func prepareControllers(deletionProtection bool) (*corev1.Namespace, context.Can Expect(err).ToNot(HaveOccurred()) err = (&atlasdeployment.AtlasDeploymentReconciler{ - Client: k8sManager.GetClient(), - Log: logger.Named("controllers").Named("AtlasDeployment").Sugar(), - AtlasDomain: atlasDomain, - ResourceWatcher: watch.NewResourceWatcher(), - GlobalAPISecret: kube.ObjectKey(namespace.Name, "atlas-operator-api-key"), - GlobalPredicates: globalPredicates, - EventRecorder: k8sManager.GetEventRecorderFor("AtlasDeployment"), + Client: k8sManager.GetClient(), + Log: logger.Named("controllers").Named("AtlasDeployment").Sugar(), + AtlasDomain: atlasDomain, + ResourceWatcher: watch.NewResourceWatcher(), + GlobalAPISecret: kube.ObjectKey(namespace.Name, "atlas-operator-api-key"), + GlobalPredicates: globalPredicates, + EventRecorder: k8sManager.GetEventRecorderFor("AtlasDeployment"), + ObjectDeletionProtection: deletionProtection, + SubObjectDeletionProtection: deletionProtection, }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred())