Skip to content

Commit

Permalink
Merge pull request #53 from cybozu-go/implement-secondary-mantle-inte…
Browse files Browse the repository at this point in the history 
…rface

Implement secondary Mantle interface
  • Loading branch information
@satoru-takeuchi
satoru-takeuchi authored Oct 25, 2024
2 parents d9f469e + 024e131 commit da5624d
Show file tree
Hide file tree
Showing 2 changed files with 235 additions and 27 deletions.
248 changes: 226 additions & 22 deletions internal/controller/mantlebackup_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,18 +283,10 @@ func (r *MantleBackupReconciler) expire(ctx context.Context, backup *mantlev1.Ma
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.16.3/pkg/reconcile
//
// Reconcile is the main component of mantle-controller, so let's admit that Reconcile can be complex by `nolint:gocyclo`
//
//nolint:gocyclo
func (r *MantleBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := log.FromContext(ctx)
var backup mantlev1.MantleBackup

if r.role == RoleSecondary {
return ctrl.Result{}, nil
}

err := r.Get(ctx, req.NamespacedName, &backup)
if aerrors.IsNotFound(err) {
logger.Info("MantleBackup is not found", "error", err)
Expand All @@ -305,14 +297,29 @@ func (r *MantleBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, err
}

if isCreatedWhenMantleControllerWasSecondary(&backup) {
switch r.role {
case RoleStandalone:
return r.reconcileAsStandalone(ctx, &backup)
case RolePrimary:
return r.reconcileAsPrimary(ctx, &backup)
case RoleSecondary:
return r.reconcileAsSecondary(ctx, &backup)
}

panic("unreachable")
}

func (r *MantleBackupReconciler) reconcileAsStandalone(ctx context.Context, backup *mantlev1.MantleBackup) (ctrl.Result, error) {
logger := log.FromContext(ctx)

if isCreatedWhenMantleControllerWasSecondary(backup) {
logger.Info(
"skipping to reconcile the MantleBackup created by a remote mantle-controller to prevent accidental data loss",
)
return ctrl.Result{}, nil
}

target, result, getSnapshotTargetErr := r.getSnapshotTarget(ctx, &backup)
target, result, getSnapshotTargetErr := r.getSnapshotTarget(ctx, backup)
switch {
case getSnapshotTargetErr == errSkipProcessing:
return ctrl.Result{}, nil
Expand All @@ -327,32 +334,32 @@ func (r *MantleBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request
}

if !backup.ObjectMeta.DeletionTimestamp.IsZero() {
return r.finalize(ctx, &backup, target, isErrTargetPVCNotFound(getSnapshotTargetErr))
return r.finalizeStandalone(ctx, backup, target, isErrTargetPVCNotFound(getSnapshotTargetErr))
}

if getSnapshotTargetErr != nil {
return ctrl.Result{}, getSnapshotTargetErr
}

if !controllerutil.ContainsFinalizer(&backup, MantleBackupFinalizerName) {
controllerutil.AddFinalizer(&backup, MantleBackupFinalizerName)
err = r.Update(ctx, &backup)
if err != nil {
if !controllerutil.ContainsFinalizer(backup, MantleBackupFinalizerName) {
controllerutil.AddFinalizer(backup, MantleBackupFinalizerName)

if err := r.Update(ctx, backup); err != nil {
logger.Error(err, "failed to add finalizer", "finalizer", MantleBackupFinalizerName)
return ctrl.Result{}, err
}
err := r.updateStatusCondition(ctx, &backup, metav1.Condition{Type: mantlev1.BackupConditionReadyToUse, Status: metav1.ConditionFalse, Reason: mantlev1.BackupReasonNone})
err := r.updateStatusCondition(ctx, backup, metav1.Condition{Type: mantlev1.BackupConditionReadyToUse, Status: metav1.ConditionFalse, Reason: mantlev1.BackupReasonNone})
if err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{Requeue: true}, nil
}

if err := r.expire(ctx, &backup); err != nil {
if err := r.expire(ctx, backup); err != nil {
return ctrl.Result{}, err
}

if err := r.provisionRBDSnapshot(ctx, &backup, target); err != nil {
if err := r.provisionRBDSnapshot(ctx, backup, target); err != nil {
return ctrl.Result{}, err
}

Expand All @@ -361,11 +368,61 @@ func (r *MantleBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, nil
}

if r.role == RolePrimary {
return r.replicate(ctx, &backup)
return ctrl.Result{}, nil
}

func (r *MantleBackupReconciler) reconcileAsPrimary(ctx context.Context, backup *mantlev1.MantleBackup) (ctrl.Result, error) {
result, err := r.reconcileAsStandalone(ctx, backup)
if err != nil || !result.IsZero() {
return result, err
}
return r.replicate(ctx, backup)
}

return ctrl.Result{}, nil
func (r *MantleBackupReconciler) reconcileAsSecondary(ctx context.Context, backup *mantlev1.MantleBackup) (ctrl.Result, error) {
logger := log.FromContext(ctx)

if !isCreatedWhenMantleControllerWasSecondary(backup) {
logger.Info(
"skipping to reconcile the MantleBackup created by a different mantle-controller to prevent accidental data loss",
)
return ctrl.Result{}, nil
}

target, result, getSnapshotTargetErr := r.getSnapshotTarget(ctx, backup)
switch {
case getSnapshotTargetErr == errSkipProcessing:
return ctrl.Result{}, nil
case isErrTargetPVCNotFound(getSnapshotTargetErr):
// deletion logic may run.
case getSnapshotTargetErr == nil:
default:
return ctrl.Result{}, getSnapshotTargetErr
}
if result.Requeue {
return result, nil
}

if !backup.ObjectMeta.DeletionTimestamp.IsZero() {
return r.finalizeSecondary(ctx, backup, target, isErrTargetPVCNotFound(getSnapshotTargetErr))
}

if getSnapshotTargetErr != nil {
return ctrl.Result{}, getSnapshotTargetErr
}

if err := r.expire(ctx, backup); err != nil {
return ctrl.Result{}, err
}

if !meta.IsStatusConditionTrue(backup.Status.Conditions, mantlev1.BackupConditionReadyToUse) {
result, err := r.startImport(ctx, backup, target)
if err != nil || !result.IsZero() {
return result, err
}
}

return r.secondaryCleanup(ctx, backup)
}

func scheduleExpire(_ context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) {
Expand Down Expand Up @@ -455,6 +512,7 @@ func (r *MantleBackupReconciler) replicateManifests(
annotRemoteUID: string(pvc.GetUID()),
})
pvcSent.Spec = pvc.Spec
pvcSent.Spec.VolumeName = "" // The VolumeName should be blank.
pvcSentJson, err := json.Marshal(pvcSent)
if err != nil {
return ctrl.Result{}, err
Expand Down Expand Up @@ -574,7 +632,7 @@ func isCreatedWhenMantleControllerWasSecondary(backup *mantlev1.MantleBackup) bo
return ok
}

func (r *MantleBackupReconciler) finalize(
func (r *MantleBackupReconciler) finalizeStandalone(
ctx context.Context,
backup *mantlev1.MantleBackup,
target *snapshotTarget,
Expand All @@ -585,15 +643,53 @@ func (r *MantleBackupReconciler) finalize(
return ctrl.Result{Requeue: true}, nil
}

if !controllerutil.ContainsFinalizer(backup, MantleBackupFinalizerName) {
return ctrl.Result{}, nil
}

// primaryClean() is called in finalizeStandalone() to delete resources for
// exported and uploaded snapshots in both standalone and primary Mantle.
result, err := r.primaryCleanup(ctx, backup)
if err != nil || result != (ctrl.Result{}) {
return result, err
}

if !targetPVCNotFound {
err := r.removeRBDSnapshot(ctx, target.poolName, target.imageName, backup.Name)
if err != nil {
return ctrl.Result{}, err
}
}

controllerutil.RemoveFinalizer(backup, MantleBackupFinalizerName)
if err := r.Update(ctx, backup); err != nil {
logger.Error(err, "failed to remove finalizer", "finalizer", MantleBackupFinalizerName)
return ctrl.Result{}, err
}

return ctrl.Result{}, nil
}

func (r *MantleBackupReconciler) finalizeSecondary(
ctx context.Context,
backup *mantlev1.MantleBackup,
target *snapshotTarget,
targetPVCNotFound bool,
) (ctrl.Result, error) {
logger := log.FromContext(ctx)
if _, ok := backup.GetAnnotations()[annotDiffTo]; ok {
return ctrl.Result{Requeue: true}, nil
}

if !controllerutil.ContainsFinalizer(backup, MantleBackupFinalizerName) {
return ctrl.Result{}, nil
}

result, err := r.secondaryCleanup(ctx, backup)
if err != nil || result != (ctrl.Result{}) {
return result, err
}

if !targetPVCNotFound {
err := r.removeRBDSnapshot(ctx, target.poolName, target.imageName, backup.Name)
if err != nil {
Expand Down Expand Up @@ -765,6 +861,107 @@ func (r *MantleBackupReconciler) export(
return ctrl.Result{}, nil
}

func (r *MantleBackupReconciler) startImport(
ctx context.Context,
backup *mantlev1.MantleBackup,
target *snapshotTarget,
) (ctrl.Result, error) { //nolint:unparam
if result, err := r.isExportDataAlreadyUploaded(ctx, backup); err != nil || !result.IsZero() {
return result, err
}

// Requeue if the PV is smaller than the PVC. (This may be the case if pvc-autoresizer is used.)
if isPVSmallerThanPVC(target.pv, target.pvc) {
return ctrl.Result{Requeue: true}, nil
}

if err := r.updateStatusManifests(ctx, backup, target.pv, target.pvc); err != nil {
return ctrl.Result{}, err
}

if result, err := r.reconcileDiscardJob(ctx, backup, target); err != nil || !result.IsZero() {
return result, err
}

if result, err := r.reconcileImportJob(ctx, backup, target); err != nil || !result.IsZero() {
return result, err
}

return ctrl.Result{}, nil
}

func (r *MantleBackupReconciler) isExportDataAlreadyUploaded(
_ context.Context,
_ *mantlev1.MantleBackup,
) (ctrl.Result, error) { //nolint:unparam
return ctrl.Result{}, nil
}

func isPVSmallerThanPVC(
pv *corev1.PersistentVolume,
pvc *corev1.PersistentVolumeClaim,
) bool {
return pv.Spec.Capacity.Storage().Cmp(*pvc.Spec.Resources.Requests.Storage()) == -1
}

func (r *MantleBackupReconciler) updateStatusManifests(
ctx context.Context,
backup *mantlev1.MantleBackup,
pv *corev1.PersistentVolume,
pvc *corev1.PersistentVolumeClaim,
) error {
if backup.Status.PVManifest != "" || backup.Status.PVCManifest != "" {
return nil
}
return updateStatus(ctx, r.Client, backup, func() error {
pvJSON, err := json.Marshal(*pv)
if err != nil {
return err
}
backup.Status.PVManifest = string(pvJSON)

pvcJSON, err := json.Marshal(*pvc)
if err != nil {
return err
}
backup.Status.PVCManifest = string(pvcJSON)

return nil
})
}

func (r *MantleBackupReconciler) reconcileDiscardJob(
_ context.Context,
backup *mantlev1.MantleBackup,
_ *snapshotTarget,
) (ctrl.Result, error) { //nolint:unparam
if backup.GetAnnotations()[annotSyncMode] != syncModeFull {
return ctrl.Result{}, nil
}

// FIXME: implement here later

return ctrl.Result{}, nil
}

func (r *MantleBackupReconciler) reconcileImportJob(
ctx context.Context,
backup *mantlev1.MantleBackup,
_ *snapshotTarget,
) (ctrl.Result, error) { //nolint:unparam
// FIXME: implement here later

if err := r.updateStatusCondition(ctx, backup, metav1.Condition{
Type: mantlev1.BackupConditionReadyToUse,
Status: metav1.ConditionTrue,
Reason: mantlev1.BackupReasonNone,
}); err != nil {
return ctrl.Result{}, err
}

return ctrl.Result{}, nil
}

func (r *MantleBackupReconciler) primaryCleanup(
ctx context.Context,
backup *mantlev1.MantleBackup,
Expand All @@ -784,3 +981,10 @@ func (r *MantleBackupReconciler) primaryCleanup(

return ctrl.Result{}, nil
}

func (r *MantleBackupReconciler) secondaryCleanup(
_ context.Context,
_ *mantlev1.MantleBackup,
) (ctrl.Result, error) { // nolint:unparam
return ctrl.Result{}, nil
}
14 changes: 9 additions & 5 deletions test/e2e/multik8s/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

mantlev1 "github.com/cybozu-go/mantle/api/v1"
corev1 "k8s.io/api/core/v1"
)

func TestMtest(t *testing.T) {
Expand Down Expand Up @@ -105,9 +106,15 @@ func replicationTestSuite() {
pvc.Annotations["mantle.cybozu.io/remote-uid"] != string(primaryPVC.GetUID()) {
return errors.New("invalid remote-uid annotation")
}
primaryPVC.Spec.VolumeName = ""
pvc.Spec.VolumeName = ""
if !reflect.DeepEqual(primaryPVC.Spec, pvc.Spec) {
return errors.New("spec not equal")
}
if pvc.Status.Phase != corev1.ClaimBound {
return errors.New("pvc not bound")
}

return nil
}).Should(Succeed())

Expand Down Expand Up @@ -151,11 +158,8 @@ func replicationTestSuite() {
if secondaryMB.Status.SnapID != nil {
return errors.New(".Status.SapID is incorrectly populated")
}
if secondaryMB.Status.PVManifest != "" {
return errors.New(".Status.PVManifest is incorrectly populated")
}
if secondaryMB.Status.PVCManifest != "" {
return errors.New(".Status.PVCManifest is incorrectly populated")
if !meta.IsStatusConditionTrue(secondaryMB.Status.Conditions, "ReadyToUse") {
return errors.New("ReadyToUse of .Status.Conditions is not True")
}

return nil
Expand Down

0 comments on commit da5624d

Please sign in to comment.