Merge pull request #39 from cybozu-go/mb-expire

Change MB expire strategy

api/v1/mantlebackup_types.go

@@ -19,6 +19,16 @@ type MantleBackupSpec struct {
 	// 'namespace' specifies backup target Namespace
 	// +kubebuilder:validation:Required
 	Namespace string `json:"namespace,omitempty"`
+
+	// NOTE: we CANNOT use metav1.Duration for Expire due to an unresolved k8s bug.
+	// See https://github.com/kubernetes/apiextensions-apiserver/issues/56 for the details.
+
+	// 'expire' specifies the expiration duration of the backup
+	//+kubebuilder:validation:Format:="duration"
+	//+kubebuilder:validation:XValidation:message="expire must be >= 1d",rule="self >= duration('24h')"
+	//+kubebuilder:validation:XValidation:message="expire must be <= 15d",rule="self <= duration('360h')"
+	//+kubebuilder:validation:XValidation:message="spec.expire is immutable",rule="self == oldSelf"
+	Expire string `json:"expire"`
 }
 
 // MantleBackupStatus defines the observed state of MantleBackup

charts/mantle-cluster-wide/templates/mantle.cybozu.io_mantlebackups.yaml

@@ -41,12 +41,26 @@ spec:
           spec:
             description: MantleBackupSpec defines the desired state of MantleBackup
             properties:
+              expire:
+                description: '''expire'' specifies the expiration duration of the
+                  backup'
+                format: duration
+                type: string
+                x-kubernetes-validations:
+                - message: expire must be >= 1d
+                  rule: self >= duration('24h')
+                - message: expire must be <= 15d
+                  rule: self <= duration('360h')
+                - message: spec.expire is immutable
+                  rule: self == oldSelf
               namespace:
                 description: '''namespace'' specifies backup target Namespace'
                 type: string
               pvc:
                 description: '''pvc'' specifies backup target PVC'
                 type: string
+            required:
+            - expire
             type: object
           status:
             description: MantleBackupStatus defines the observed state of MantleBackup

charts/mantle/templates/deployment.yaml

@@ -64,9 +64,6 @@ spec:
             {{- with .Values.controller.mantleServiceEndpoint }}
             - --mantle-service-endpoint={{ . }}
             {{- end }}
-            {{- with .Values.controller.expireOffset }}
-            - --expire-offset={{ . }}
-            {{- end }}
             {{- with .Values.controller.overwriteMBCSchedule }}
             - --overwrite-mbc-schedule={{ . }}
             {{- end }}

cmd/backupandrotate/main.go → cmd/backup/main.go

@@ -1,4 +1,4 @@
-package backupandrotate
+package backup
 
 import (
 	"context"
@@ -8,16 +8,13 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"time"
 
 	"github.com/spf13/cobra"
 	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/kube-openapi/pkg/validation/strfmt"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -29,29 +26,24 @@ import (
 
 var (
 	mbcName, mbcNamespace string
-	expireOffset          string
 	zapOpts               zap.Options
 
 	scheme                = runtime.NewScheme()
 	MantleBackupConfigUID = "mantle.cybozu.io/mbc-uid"
-	MantleRetainIfExpired = "mantle.cybozu.io/retainIfExpired"
-	logger                = ctrl.Log.WithName("backup-and-rotate")
+	logger                = ctrl.Log.WithName("backup")
 
-	BackupAndRotateCmd = &cobra.Command{
-		Use: "backup-and-rotate",
+	BackupCmd = &cobra.Command{
+		Use: "backup",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return subMain(cmd.Context())
 		},
 	}
 )
 
 func init() {
-	flags := BackupAndRotateCmd.Flags()
+	flags := BackupCmd.Flags()
 	flags.StringVar(&mbcName, "name", "", "MantleBackupConfig resource's name")
 	flags.StringVar(&mbcNamespace, "namespace", "", "MantleBackupConfig resource's namespace")
-	flags.StringVar(&expireOffset, "expire-offset", "0s",
-		"An offset for MantleBackupConfig's .spec.expire field. A MantleBackup will expire after "+
-			"it has been active for (.spec.expire - expire-offset) time. This option is intended for testing purposes only.")
 
 	goflags := flag.NewFlagSet("goflags", flag.ExitOnError)
 	zapOpts.Development = true
@@ -91,11 +83,6 @@ func fetchJobID() (string, error) {
 func subMain(ctx context.Context) error {
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOpts)))
 
-	parsedExpireOffset, err := strfmt.ParseDuration(expireOffset)
-	if err != nil {
-		return fmt.Errorf("couldn't parse the expire offset: %w", err)
-	}
-
 	cli, err := client.New(config.GetConfigOrDie(), client.Options{Scheme: scheme})
 	if err != nil {
 		return fmt.Errorf("couldn't create a new client: %w", err)
@@ -111,10 +98,6 @@ func subMain(ctx context.Context) error {
 		return fmt.Errorf("backup failed: %s: %s: %w", mbcName, mbcNamespace, err)
 	}
 
-	if err := rotateMantleBackup(ctx, cli, &mbc, parsedExpireOffset); err != nil {
-		return fmt.Errorf("rotation failed: %s: %s: %w", mbcName, mbcNamespace, err)
-	}
-
 	return nil
 }
 
@@ -136,7 +119,8 @@ func createMantleBackup(ctx context.Context, cli client.Client, mbc *mantlev1.Ma
 			Labels:    map[string]string{MantleBackupConfigUID: string(mbc.GetUID())},
 		},
 		Spec: mantlev1.MantleBackupSpec{
-			PVC: mbc.Spec.PVC,
+			Expire: mbc.Spec.Expire,
+			PVC:    mbc.Spec.PVC,
 		},
 	})
 	if err == nil {
@@ -169,62 +153,3 @@ func createMantleBackup(ctx context.Context, cli client.Client, mbc *mantlev1.Ma
 		"mbcName", mbcName, "mbcNamespace", mbcNamespace)
 	return nil
 }
-
-func rotateMantleBackup(
-	ctx context.Context,
-	cli client.Client,
-	mbc *mantlev1.MantleBackupConfig,
-	expireOffset time.Duration,
-) error {
-	// List all MantleBackup objects associated with the mbc.
-	var mbList mantlev1.MantleBackupList
-	if err := cli.List(ctx, &mbList, &client.ListOptions{
-		LabelSelector: labels.SelectorFromSet(map[string]string{MantleBackupConfigUID: string(mbc.GetUID())}),
-	}); err != nil {
-		return fmt.Errorf("couldn't list MantleBackups: %s: %w", string(mbc.UID), err)
-	}
-
-	// Delete the MantleBackup objects that are already expired and don't have the retainIfExpired label.
-	expire, err := strfmt.ParseDuration(mbc.Spec.Expire)
-	if err != nil {
-		return fmt.Errorf("couldn't parse spec.expire: %s: %w", mbc.Spec.Expire, err)
-	}
-	if expire >= expireOffset {
-		expire -= expireOffset
-	} else {
-		expire = 0
-	}
-	for _, mb := range mbList.Items {
-		if mb.Status.CreatedAt == (metav1.Time{}) {
-			// mb is not created yet (at least from the cache's perspective), so let's ignore it.
-			continue
-		}
-		elapsed := time.Since(mb.Status.CreatedAt.Time)
-		if elapsed <= expire {
-			continue
-		}
-		retain, ok := mb.GetAnnotations()[MantleRetainIfExpired]
-		if ok && retain == "true" {
-			continue
-		}
-
-		if err := cli.Delete(ctx, &mb, &client.DeleteOptions{
-			Preconditions: &metav1.Preconditions{
-				UID:             &mb.UID,
-				ResourceVersion: &mb.ResourceVersion,
-			},
-		}); err == nil || errors.IsNotFound(err) {
-			logger.Info("MantleBackup deleted",
-				"mb.Name", mb.Name, "mb.Namespace", mb.Namespace,
-				"mb.UID", mb.UID, "mb.ResourceVersion", mb.ResourceVersion,
-				"mbcName", mbcName, "mbcNamespace", mbcNamespace,
-				"elapsed", elapsed, "expire", expire,
-			)
-		} else {
-			return fmt.Errorf("couldn't delete MantleBackup: %s: %s: %s: %s: %w",
-				mb.Name, mb.Namespace, mb.UID, mb.ResourceVersion, err)
-		}
-	}
-
-	return nil
-}

cmd/controller/main.go

@@ -46,7 +46,6 @@ var (
 	enableLeaderElection  bool
 	probeAddr             string
 	zapOpts               zap.Options
-	expireOffset          string
 	overwriteMBCSchedule  string
 	role                  string
 	mantleServiceEndpoint string
@@ -62,9 +61,6 @@ func init() {
 	flags.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	flags.StringVar(&expireOffset, "expire-offset", "",
-		"An offset for MantleBackupConfig's .spec.expire field. A MantleBackup will expire after "+
-			"it has been active for (.spec.expire - expire-offset) time. This option is intended for testing purposes only.")
 	flags.StringVar(&overwriteMBCSchedule, "overwrite-mbc-schedule", "",
 		"By setting this option, every CronJob created by this controller for every MantleBackupConfig "+
 			"will use its value as .spec.schedule. This option is intended for testing purposes only.")
@@ -142,7 +138,6 @@ func setupReconcilers(mgr manager.Manager, primarySettings *controller.PrimarySe
 		mgr.GetClient(),
 		mgr.GetScheme(),
 		managedCephClusterID,
-		expireOffset,
 		overwriteMBCSchedule,
 		role,
 	).SetupWithManager(mgr); err != nil {

cmd/root.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"os"
 
-	"github.com/cybozu-go/mantle/cmd/backupandrotate"
+	"github.com/cybozu-go/mantle/cmd/backup"
 	"github.com/cybozu-go/mantle/cmd/controller"
 	"github.com/spf13/cobra"
 )
@@ -13,7 +13,7 @@ var rootCmd = &cobra.Command{
 }
 
 func init() {
-	rootCmd.AddCommand(backupandrotate.BackupAndRotateCmd)
+	rootCmd.AddCommand(backup.BackupCmd)
 	rootCmd.AddCommand(controller.ControllerCmd)
 }
 

config/crd/bases/mantle.cybozu.io_mantlebackups.yaml

@@ -41,12 +41,26 @@ spec:
           spec:
             description: MantleBackupSpec defines the desired state of MantleBackup
             properties:
+              expire:
+                description: '''expire'' specifies the expiration duration of the
+                  backup'
+                format: duration
+                type: string
+                x-kubernetes-validations:
+                - message: expire must be >= 1d
+                  rule: self >= duration('24h')
+                - message: expire must be <= 15d
+                  rule: self <= duration('360h')
+                - message: spec.expire is immutable
+                  rule: self == oldSelf
               namespace:
                 description: '''namespace'' specifies backup target Namespace'
                 type: string
               pvc:
                 description: '''pvc'' specifies backup target PVC'
                 type: string
+            required:
+            - expire
             type: object
           status:
             description: MantleBackupStatus defines the observed state of MantleBackup

docs/design.md

@@ -11,7 +11,7 @@ We want to backup and restore RBD PVCs managed by a Rook/Ceph cluster, either by
 3. Backup arbitrary RBD PVCs periodically.
 4. Copy backup data to another cluster in another data center.
 
-Currently, the goal 1 and 3 are implemented. Other goals will be achieved later.
+Currently, the goal 1, 2, and 3 are implemented. Other goals will be achieved later.
 
 ## Architecture
 
@@ -39,7 +39,7 @@ flowchart LR
       RC -- point --> RS
 
       %% backup config
-      MBCCronJob -- create/delete --> MB
+      MBCCronJob -- create --> MB
       MBCR -- watch --> MBC
       MBC -- point --> SRC_PVC
       MBCR -- create --> MBCCronJob
@@ -48,7 +48,7 @@ flowchart LR
       %% backup
       MB -.-|related| RS
       USER -- create/delete --> MB
-      MBR -- watch --> MB
+      MBR -- watch/delete --> MB
       MB -- point --> SRC_PVC
       SRC_PVC -- consume --> SRC_PV
       USER -- create/delete --> MBC
@@ -103,13 +103,20 @@ To create/delete a backup, mantle works as follows:
 
 ### Periodic backup flow
 
-To create and rotate backups periodically, Mantle works as follows:
+To create backups periodically, Mantle works as follows:
 
 1. Users create a `MantleBackupConfig`.
 2. The mantle-controller then creates a `CronJob` based on the `MantleBackupConfig`.
-3. The Pod, which is periodically created by the `CronJob`, creates a new `MantleBackup` and deletes any expired `MantleBackup` resources.
+3. The Pod, which is periodically created by the `CronJob`, creates a new `MantleBackup` resources.
 
-If a `MantleBackupConfig` is deleted, the associated `MantleBackup`s won't be removed automatically and won't expire on their own. The users need to delete them manually if they wish to do so.
+If a `MantleBackupConfig` is deleted, the associated `MantleBackup`s won't be removed automatically. The users need to delete them manually if they wish to do so, or use expiration.
+
+### Backup expiration flow
+
+`MantleBackup` resource has an `expire` field. If time will pass the `expire` duration, the controller will delete the `MantleBackup` resource.
+This process can be stopped by adding `mantle.cybozu.io/retain-if-expired` annotation to the `MantleBackup` resource.
+
+`MantleBackupConfig` also has an `expire` field. The `CronJob` set the value to the `MantleBackup` resource created by the `MantleBackupConfig`. Therefore, the periodic backups will be deleted automatically.
 
 ### Sample manifests
 
@@ -124,6 +131,7 @@ metadata:
 spec:
   # The name of the backup target PVC
   pvc: <target PVC name>
+  expire: 2w # when the MantleBackup should expire.
 status:
   conditions:
     # The corresponding backup data is ready to use if `status` is "True"

internal/ceph/ceph.go

@@ -1,16 +1,46 @@
 package ceph
 
+import (
+	"strings"
+	"time"
+)
+
 type RBDInfo struct {
 	ParentPool  string
 	ParentImage string
 	ParentSnap  string
 }
 
+type RBDTimeStamp struct {
+	time.Time
+}
+
+func NewRBDTimeStamp(t time.Time) RBDTimeStamp {
+	return RBDTimeStamp{t}
+}
+
+func (t *RBDTimeStamp) UnmarshalJSON(data []byte) error {
+	var err error
+	t.Time, err = time.Parse("Mon Jan  2 15:04:05 2006", strings.Trim(string(data), `"`))
+	return err
+}
+
+type RBDSnapshot struct {
+	Id        int          `json:"id,omitempty"`
+	Name      string       `json:"name,omitempty"`
+	Size      int          `json:"size,omitempty"`
+	Protected bool         `json:"protected,string,omitempty"`
+	Timestamp RBDTimeStamp `json:"timestamp,omitempty"`
+}
+
 type CephCmd interface {
 	RBDClone(pool, srcImage, srcSnap, dstImage, features string) error
 	RBDInfo(pool, image string) (*RBDInfo, error)
 	RBDLs(pool string) ([]string, error)
 	RBDRm(pool, image string) error
+	RBDSnapCreate(pool, image, snap string) error
+	RBDSnapLs(pool, image string) ([]RBDSnapshot, error)
+	RBDSnapRm(pool, image, snap string) error
 }
 
 type cephCmdImpl struct {