From c6f333db323cd4e8528f36934305d1ccf4138379 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Fri, 30 Aug 2024 04:21:08 +0000 Subject: [PATCH 1/4] add .status.snapID to MantleBackup CRD Signed-off-by: Ryotaro Banno --- api/v1/mantlebackup_types.go | 3 +++ .../templates/mantle.cybozu.io_mantlebackups.yaml | 3 +++ config/crd/bases/mantle.cybozu.io_mantlebackups.yaml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/api/v1/mantlebackup_types.go b/api/v1/mantlebackup_types.go index 243c2c4..9583213 100644 --- a/api/v1/mantlebackup_types.go +++ b/api/v1/mantlebackup_types.go @@ -36,6 +36,9 @@ type MantleBackupStatus struct { PVCManifest string `json:"pvcManifest,omitempty"` // 'pvManifest' saving backup target PV manifest PVManifest string `json:"pvManifest,omitempty"` + + // 'snapID' indicates SNAPID of `rbd snap ls` + SnapID int `json:"snapID,omitempty"` } const ( diff --git a/charts/mantle-cluster-wide/templates/mantle.cybozu.io_mantlebackups.yaml b/charts/mantle-cluster-wide/templates/mantle.cybozu.io_mantlebackups.yaml index f1c2633..2f6d05c 100644 --- a/charts/mantle-cluster-wide/templates/mantle.cybozu.io_mantlebackups.yaml +++ b/charts/mantle-cluster-wide/templates/mantle.cybozu.io_mantlebackups.yaml @@ -131,6 +131,9 @@ spec: pvcManifest: description: '''pvcManifest'' saving backup target PVC manifests' type: string + snapID: + description: '''snapID'' indicates SNAPID of `rbd snap ls`' + type: integer type: object type: object served: true diff --git a/config/crd/bases/mantle.cybozu.io_mantlebackups.yaml b/config/crd/bases/mantle.cybozu.io_mantlebackups.yaml index f1c2633..2f6d05c 100644 --- a/config/crd/bases/mantle.cybozu.io_mantlebackups.yaml +++ b/config/crd/bases/mantle.cybozu.io_mantlebackups.yaml @@ -131,6 +131,9 @@ spec: pvcManifest: description: '''pvcManifest'' saving backup target PVC manifests' type: string + snapID: + description: '''snapID'' indicates SNAPID of `rbd snap ls`' + type: integer type: object type: object served: true From 11fc6300e1887410c78668abfbb98aa84d0b27b5 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Fri, 30 Aug 2024 04:36:17 +0000 Subject: [PATCH 2/4] chore: make separate functions for finding and listing RBD snapshots Signed-off-by: Ryotaro Banno --- .../controller/mantlebackup_controller.go | 69 ++++++++++--------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/internal/controller/mantlebackup_controller.go b/internal/controller/mantlebackup_controller.go index be05ec4..751183c 100644 --- a/internal/controller/mantlebackup_controller.go +++ b/internal/controller/mantlebackup_controller.go @@ -131,42 +131,49 @@ func (r *MantleBackupReconciler) removeRBDSnapshot(poolName, imageName, snapshot return nil } +func listRBDSnapshots(poolName, imageName string) ([]Snapshot, error) { + command := []string{"rbd", "snap", "ls", poolName + "/" + imageName, "--format=json"} + out, err := executeCommand(command, nil) + if err != nil { + return nil, fmt.Errorf("failed to execute `rbd snap ls`: %s: %s: %w", poolName, imageName, err) + } + + var snapshots []Snapshot + err = json.Unmarshal(out, &snapshots) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal the output of `rbd snap ls`: %s: %s: %w", poolName, imageName, err) + } + + return snapshots, nil +} + +func findRBDSnapshot(poolName, imageName, snapshotName string) (*Snapshot, error) { + snapshots, err := listRBDSnapshots(poolName, imageName) + if err != nil { + return nil, err + } + for _, s := range snapshots { + if s.Name == snapshotName { + return &s, nil + } + } + return nil, fmt.Errorf("snapshot not found: %s: %s: %s", poolName, imageName, snapshotName) +} + func (r *MantleBackupReconciler) createRBDSnapshot(ctx context.Context, poolName, imageName string, backup *mantlev1.MantleBackup) (ctrl.Result, error) { command := []string{"rbd", "snap", "create", poolName + "/" + imageName + "@" + backup.Name} _, err := executeCommand(command, nil) if err != nil { - command = []string{"rbd", "snap", "ls", poolName + "/" + imageName, "--format=json"} - out, err := executeCommand(command, nil) - if err != nil { - logger.Info("failed to run `rbd snap ls`", "poolName", poolName, "imageName", imageName, "error", err) - err2 := r.updateStatus(ctx, backup, metav1.Condition{Type: mantlev1.BackupConditionReadyToUse, Status: metav1.ConditionFalse, Reason: mantlev1.BackupReasonFailedToCreateBackup}) - if err2 != nil { - return ctrl.Result{}, err2 - } - return ctrl.Result{Requeue: true}, nil - } - var snapshots []Snapshot - err = json.Unmarshal(out, &snapshots) + _, err := findRBDSnapshot(poolName, imageName, backup.Name) if err != nil { - logger.Error("failed to unmarshal json", "json", out, "error", err) - err2 := r.updateStatus(ctx, backup, metav1.Condition{Type: mantlev1.BackupConditionReadyToUse, Status: metav1.ConditionFalse, Reason: mantlev1.BackupReasonFailedToCreateBackup}) - if err2 != nil { - return ctrl.Result{}, err2 - } - return ctrl.Result{Requeue: true}, err - } - existSnapshot := false - for _, s := range snapshots { - if s.Name == backup.Name { - existSnapshot = true - break - } - } - if !existSnapshot { - logger.Info("snapshot does not exists", "snapshotName", backup.Name) - err2 := r.updateStatus(ctx, backup, metav1.Condition{Type: mantlev1.BackupConditionReadyToUse, Status: metav1.ConditionFalse, Reason: mantlev1.BackupReasonFailedToCreateBackup}) - if err2 != nil { - return ctrl.Result{}, err2 + logger.Error("failed to find rbd snapshot", "error", err) + err := r.updateStatus(ctx, backup, metav1.Condition{ + Type: mantlev1.BackupConditionReadyToUse, + Status: metav1.ConditionFalse, + Reason: mantlev1.BackupReasonFailedToCreateBackup, + }) + if err != nil { + return ctrl.Result{}, err } return ctrl.Result{Requeue: true}, nil } From 199b7abb10ca7ac745d22574f5e1961b4a12ef27 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Fri, 30 Aug 2024 05:01:43 +0000 Subject: [PATCH 3/4] set .status.snapID of MantleBackup when snapshot is created Signed-off-by: Ryotaro Banno --- internal/controller/mantlebackup_controller.go | 6 ++++++ .../controller/mantlebackup_controller_test.go | 14 +++++++++----- .../mantlebackupconfig_controller_test.go | 5 ++++- .../controller/mantlerestore_controller_test.go | 8 +++++++- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/internal/controller/mantlebackup_controller.go b/internal/controller/mantlebackup_controller.go index 751183c..8a4f835 100644 --- a/internal/controller/mantlebackup_controller.go +++ b/internal/controller/mantlebackup_controller.go @@ -355,6 +355,12 @@ func (r *MantleBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request } backup.Status.PVManifest = string(pvJs) + snapshot, err := findRBDSnapshot(poolName, imageName, backup.Name) + if err != nil { + return ctrl.Result{}, err + } + backup.Status.SnapID = snapshot.Id + backup.Status.CreatedAt = metav1.NewTime(time.Now()) err = r.updateStatus(ctx, &backup, metav1.Condition{Type: mantlev1.BackupConditionReadyToUse, Status: metav1.ConditionTrue, Reason: mantlev1.BackupReasonNone}) if err != nil { diff --git a/internal/controller/mantlebackup_controller_test.go b/internal/controller/mantlebackup_controller_test.go index 3b55d77..6fe57d3 100644 --- a/internal/controller/mantlebackup_controller_test.go +++ b/internal/controller/mantlebackup_controller_test.go @@ -24,10 +24,6 @@ import ( "k8s.io/client-go/kubernetes/scheme" ) -func mockExecuteCommand(command []string, input io.Reader) ([]byte, error) { - return nil, nil -} - var _ = Describe("MantleBackup controller", func() { ctx := context.Background() var mgrUtil util.ManagerUtil @@ -43,7 +39,12 @@ var _ = Describe("MantleBackup controller", func() { err := reconciler.SetupWithManager(mgrUtil.GetManager()) Expect(err).NotTo(HaveOccurred()) - executeCommand = mockExecuteCommand + executeCommand = func(command []string, _ io.Reader) ([]byte, error) { + if command[0] == "rbd" && command[1] == "snap" && command[2] == "ls" { + return []byte(fmt.Sprintf("[{\"id\":1000,\"name\":\"backup\"}]")), nil + } + return nil, nil + } mgrUtil.Start() time.Sleep(100 * time.Millisecond) @@ -177,6 +178,9 @@ var _ = Describe("MantleBackup controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(pvStored.Name).To(Equal(pv.Name)) + snapID := backup.Status.SnapID + Expect(snapID).To(Equal(1000)) + err = k8sClient.Delete(ctx, &backup) Expect(err).NotTo(HaveOccurred()) diff --git a/internal/controller/mantlebackupconfig_controller_test.go b/internal/controller/mantlebackupconfig_controller_test.go index 568c8a0..fcb7f32 100644 --- a/internal/controller/mantlebackupconfig_controller_test.go +++ b/internal/controller/mantlebackupconfig_controller_test.go @@ -3,6 +3,7 @@ package controller import ( "context" "fmt" + "io" "time" mantlev1 "github.com/cybozu-go/mantle/api/v1" @@ -174,7 +175,9 @@ var _ = Describe("MantleBackupConfig controller", func() { err = reconciler.SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred()) - executeCommand = mockExecuteCommand + executeCommand = func(_ []string, _ io.Reader) ([]byte, error) { + return nil, nil + } ctx, cancel := context.WithCancel(ctx) stopFunc = cancel diff --git a/internal/controller/mantlerestore_controller_test.go b/internal/controller/mantlerestore_controller_test.go index f867080..ff5233e 100644 --- a/internal/controller/mantlerestore_controller_test.go +++ b/internal/controller/mantlerestore_controller_test.go @@ -3,6 +3,7 @@ package controller import ( "context" "fmt" + "io" "time" mantlev1 "github.com/cybozu-go/mantle/api/v1" @@ -67,7 +68,12 @@ func (test *mantleRestoreControllerUnitTest) setupEnv() { It("prepare reconcilers", func() { By("prepare MantleBackup reconciler") - executeCommand = mockExecuteCommand + executeCommand = func(command []string, _ io.Reader) ([]byte, error) { + if command[0] == "rbd" && command[1] == "snap" && command[2] == "ls" { + return []byte(fmt.Sprintf("[{\"id\":1000,\"name\":\"%s\"}]", test.backupName)), nil + } + return nil, nil + } test.mgrUtil = testutil.NewManagerUtil(ctx, cfg, scheme.Scheme) backupReconciler := NewMantleBackupReconciler(k8sClient, test.mgrUtil.GetScheme(), test.cephClusterID, RoleStandalone, nil) err := backupReconciler.SetupWithManager(test.mgrUtil.GetManager()) From 79c269f46631a851f340d9aae6f457893af188a5 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Fri, 30 Aug 2024 05:52:22 +0000 Subject: [PATCH 4/4] set .status.createdAt according to timestamp of the snapshot Signed-off-by: Ryotaro Banno --- internal/controller/mantlebackup_controller.go | 7 ++++++- internal/controller/mantlebackup_controller_test.go | 3 ++- internal/controller/mantlerestore_controller_test.go | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/controller/mantlebackup_controller.go b/internal/controller/mantlebackup_controller.go index 8a4f835..a24f76b 100644 --- a/internal/controller/mantlebackup_controller.go +++ b/internal/controller/mantlebackup_controller.go @@ -361,7 +361,12 @@ func (r *MantleBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request } backup.Status.SnapID = snapshot.Id - backup.Status.CreatedAt = metav1.NewTime(time.Now()) + createdAt, err := time.Parse("Mon Jan 2 15:04:05 2006", snapshot.Timestamp) + if err != nil { + return ctrl.Result{}, err + } + backup.Status.CreatedAt = metav1.NewTime(createdAt) + err = r.updateStatus(ctx, &backup, metav1.Condition{Type: mantlev1.BackupConditionReadyToUse, Status: metav1.ConditionTrue, Reason: mantlev1.BackupReasonNone}) if err != nil { return ctrl.Result{}, err diff --git a/internal/controller/mantlebackup_controller_test.go b/internal/controller/mantlebackup_controller_test.go index 6fe57d3..1009971 100644 --- a/internal/controller/mantlebackup_controller_test.go +++ b/internal/controller/mantlebackup_controller_test.go @@ -41,7 +41,8 @@ var _ = Describe("MantleBackup controller", func() { executeCommand = func(command []string, _ io.Reader) ([]byte, error) { if command[0] == "rbd" && command[1] == "snap" && command[2] == "ls" { - return []byte(fmt.Sprintf("[{\"id\":1000,\"name\":\"backup\"}]")), nil + return []byte(fmt.Sprintf("[{\"id\":1000,\"name\":\"backup\"," + + "\"timestamp\":\"Mon Sep 2 00:42:00 2024\"}]")), nil } return nil, nil } diff --git a/internal/controller/mantlerestore_controller_test.go b/internal/controller/mantlerestore_controller_test.go index ff5233e..8a3aa51 100644 --- a/internal/controller/mantlerestore_controller_test.go +++ b/internal/controller/mantlerestore_controller_test.go @@ -70,7 +70,7 @@ func (test *mantleRestoreControllerUnitTest) setupEnv() { By("prepare MantleBackup reconciler") executeCommand = func(command []string, _ io.Reader) ([]byte, error) { if command[0] == "rbd" && command[1] == "snap" && command[2] == "ls" { - return []byte(fmt.Sprintf("[{\"id\":1000,\"name\":\"%s\"}]", test.backupName)), nil + return []byte(fmt.Sprintf("[{\"id\":1000,\"name\":\"%s\",\"timestamp\":\"Mon Sep 2 00:42:00 2024\"}]", test.backupName)), nil } return nil, nil }