Skip to content

Commit

Permalink
feat(system-backup): check outdated backup in if-not-present volume b…
Browse files Browse the repository at this point in the history 
…ackup policy

longhorn/longhorn-6027

Signed-off-by: Chin-Ya Huang <chin-ya.huang@suse.com>
  • Loading branch information
@c3y1huang
c3y1huang committed Oct 30, 2024
1 parent 235b1ae commit 70ad3d0
Showing 1 changed file with 132 additions and 31 deletions.
163 changes: 132 additions & 31 deletions controller/system_backup_controller.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package controller

import (
"context"
"fmt"
"os"
"os/exec"
Expand Down Expand Up @@ -46,20 +47,21 @@ const (

SystemBackupTempDir = "/tmp"

SystemBackupErrArchive = "failed to archive system backup file"
SystemBackupErrDelete = "failed to delete system backup in backup target"
SystemBackupErrGenerate = "failed to generate system backup file"
SystemBackupErrGenerateYAML = "failed to generate resource YAMLs"
SystemBackupErrGetFmt = "failed to get %v"
SystemBackupErrGetConfig = "failed to get system backup config"
SystemBackupErrMkdir = "failed to create system backup file directory"
SystemBackupErrRemoveAll = "failed to remove system backup directory"
SystemBackupErrRemove = "failed to remove system backup file"
SystemBackupErrOSStat = "failed to compute system backup file size"
SystemBackupErrSync = "failed to sync from backup target"
SystemBackupErrTimeoutUpload = "timeout uploading system backup"
SystemBackupErrUpload = "failed to upload system backup file"
SystemBackupErrVolumeBackup = "failed to backup volumes"
SystemBackupErrArchive = "failed to archive system backup file"
SystemBackupErrDelete = "failed to delete system backup in backup target"
SystemBackupErrGenerate = "failed to generate system backup file"
SystemBackupErrGenerateYAML = "failed to generate resource YAMLs"
SystemBackupErrGetFmt = "failed to get %v"
SystemBackupErrGetConfig = "failed to get system backup config"
SystemBackupErrMkdir = "failed to create system backup file directory"
SystemBackupErrRemoveAll = "failed to remove system backup directory"
SystemBackupErrRemove = "failed to remove system backup file"
SystemBackupErrOSStat = "failed to compute system backup file size"
SystemBackupErrSync = "failed to sync from backup target"
SystemBackupErrTimeoutSnapshot = "timeout taking volume snapshot for system backup"
SystemBackupErrTimeoutUpload = "timeout uploading system backup"
SystemBackupErrUpload = "failed to upload system backup file"
SystemBackupErrVolumeBackup = "failed to backup volumes"

SystemBackupMsgCreatedArchieveFmt = "Created system backup file: %v"
SystemBackupMsgDeletingRemote = "Deleting system backup in backup target"
Expand Down Expand Up @@ -735,6 +737,9 @@ func (c *SystemBackupController) backupVolumesAlways(systemBackup *longhorn.Syst
return nil, err
}

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

volumeBackups := make(map[string]*longhorn.Backup, len(volumes))
for _, volume := range volumes {
// Don't need to create volume data backup for DR volumes since it will
Expand All @@ -744,7 +749,14 @@ func (c *SystemBackupController) backupVolumesAlways(systemBackup *longhorn.Syst
continue
}

backup, err := c.createVolumeBackup(volume, systemBackup)
volumeBackupName := bsutil.GenerateName("system-backup")

snapshot, err := c.createVolumeSnapshot(ctx, volume, volumeBackupName)
if err != nil {
return nil, err
}

backup, err := c.createVolumeBackupFromSnapshot(volume, snapshot)
if err != nil {
return nil, err
}
Expand All @@ -761,13 +773,28 @@ func (c *SystemBackupController) backupVolumesIfNotPresent(systemBackup *longhor
return nil, err
}

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

volumeBackups := make(map[string]*longhorn.Backup, len(volumes))
for _, volume := range volumes {
if volume.Status.LastBackup != "" {
volumeBackupName := bsutil.GenerateName("system-backup")

snapshot, err := c.createVolumeSnapshot(ctx, volume, volumeBackupName)
if err != nil {
return nil, err
}

isUpToDate, err := c.isVolumeBackupUpToDate(volume, systemBackup)
if err != nil {
return nil, err
}

if isUpToDate {
continue
}

backup, err := c.createVolumeBackup(volume, systemBackup)
backup, err := c.createVolumeBackupFromSnapshot(volume, snapshot)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -837,39 +864,113 @@ func (c *SystemBackupController) WaitForVolumeBackupToComplete(backups map[strin
return fmt.Errorf("unexpected error: stopped waiting for Volume backups without completing, failing or timing out")
}

func (c *SystemBackupController) createVolumeBackup(volume *longhorn.Volume, systemBackup *longhorn.SystemBackup) (backup *longhorn.Backup, err error) {
volumeBackupName := bsutil.GenerateName("system-backup")
func (c *SystemBackupController) isVolumeBackupUpToDate(volume *longhorn.Volume, systemBackup *longhorn.SystemBackup) (bool, error) {
log := getLoggerForSystemBackup(c.logger, systemBackup)
log = log.WithField("volume", volume.Name)

snapshotCR := &longhorn.Snapshot{
ObjectMeta: metav1.ObjectMeta{
Name: volumeBackupName,
},
Spec: longhorn.SnapshotSpec{
Volume: volume.Name,
CreateSnapshot: true,
},
// Return early if there is no recorded last backup.
if volume.Status.LastBackup == "" {
return false, nil
}

snapshot, err := c.ds.CreateSnapshot(snapshotCR)
// Retrieve last backup and its snapshot.
lastBackup, err := c.ds.GetBackup(volume.Status.LastBackup)
if err != nil {
return nil, errors.Wrapf(err, "failed to create Volume %v snapshot %s", volume.Name, snapshot.Name)
return false, err
}

lastBackupSnapshot, err := c.ds.GetSnapshot(lastBackup.Status.SnapshotName)
if err != nil {
return false, err
}

lastBackupTime, err := time.Parse(time.RFC3339, lastBackupSnapshot.Status.CreationTime)
if err != nil {
return false, err
}

// Identify snapshots created after the last backup.
snapshots, err := c.ds.ListVolumeSnapshotsRO(volume.Name)
if err != nil {
return false, err
}

for _, snapshot := range snapshots {
if snapshot.Status.Size == 0 {
continue
}

snapshotTime, err := time.Parse(time.RFC3339, snapshot.Status.CreationTime)
if err != nil {
return false, err
}

if snapshotTime.After(lastBackupTime) {
log.WithField("snapshot", snapshot.Name).Infof("Detected new data in snapshot %v after last backup %v", snapshot.Name, lastBackup.Name)
return false, nil // Backup is not up-to-date
}
}

log.Infof("No snapshots with new data after last backup %v; backup is up-to-date", lastBackup.Name)
return true, nil // Backup is up-to-date
}

func (c *SystemBackupController) createVolumeBackupFromSnapshot(volume *longhorn.Volume, snapshot *longhorn.Snapshot) (backup *longhorn.Backup, err error) {
backup = &longhorn.Backup{
ObjectMeta: metav1.ObjectMeta{
Name: volumeBackupName,
Name: snapshot.Name,
},
Spec: longhorn.BackupSpec{
SnapshotName: snapshot.Name,
},
}
backup, err = c.ds.CreateBackup(backup, volume.Name)
if err != nil {
return nil, errors.Wrapf(err, "failed to create Volume %v backup %s", volume.Name, volumeBackupName)
return nil, errors.Wrapf(err, "failed to create Volume %v backup %s", volume.Name, snapshot.Name)
}
return backup, nil
}

func (c *SystemBackupController) createVolumeSnapshot(ctx context.Context, volume *longhorn.Volume, volumeSnapshotName string) (snapshot *longhorn.Snapshot, err error) {
snapshotCR := &longhorn.Snapshot{
ObjectMeta: metav1.ObjectMeta{
Name: volumeSnapshotName,
},
Spec: longhorn.SnapshotSpec{
Volume: volume.Name,
CreateSnapshot: true,
},
}

snapshot, err = c.ds.CreateSnapshot(snapshotCR)
if err != nil {
return nil, errors.Wrapf(err, "failed to create Volume %v snapshot %s", volume.Name, snapshot.Name)
}

// Poll until snapshot is ready or timeout exceeded.
timeout := time.After(datastore.SystemBackupTimeout)
ticker := time.NewTicker(time.Second)
defer ticker.Stop()

for {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-timeout:
return nil, errors.New(SystemBackupErrTimeoutSnapshot)
case <-ticker.C:
snapshot, err = c.ds.GetSnapshot(snapshot.Name)
if err != nil {
return nil, err
}

if snapshot.Status.ReadyToUse {
return snapshot, nil
}
}
}
}

func (c *SystemBackupController) BackupBackingImage() (map[string]*longhorn.BackupBackingImage, error) {
backingImages, err := c.ds.ListBackingImagesRO()
if err != nil {
Expand Down

0 comments on commit 70ad3d0

Please sign in to comment.