feat(system-backup): support backup backing image

controller/controller_test.go

@@ -465,6 +465,27 @@ func newBackup(name string) *longhorn.Backup {
 	}
 }
 
+func newBackupBackingImage(name string) *longhorn.BackupBackingImage {
+	return &longhorn.BackupBackingImage{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: TestNamespace,
+		},
+	}
+}
+
+func newBackingIamge(name string, sourceType longhorn.BackingImageDataSourceType) *longhorn.BackingImage {
+	return &longhorn.BackingImage{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: TestNamespace,
+		},
+		Spec: longhorn.BackingImageSpec{
+			SourceType: sourceType,
+		},
+	}
+}
+
 func newKubernetesNode(name string, readyStatus, diskPressureStatus, memoryStatus, pidStatus, networkStatus, kubeletStatus corev1.ConditionStatus) *corev1.Node {
 	return &corev1.Node{
 		ObjectMeta: metav1.ObjectMeta{

controller/system_backup_controller.go

@@ -435,6 +435,18 @@ func (c *SystemBackupController) reconcile(name string, backupTargetClient engin
 		// TODO: handle error check
 		go c.WaitForVolumeBackupToComplete(backups, systemBackup) // nolint: errcheck
 
+	case longhorn.SystemBackupStateBackingImageBackup:
+		backupBackingImages, err := c.BackupBackingImage()
+		if err != nil {
+			c.updateSystemBackupRecord(record,
+				systemBackupRecordTypeError, longhorn.SystemBackupStateError,
+				constant.EventReasonStart, err.Error(),
+			)
+			return nil
+		}
+
+		go c.WaitForBackingImageBackupToComplete(backupBackingImages, systemBackup)
+
 	case longhorn.SystemBackupStateGenerating:
 		go c.GenerateSystemBackup(systemBackup, tempBackupArchivePath, tempBackupDir)
 
@@ -764,13 +776,13 @@ func (c *SystemBackupController) WaitForVolumeBackupToComplete(backups map[strin
 				systemBackupRecordTypeError, longhorn.SystemBackupStateError,
 				constant.EventReasonStart, err.Error(),
 			)
+		} else {
+			c.updateSystemBackupRecord(record,
+				systemBackupRecordTypeNormal, longhorn.SystemBackupStateBackingImageBackup,
+				constant.EventReasonStart, SystemBackupMsgStarting,
+			)
 		}
 
-		c.updateSystemBackupRecord(record,
-			systemBackupRecordTypeNormal, longhorn.SystemBackupStateGenerating,
-			constant.EventReasonStart, SystemBackupMsgStarting,
-		)
-
 		c.handleStatusUpdate(record, systemBackup, existingSystemBackup, err, log)
 	}()
 
@@ -785,6 +797,7 @@ func (c *SystemBackupController) WaitForVolumeBackupToComplete(backups map[strin
 			backup, err = c.ds.GetBackup(name)
 			if err != nil {
 				if apierrors.IsNotFound(err) {
+					log.Warnf("Backup %v not found when checking the volume backup status in system backup", name)
 					continue
 				}
 				break
@@ -844,6 +857,112 @@ func (c *SystemBackupController) createVolumeBackup(volume *longhorn.Volume, sys
 	return backup, nil
 }
 
+func (c *SystemBackupController) BackupBackingImage() (map[string]*longhorn.BackupBackingImage, error) {
+	backingImages, err := c.ds.ListBackingImagesRO()
+	if err != nil {
+		return nil, err
+	}
+
+	backingImageBackups := make(map[string]*longhorn.BackupBackingImage, len(backingImages))
+	for _, backingImage := range backingImages {
+		backupBackingImage, err := c.createBackingImageBackup(backingImage)
+		if err != nil {
+			return nil, err
+		}
+
+		backingImageBackups[backupBackingImage.Name] = backupBackingImage
+	}
+
+	return backingImageBackups, nil
+}
+
+func (c *SystemBackupController) WaitForBackingImageBackupToComplete(backupBackingImages map[string]*longhorn.BackupBackingImage, systemBackup *longhorn.SystemBackup) {
+	var err error
+	log := getLoggerForSystemBackup(c.logger, systemBackup)
+
+	existingSystemBackup := systemBackup.DeepCopy()
+	defer func() {
+		record := &systemBackupRecord{}
+		if err != nil {
+			c.updateSystemBackupRecord(record,
+				systemBackupRecordTypeError, longhorn.SystemBackupStateError,
+				constant.EventReasonStart, err.Error(),
+			)
+		} else {
+			c.updateSystemBackupRecord(record,
+				systemBackupRecordTypeNormal, longhorn.SystemBackupStateGenerating,
+				constant.EventReasonStart, SystemBackupMsgStarting,
+			)
+		}
+
+		c.handleStatusUpdate(record, systemBackup, existingSystemBackup, err, log)
+	}()
+
+	startTime := time.Now()
+	ticker := time.NewTicker(time.Second * 5)
+	defer ticker.Stop()
+
+	for range ticker.C {
+		for name := range backupBackingImages {
+			// Retrieve the latest backup
+			var backupBackingImage *longhorn.BackupBackingImage
+			backupBackingImage, err = c.ds.GetBackupBackingImageRO(name)
+			if err != nil {
+				if apierrors.IsNotFound(err) {
+					log.Warnf("Backup backing image %v not found when checking the backing image backup status in system backup", name)
+					continue
+				}
+				break
+			}
+
+			switch backupBackingImage.Status.State {
+			case longhorn.BackupStateCompleted:
+				delete(backupBackingImages, name)
+			case longhorn.BackupStateError:
+				log.Warnf(errors.Wrapf(fmt.Errorf(backupBackingImage.Status.Error), "Failed to create BackingImage backup %v", name).Error())
+				return
+			}
+		}
+
+		if len(backupBackingImages) == 0 {
+			return
+		}
+
+		// Return when BackingImage backup exceeds timeout
+		if time.Since(startTime) > datastore.BackingImageBackupTimeout {
+			log.Warn("Timed out waiting for BackingImage backups to complete")
+			return
+		}
+	}
+	// This should never be reached.
+	log.Warn("Unexpected error: stopped waiting for BackingImage backups without completing, failing or timing out")
+}
+
+func (c *SystemBackupController) createBackingImageBackup(backingImage *longhorn.BackingImage) (backupBackingImage *longhorn.BackupBackingImage, err error) {
+	backupBackingImage, err = c.ds.GetBackupBackingImage(backingImage.Name)
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			return nil, errors.Wrapf(err, "failed to get backup backing image %v", backingImage.Name)
+		}
+	}
+
+	if backupBackingImage == nil {
+		backupBackingImage = &longhorn.BackupBackingImage{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: backingImage.Name,
+			},
+			Spec: longhorn.BackupBackingImageSpec{
+				UserCreated: true,
+			},
+		}
+		backupBackingImage, err = c.ds.CreateBackupBackingImage(backupBackingImage)
+		if err != nil && !apierrors.IsAlreadyExists(err) {
+			return nil, errors.Wrapf(err, "failed to create backup backing image %s", backingImage.Name)
+		}
+	}
+	return backupBackingImage, nil
+}
+
 func (c *SystemBackupController) generateSystemBackup(systemBackupMeta *systemBackupMeta, tempDir string) (err error) {
 	metaFile := filepath.Join(tempDir, "metadata.yaml")
 	err = util.EncodeToYAMLFile(systemBackupMeta, metaFile)
@@ -886,6 +1005,7 @@ func (c *SystemBackupController) generateSystemBackupYAMLsForLonghorn(dir string
 		"engineimages":  c.ds.GetAllLonghornEngineImages,
 		"volumes":       c.ds.GetAllLonghornVolumes,
 		"recurringjobs": c.ds.GetAllLonghornRecurringJobs,
+		"backingimages": c.ds.GetAllLonghornBackingImages,
 	}
 
 	for name, fn := range resourceGetFns {

controller/system_backup_controller_test.go

@@ -45,6 +45,7 @@ type SystemBackupTestCase struct {
 
 	existPersistentVolumes map[SystemRolloutCRName]*corev1.PersistentVolume
 	existVolumes           map[SystemRolloutCRName]*longhorn.Volume
+	existBackingImages     map[SystemRolloutCRName]*longhorn.BackingImage
 
 	expectError                 bool
 	expectErrorConditionMessage string
@@ -87,7 +88,7 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) {
 					},
 				},
 			},
-			expectState:               longhorn.SystemBackupStateGenerating,
+			expectState:               longhorn.SystemBackupStateBackingImageBackup,
 			expectNewVolumBackupCount: 1,
 		},
 		"system backup create volume backup if-not-present when backup exists": {
@@ -100,7 +101,7 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) {
 					},
 				},
 			},
-			expectState:               longhorn.SystemBackupStateGenerating,
+			expectState:               longhorn.SystemBackupStateBackingImageBackup,
 			expectNewVolumBackupCount: 0,
 		},
 		"system backup create volume backup always": {
@@ -113,9 +114,13 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) {
 					},
 				},
 			},
-			expectState:               longhorn.SystemBackupStateGenerating,
+			expectState:               longhorn.SystemBackupStateBackingImageBackup,
 			expectNewVolumBackupCount: 1,
 		},
+		"system backup create backingimage backup": {
+			state:       longhorn.SystemBackupStateBackingImageBackup,
+			expectState: longhorn.SystemBackupStateGenerating,
+		},
 		"system backup generate": {
 			state:       longhorn.SystemBackupStateGenerating,
 			expectState: longhorn.SystemBackupStateUploading,
@@ -229,6 +234,7 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) {
 		fakeSystemRolloutStorageClassesDefault(c, informerFactories.KubeInformerFactory, kubeClient)
 
 		fakeSystemRolloutVolumes(tc.existVolumes, c, informerFactories.LhInformerFactory, lhClient)
+		fakeSystemRolloutBackingImages(tc.existBackingImages, c, informerFactories.LhInformerFactory, lhClient)
 		fakeSystemRolloutPersistentVolumes(tc.existPersistentVolumes, c, informerFactories.KubeInformerFactory, kubeClient)
 
 		systemBackupController, err := newFakeSystemBackupController(lhClient, kubeClient, extensionsClient, informerFactories, tc.controllerID)
@@ -257,6 +263,14 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) {
 			err = systemBackupController.WaitForVolumeBackupToComplete(backups, systemBackup)
 			c.Assert(err, IsNil)
 
+		case longhorn.SystemBackupStateBackingImageBackup:
+			backupBackingImages, _ := systemBackupController.BackupBackingImage()
+			for _, backupBackingImage := range backupBackingImages {
+				backupBackingImage.Status.State = longhorn.BackupStateCompleted
+			}
+			fakeSystemRolloutBackupBackingImages(backupBackingImages, c, informerFactories.LhInformerFactory, lhClient)
+			systemBackupController.WaitForBackingImageBackupToComplete(backupBackingImages, systemBackup)
+
 		case longhorn.SystemBackupStateGenerating:
 			systemBackupController.GenerateSystemBackup(systemBackup, archievePath, tempDir)
 

controller/system_restore_controller_test.go

@@ -371,6 +371,41 @@ func fakeSystemRolloutBackups(fakeObjs map[string]*longhorn.Backup, c *C, inform
 	}
 }
 
+func fakeSystemRolloutBackupBackingImages(fakeObjs map[string]*longhorn.BackupBackingImage, c *C, informerFactory lhinformers.SharedInformerFactory, client *lhfake.Clientset) {
+	indexer := informerFactory.Longhorn().V1beta2().BackupBackingImages().Informer().GetIndexer()
+
+	clientInterface := client.LonghornV1beta2().BackupBackingImages(TestNamespace)
+
+	exists, err := clientInterface.List(context.TODO(), metav1.ListOptions{})
+	c.Assert(err, IsNil)
+
+	for _, exist := range exists.Items {
+		exist, err := clientInterface.Get(context.TODO(), exist.Name, metav1.GetOptions{})
+		c.Assert(err, IsNil)
+
+		err = clientInterface.Delete(context.TODO(), exist.Name, metav1.DeleteOptions{})
+		c.Assert(err, IsNil)
+
+		err = indexer.Delete(exist)
+		c.Assert(err, IsNil)
+	}
+
+	for k, fakeObj := range fakeObjs {
+		name := string(k)
+		if strings.HasSuffix(name, TestIgnoreSuffix) {
+			continue
+		}
+
+		backupBackingImage := newBackupBackingImage(name)
+		backupBackingImage.Status = fakeObj.Status
+		exist, err := clientInterface.Create(context.TODO(), backupBackingImage, metav1.CreateOptions{})
+		c.Assert(err, IsNil)
+
+		err = indexer.Add(exist)
+		c.Assert(err, IsNil)
+	}
+}
+
 func fakeSystemRolloutDaemonSets(fakeObjs map[SystemRolloutCRName]*appsv1.DaemonSet, c *C, informerFactory informers.SharedInformerFactory, client *fake.Clientset) {
 	indexer := informerFactory.Apps().V1().DaemonSets().Informer().GetIndexer()
 
@@ -685,6 +720,40 @@ func fakeSystemRolloutVolumes(fakeObjs map[SystemRolloutCRName]*longhorn.Volume,
 	}
 }
 
+func fakeSystemRolloutBackingImages(fakeObjs map[SystemRolloutCRName]*longhorn.BackingImage, c *C, informerFactory lhinformers.SharedInformerFactory, client *lhfake.Clientset) {
+	indexer := informerFactory.Longhorn().V1beta2().BackingImages().Informer().GetIndexer()
+
+	clientInterface := client.LonghornV1beta2().BackingImages(TestNamespace)
+
+	exists, err := clientInterface.List(context.TODO(), metav1.ListOptions{})
+	c.Assert(err, IsNil)
+
+	for _, exist := range exists.Items {
+		exist, err := clientInterface.Get(context.TODO(), exist.Name, metav1.GetOptions{})
+		c.Assert(err, IsNil)
+
+		err = clientInterface.Delete(context.TODO(), exist.Name, metav1.DeleteOptions{})
+		c.Assert(err, IsNil)
+
+		err = indexer.Delete(exist)
+		c.Assert(err, IsNil)
+	}
+
+	for k, fakeObj := range fakeObjs {
+		name := string(k)
+		if strings.HasSuffix(name, TestIgnoreSuffix) {
+			continue
+		}
+
+		backingImage := newBackingIamge(name, fakeObj.Spec.SourceType)
+		exist, err := clientInterface.Create(context.TODO(), backingImage, metav1.CreateOptions{})
+		c.Assert(err, IsNil)
+
+		err = indexer.Add(exist)
+		c.Assert(err, IsNil)
+	}
+}
+
 func fakeSystemRolloutManagerPod(c *C, informerFactory informers.SharedInformerFactory, kubeClient *fake.Clientset) {
 	pIndexer := informerFactory.Core().V1().Pods().Informer().GetIndexer()
 	pod := newDaemonPod(corev1.PodRunning, TestDaemon1, TestNamespace, TestNode1, TestIP1, nil)