diff --git a/controller/controller_test.go b/controller/controller_test.go index 98ee57dbd9..49a05551e3 100644 --- a/controller/controller_test.go +++ b/controller/controller_test.go @@ -465,6 +465,27 @@ func newBackup(name string) *longhorn.Backup { } } +func newBackupBackingImage(name string) *longhorn.BackupBackingImage { + return &longhorn.BackupBackingImage{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: TestNamespace, + }, + } +} + +func newBackingIamge(name string, sourceType longhorn.BackingImageDataSourceType) *longhorn.BackingImage { + return &longhorn.BackingImage{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: TestNamespace, + }, + Spec: longhorn.BackingImageSpec{ + SourceType: sourceType, + }, + } +} + func newKubernetesNode(name string, readyStatus, diskPressureStatus, memoryStatus, pidStatus, networkStatus, kubeletStatus corev1.ConditionStatus) *corev1.Node { return &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ diff --git a/controller/system_backup_controller.go b/controller/system_backup_controller.go index 4bdd8e40d8..54c759d789 100644 --- a/controller/system_backup_controller.go +++ b/controller/system_backup_controller.go @@ -435,6 +435,18 @@ func (c *SystemBackupController) reconcile(name string, backupTargetClient engin // TODO: handle error check go c.WaitForVolumeBackupToComplete(backups, systemBackup) // nolint: errcheck + case longhorn.SystemBackupStateBackingImageBackup: + backupBackingImages, err := c.BackupBackingImage() + if err != nil { + c.updateSystemBackupRecord(record, + systemBackupRecordTypeError, longhorn.SystemBackupStateError, + constant.EventReasonStart, err.Error(), + ) + return nil + } + + go c.WaitForBackingImageBackupToComplete(backupBackingImages, systemBackup) + case longhorn.SystemBackupStateGenerating: go c.GenerateSystemBackup(systemBackup, tempBackupArchivePath, tempBackupDir) @@ -764,13 +776,13 @@ func (c *SystemBackupController) WaitForVolumeBackupToComplete(backups map[strin systemBackupRecordTypeError, longhorn.SystemBackupStateError, constant.EventReasonStart, err.Error(), ) + } else { + c.updateSystemBackupRecord(record, + systemBackupRecordTypeNormal, longhorn.SystemBackupStateBackingImageBackup, + constant.EventReasonStart, SystemBackupMsgStarting, + ) } - c.updateSystemBackupRecord(record, - systemBackupRecordTypeNormal, longhorn.SystemBackupStateGenerating, - constant.EventReasonStart, SystemBackupMsgStarting, - ) - c.handleStatusUpdate(record, systemBackup, existingSystemBackup, err, log) }() @@ -785,6 +797,7 @@ func (c *SystemBackupController) WaitForVolumeBackupToComplete(backups map[strin backup, err = c.ds.GetBackup(name) if err != nil { if apierrors.IsNotFound(err) { + log.Warnf("Backup %v not found when checking the volume backup status in system backup", name) continue } break @@ -844,6 +857,112 @@ func (c *SystemBackupController) createVolumeBackup(volume *longhorn.Volume, sys return backup, nil } +func (c *SystemBackupController) BackupBackingImage() (map[string]*longhorn.BackupBackingImage, error) { + backingImages, err := c.ds.ListBackingImagesRO() + if err != nil { + return nil, err + } + + backingImageBackups := make(map[string]*longhorn.BackupBackingImage, len(backingImages)) + for _, backingImage := range backingImages { + backupBackingImage, err := c.createBackingImageBackup(backingImage) + if err != nil { + return nil, err + } + + backingImageBackups[backupBackingImage.Name] = backupBackingImage + } + + return backingImageBackups, nil +} + +func (c *SystemBackupController) WaitForBackingImageBackupToComplete(backupBackingImages map[string]*longhorn.BackupBackingImage, systemBackup *longhorn.SystemBackup) { + var err error + log := getLoggerForSystemBackup(c.logger, systemBackup) + + existingSystemBackup := systemBackup.DeepCopy() + defer func() { + record := &systemBackupRecord{} + if err != nil { + c.updateSystemBackupRecord(record, + systemBackupRecordTypeError, longhorn.SystemBackupStateError, + constant.EventReasonStart, err.Error(), + ) + } else { + c.updateSystemBackupRecord(record, + systemBackupRecordTypeNormal, longhorn.SystemBackupStateGenerating, + constant.EventReasonStart, SystemBackupMsgStarting, + ) + } + + c.handleStatusUpdate(record, systemBackup, existingSystemBackup, err, log) + }() + + startTime := time.Now() + ticker := time.NewTicker(time.Second * 5) + defer ticker.Stop() + + for range ticker.C { + for name := range backupBackingImages { + // Retrieve the latest backup + var backupBackingImage *longhorn.BackupBackingImage + backupBackingImage, err = c.ds.GetBackupBackingImageRO(name) + if err != nil { + if apierrors.IsNotFound(err) { + log.Warnf("Backup backing image %v not found when checking the backing image backup status in system backup", name) + continue + } + break + } + + switch backupBackingImage.Status.State { + case longhorn.BackupStateCompleted: + delete(backupBackingImages, name) + case longhorn.BackupStateError: + log.Warnf(errors.Wrapf(fmt.Errorf(backupBackingImage.Status.Error), "Failed to create BackingImage backup %v", name).Error()) + return + } + } + + if len(backupBackingImages) == 0 { + return + } + + // Return when BackingImage backup exceeds timeout + if time.Since(startTime) > datastore.BackingImageBackupTimeout { + log.Warn("Timed out waiting for BackingImage backups to complete") + return + } + } + // This should never be reached. + log.Warn("Unexpected error: stopped waiting for BackingImage backups without completing, failing or timing out") +} + +func (c *SystemBackupController) createBackingImageBackup(backingImage *longhorn.BackingImage) (backupBackingImage *longhorn.BackupBackingImage, err error) { + backupBackingImage, err = c.ds.GetBackupBackingImage(backingImage.Name) + if err != nil { + if !apierrors.IsNotFound(err) { + return nil, errors.Wrapf(err, "failed to get backup backing image %v", backingImage.Name) + } + } + + if backupBackingImage == nil { + backupBackingImage = &longhorn.BackupBackingImage{ + ObjectMeta: metav1.ObjectMeta{ + Name: backingImage.Name, + }, + Spec: longhorn.BackupBackingImageSpec{ + UserCreated: true, + }, + } + backupBackingImage, err = c.ds.CreateBackupBackingImage(backupBackingImage) + if err != nil && !apierrors.IsAlreadyExists(err) { + return nil, errors.Wrapf(err, "failed to create backup backing image %s", backingImage.Name) + } + } + return backupBackingImage, nil +} + func (c *SystemBackupController) generateSystemBackup(systemBackupMeta *systemBackupMeta, tempDir string) (err error) { metaFile := filepath.Join(tempDir, "metadata.yaml") err = util.EncodeToYAMLFile(systemBackupMeta, metaFile) @@ -886,6 +1005,7 @@ func (c *SystemBackupController) generateSystemBackupYAMLsForLonghorn(dir string "engineimages": c.ds.GetAllLonghornEngineImages, "volumes": c.ds.GetAllLonghornVolumes, "recurringjobs": c.ds.GetAllLonghornRecurringJobs, + "backingimages": c.ds.GetAllLonghornBackingImages, } for name, fn := range resourceGetFns { diff --git a/controller/system_backup_controller_test.go b/controller/system_backup_controller_test.go index b771163b27..f7a0fa72ba 100644 --- a/controller/system_backup_controller_test.go +++ b/controller/system_backup_controller_test.go @@ -45,6 +45,7 @@ type SystemBackupTestCase struct { existPersistentVolumes map[SystemRolloutCRName]*corev1.PersistentVolume existVolumes map[SystemRolloutCRName]*longhorn.Volume + existBackingImages map[SystemRolloutCRName]*longhorn.BackingImage expectError bool expectErrorConditionMessage string @@ -87,7 +88,7 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) { }, }, }, - expectState: longhorn.SystemBackupStateGenerating, + expectState: longhorn.SystemBackupStateBackingImageBackup, expectNewVolumBackupCount: 1, }, "system backup create volume backup if-not-present when backup exists": { @@ -100,7 +101,7 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) { }, }, }, - expectState: longhorn.SystemBackupStateGenerating, + expectState: longhorn.SystemBackupStateBackingImageBackup, expectNewVolumBackupCount: 0, }, "system backup create volume backup always": { @@ -113,9 +114,13 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) { }, }, }, - expectState: longhorn.SystemBackupStateGenerating, + expectState: longhorn.SystemBackupStateBackingImageBackup, expectNewVolumBackupCount: 1, }, + "system backup create backingimage backup": { + state: longhorn.SystemBackupStateBackingImageBackup, + expectState: longhorn.SystemBackupStateGenerating, + }, "system backup generate": { state: longhorn.SystemBackupStateGenerating, expectState: longhorn.SystemBackupStateUploading, @@ -229,6 +234,7 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) { fakeSystemRolloutStorageClassesDefault(c, informerFactories.KubeInformerFactory, kubeClient) fakeSystemRolloutVolumes(tc.existVolumes, c, informerFactories.LhInformerFactory, lhClient) + fakeSystemRolloutBackingImages(tc.existBackingImages, c, informerFactories.LhInformerFactory, lhClient) fakeSystemRolloutPersistentVolumes(tc.existPersistentVolumes, c, informerFactories.KubeInformerFactory, kubeClient) systemBackupController, err := newFakeSystemBackupController(lhClient, kubeClient, extensionsClient, informerFactories, tc.controllerID) @@ -257,6 +263,14 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) { err = systemBackupController.WaitForVolumeBackupToComplete(backups, systemBackup) c.Assert(err, IsNil) + case longhorn.SystemBackupStateBackingImageBackup: + backupBackingImages, _ := systemBackupController.BackupBackingImage() + for _, backupBackingImage := range backupBackingImages { + backupBackingImage.Status.State = longhorn.BackupStateCompleted + } + fakeSystemRolloutBackupBackingImages(backupBackingImages, c, informerFactories.LhInformerFactory, lhClient) + systemBackupController.WaitForBackingImageBackupToComplete(backupBackingImages, systemBackup) + case longhorn.SystemBackupStateGenerating: systemBackupController.GenerateSystemBackup(systemBackup, archievePath, tempDir) diff --git a/controller/system_restore_controller_test.go b/controller/system_restore_controller_test.go index 7ed0f9c0c9..c5ddde1c14 100644 --- a/controller/system_restore_controller_test.go +++ b/controller/system_restore_controller_test.go @@ -371,6 +371,41 @@ func fakeSystemRolloutBackups(fakeObjs map[string]*longhorn.Backup, c *C, inform } } +func fakeSystemRolloutBackupBackingImages(fakeObjs map[string]*longhorn.BackupBackingImage, c *C, informerFactory lhinformers.SharedInformerFactory, client *lhfake.Clientset) { + indexer := informerFactory.Longhorn().V1beta2().BackupBackingImages().Informer().GetIndexer() + + clientInterface := client.LonghornV1beta2().BackupBackingImages(TestNamespace) + + exists, err := clientInterface.List(context.TODO(), metav1.ListOptions{}) + c.Assert(err, IsNil) + + for _, exist := range exists.Items { + exist, err := clientInterface.Get(context.TODO(), exist.Name, metav1.GetOptions{}) + c.Assert(err, IsNil) + + err = clientInterface.Delete(context.TODO(), exist.Name, metav1.DeleteOptions{}) + c.Assert(err, IsNil) + + err = indexer.Delete(exist) + c.Assert(err, IsNil) + } + + for k, fakeObj := range fakeObjs { + name := string(k) + if strings.HasSuffix(name, TestIgnoreSuffix) { + continue + } + + backupBackingImage := newBackupBackingImage(name) + backupBackingImage.Status = fakeObj.Status + exist, err := clientInterface.Create(context.TODO(), backupBackingImage, metav1.CreateOptions{}) + c.Assert(err, IsNil) + + err = indexer.Add(exist) + c.Assert(err, IsNil) + } +} + func fakeSystemRolloutDaemonSets(fakeObjs map[SystemRolloutCRName]*appsv1.DaemonSet, c *C, informerFactory informers.SharedInformerFactory, client *fake.Clientset) { indexer := informerFactory.Apps().V1().DaemonSets().Informer().GetIndexer() @@ -685,6 +720,40 @@ func fakeSystemRolloutVolumes(fakeObjs map[SystemRolloutCRName]*longhorn.Volume, } } +func fakeSystemRolloutBackingImages(fakeObjs map[SystemRolloutCRName]*longhorn.BackingImage, c *C, informerFactory lhinformers.SharedInformerFactory, client *lhfake.Clientset) { + indexer := informerFactory.Longhorn().V1beta2().BackingImages().Informer().GetIndexer() + + clientInterface := client.LonghornV1beta2().BackingImages(TestNamespace) + + exists, err := clientInterface.List(context.TODO(), metav1.ListOptions{}) + c.Assert(err, IsNil) + + for _, exist := range exists.Items { + exist, err := clientInterface.Get(context.TODO(), exist.Name, metav1.GetOptions{}) + c.Assert(err, IsNil) + + err = clientInterface.Delete(context.TODO(), exist.Name, metav1.DeleteOptions{}) + c.Assert(err, IsNil) + + err = indexer.Delete(exist) + c.Assert(err, IsNil) + } + + for k, fakeObj := range fakeObjs { + name := string(k) + if strings.HasSuffix(name, TestIgnoreSuffix) { + continue + } + + backingImage := newBackingIamge(name, fakeObj.Spec.SourceType) + exist, err := clientInterface.Create(context.TODO(), backingImage, metav1.CreateOptions{}) + c.Assert(err, IsNil) + + err = indexer.Add(exist) + c.Assert(err, IsNil) + } +} + func fakeSystemRolloutManagerPod(c *C, informerFactory informers.SharedInformerFactory, kubeClient *fake.Clientset) { pIndexer := informerFactory.Core().V1().Pods().Informer().GetIndexer() pod := newDaemonPod(corev1.PodRunning, TestDaemon1, TestNamespace, TestNode1, TestIP1, nil) diff --git a/controller/system_rollout_controller.go b/controller/system_rollout_controller.go index 0d7a147777..ad6610f113 100644 --- a/controller/system_rollout_controller.go +++ b/controller/system_rollout_controller.go @@ -6,6 +6,7 @@ import ( "os/exec" "path/filepath" "reflect" + "strconv" "sync" "time" @@ -36,6 +37,7 @@ import ( v1core "k8s.io/client-go/kubernetes/typed/core/v1" "github.com/longhorn/backupstore" + "github.com/longhorn/backupstore/backupbackingimage" systembackupstore "github.com/longhorn/backupstore/systembackup" @@ -115,6 +117,7 @@ type extractedResources struct { recurringJobList *longhorn.RecurringJobList settingList *longhorn.SettingList volumeList *longhorn.VolumeList + backingImageList *longhorn.BackingImageList } type SystemRolloutController struct { @@ -441,7 +444,7 @@ func (c *SystemRolloutController) systemRollout() error { types.KubernetesKindStorageClassList: c.restoreStorageClasses, types.KubernetesKindConfigMapList: c.restoreConfigMaps, types.KubernetesKindDeploymentList: c.restoreDeployments, - types.LonghornKindVolumeList: c.restoreVolumes, + types.LonghornKindBackingImageList: c.restoreBackingIamges, types.KubernetesKindPersistentVolumeList: c.restorePersistentVolumes, types.KubernetesKindPersistentVolumeClaimList: c.restorePersistentVolumeClaims, types.LonghornKindRecurringJobList: c.restoreRecurringJobs, @@ -456,6 +459,9 @@ func (c *SystemRolloutController) systemRollout() error { } wg.Wait() + // Need to wait until backingimages are restored, so the volumes with backingimages can be restored. + c.restore(types.LonghornKindVolumeList, c.restoreVolumes, log) + if len(c.cacheErrors) == 0 { c.updateSystemRolloutRecord(record, systemRolloutRecordTypeNormal, longhorn.SystemRestoreStateCompleted, @@ -686,6 +692,8 @@ func (c *SystemRolloutController) cacheResourcesFromDirectory(name string, schem c.settingList = obj.(*longhorn.SettingList) case types.LonghornKindVolumeList: c.volumeList = obj.(*longhorn.VolumeList) + case types.LonghornKindBackingImageList: + c.backingImageList = obj.(*longhorn.BackingImageList) default: log := c.getLoggerForSystemRollout() log.Warnf("Unknown resource kind %v", gvk.Kind) @@ -1781,6 +1789,77 @@ func (c *SystemRolloutController) restoreStorageClasses() (err error) { return nil } +func (c *SystemRolloutController) restoreBackingIamges() (err error) { + if c.backingImageList == nil { + return nil + } + + for _, restore := range c.backingImageList.Items { + log := c.logger.WithField(types.LonghornKindBackingImage, restore.Name) + log = getLoggerForBackingImage(log, &restore) + + exist, err := c.ds.GetBackingImage(restore.Name) + if err != nil { + if !datastore.ErrorIsNotFound(err) { + return err + } + + concurrentLimit, err := c.ds.GetSettingAsInt(types.SettingNameBackupConcurrentLimit) + if err != nil { + return errors.Wrapf(err, "failed to get %v value", types.SettingNameBackupConcurrentLimit) + } + // restore by creating backing image with type restore + backingImage := &longhorn.BackingImage{ + ObjectMeta: metav1.ObjectMeta{ + Name: restore.Name, + }, + Spec: longhorn.BackingImageSpec{ + Checksum: restore.Status.Checksum, + SourceType: longhorn.BackingImageDataSourceTypeRestore, + SourceParameters: map[string]string{ + longhorn.DataSourceTypeRestoreParameterBackupURL: backupbackingimage.EncodeBackupBackingImageURL(restore.Name, c.backupTargetURL), + longhorn.DataSourceTypeRestoreParameterConcurrentLimit: strconv.FormatInt(concurrentLimit, 10), + }, + }, + } + + log.Info(SystemRolloutMsgCreating) + + fnCreate := func(backingImage runtime.Object) (runtime.Object, error) { + obj, ok := backingImage.(*longhorn.BackingImage) + if !ok { + return nil, fmt.Errorf(SystemRolloutErrFailedConvertToObjectFmt, restore.GetObjectKind(), types.LonghornKindBackingImage) + } + return c.ds.CreateBackingImage(obj) + } + _, err = c.rolloutResource(backingImage, fnCreate, false, log, SystemRolloutMsgRestoredItem) + if err != nil && !apierrors.IsAlreadyExists(err) { + err = errors.Wrapf(err, SystemRolloutErrFailedToCreateFmt, types.LonghornKindBackingImage, restore.Name) + message := util.CapitalizeFirstLetter(err.Error()) + log.Warn(message) + + reason := fmt.Sprintf(constant.EventReasonFailedCreatingFmt, types.LonghornKindBackingImage, restore.Name) + c.eventRecorder.Event(c.systemRestore, corev1.EventTypeWarning, reason, message) + } + continue + } + + fnUpdate := func(exist runtime.Object) (runtime.Object, error) { + obj, ok := exist.(*longhorn.BackingImage) + if !ok { + return nil, fmt.Errorf(SystemRolloutErrFailedConvertToObjectFmt, exist.GetObjectKind(), types.LonghornKindBackingImage) + } + return c.ds.UpdateBackingImage(obj) + } + _, err = c.rolloutResource(exist, fnUpdate, true, log, SystemRolloutMsgSkipIdentical) + if err != nil { + return err + } + } + + return nil +} + func (c *SystemRolloutController) restoreVolumes() (err error) { if c.engineImageList != nil { for _, restoreEngineImage := range c.engineImageList.Items { diff --git a/controller/system_rollout_controller_test.go b/controller/system_rollout_controller_test.go index 6a5f255c13..38546fe724 100644 --- a/controller/system_rollout_controller_test.go +++ b/controller/system_rollout_controller_test.go @@ -61,6 +61,7 @@ type SystemRolloutTestCase struct { backupSettings map[SystemRolloutCRName]*longhorn.Setting backupStorageClasses map[SystemRolloutCRName]*storagev1.StorageClass backupVolumes map[SystemRolloutCRName]*longhorn.Volume + backupBackingImages map[SystemRolloutCRName]*longhorn.BackingImage existClusterRoles map[SystemRolloutCRName]*rbacv1.ClusterRole existClusterRoleBindings map[SystemRolloutCRName]*rbacv1.ClusterRoleBinding @@ -79,6 +80,7 @@ type SystemRolloutTestCase struct { existSettings map[SystemRolloutCRName]*longhorn.Setting existStorageClasses map[SystemRolloutCRName]*storagev1.StorageClass existVolumes map[SystemRolloutCRName]*longhorn.Volume + existBackingImages map[SystemRolloutCRName]*longhorn.BackingImage expectRestoredClusterRoles map[SystemRolloutCRName]*rbacv1.ClusterRole expectRestoredClusterRoleBindings map[SystemRolloutCRName]*rbacv1.ClusterRoleBinding @@ -97,6 +99,7 @@ type SystemRolloutTestCase struct { expectRestoredSettings map[SystemRolloutCRName]*longhorn.Setting expectRestoredStorageClasses map[SystemRolloutCRName]*storagev1.StorageClass expectRestoredVolumes map[SystemRolloutCRName]*longhorn.Volume + expectRestoredBackingImages map[SystemRolloutCRName]*longhorn.BackingImage expectError string expectErrorConditionMessage string @@ -864,6 +867,57 @@ func (s *TestSuite) TestSystemRollout(c *C) { }, }, }, + "system rollout BackingImage exist in cluster": { + state: longhorn.SystemRestoreStateRestoring, + isInProgress: true, + expectState: longhorn.SystemRestoreStateCompleted, + + existBackingImages: map[SystemRolloutCRName]*longhorn.BackingImage{ + SystemRolloutCRName(TestBackingImage): { + Spec: longhorn.BackingImageSpec{ + SourceType: longhorn.BackingImageDataSourceTypeDownload, + }, + }, + }, + backupBackingImages: map[SystemRolloutCRName]*longhorn.BackingImage{ + SystemRolloutCRName(TestBackingImage): { + Spec: longhorn.BackingImageSpec{ + SourceType: longhorn.BackingImageDataSourceTypeUpload, + }, + }, + }, + expectRestoredBackingImages: map[SystemRolloutCRName]*longhorn.BackingImage{ + SystemRolloutCRName(TestBackingImage): { + Spec: longhorn.BackingImageSpec{ + SourceType: longhorn.BackingImageDataSourceTypeDownload, + }, + }, + }, + }, + "system rollout BackingImage not exist in cluster": { + state: longhorn.SystemRestoreStateRestoring, + isInProgress: true, + expectState: longhorn.SystemRestoreStateCompleted, + + existBackingImages: nil, + // The original sourceType is upload, + // but when restoring, we recreate the BackingImage with type restore. + backupBackingImages: map[SystemRolloutCRName]*longhorn.BackingImage{ + SystemRolloutCRName(TestBackingImage): { + Spec: longhorn.BackingImageSpec{ + SourceType: longhorn.BackingImageDataSourceTypeUpload, + }, + }, + }, + expectRestoredBackingImages: map[SystemRolloutCRName]*longhorn.BackingImage{ + SystemRolloutCRName(TestBackingImage): { + Spec: longhorn.BackingImageSpec{ + SourceType: longhorn.BackingImageDataSourceTypeRestore, + }, + }, + }, + }, + "system rollout Service exist in cluster": { state: longhorn.SystemRestoreStateRestoring, isInProgress: true, @@ -951,6 +1005,7 @@ func (s *TestSuite) TestSystemRollout(c *C) { fakeSystemRolloutServiceAccounts(tc.backupServiceAccounts, c, informerFactories.KubeInformerFactory, kubeClient) fakeSystemRolloutStorageClasses(tc.backupStorageClasses, c, informerFactories.KubeInformerFactory, kubeClient) fakeSystemRolloutVolumes(tc.backupVolumes, c, informerFactories.LhInformerFactory, lhClient) + fakeSystemRolloutBackingImages(tc.backupBackingImages, c, informerFactories.LhInformerFactory, lhClient) ds := datastore.NewDataStore(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) doneCh := make(chan struct{}) @@ -999,6 +1054,7 @@ func (s *TestSuite) TestSystemRollout(c *C) { fakeSystemRolloutServiceAccounts(tc.existServiceAccounts, c, informerFactories.KubeInformerFactory, kubeClient) fakeSystemRolloutSettings(tc.existSettings, c, informerFactories.LhInformerFactory, lhClient) fakeSystemRolloutVolumes(tc.existVolumes, c, informerFactories.LhInformerFactory, lhClient) + fakeSystemRolloutBackingImages(tc.existBackingImages, c, informerFactories.LhInformerFactory, lhClient) if tc.state == longhorn.SystemRestoreStateRestoring { err := controller.Unpack(controller.logger) @@ -1039,6 +1095,7 @@ func (s *TestSuite) TestSystemRollout(c *C) { assertRolloutServices(tc.expectRestoredServices, c, kubeClient) assertRolloutServiceAccounts(tc.expectRestoredServiceAccounts, c, kubeClient) assertRolloutVolumes(tc.expectRestoredVolumes, tc.backupVolumes, c, lhClient) + assertRolloutBackingImages(tc.expectRestoredBackingImages, tc.backupBackingImages, c, lhClient) assertRolloutSettings(tc.expectRestoredSettings, tc.existSettings, c, lhClient) } @@ -1365,6 +1422,14 @@ func (tc *SystemRolloutTestCase) initTestCase() { } tc.backupVolumes[volumeName] = tc.expectRestoredVolumes[volumeName] } + + // init BackingImages + if tc.existBackingImages == nil { + tc.existBackingImages = map[SystemRolloutCRName]*longhorn.BackingImage{} + } + if tc.expectRestoredBackingImages == nil { + tc.expectRestoredBackingImages = tc.existBackingImages + } } func fakeSystemBackupArchieve(c *C, systemBackupName, systemRolloutOwnerID, rolloutControllerID, tempDir, downloadPath string, @@ -1685,6 +1750,28 @@ func assertRolloutVolumes(expectRestored map[SystemRolloutCRName]*longhorn.Volum } } +func assertRolloutBackingImages(expectRestored map[SystemRolloutCRName]*longhorn.BackingImage, backups map[SystemRolloutCRName]*longhorn.BackingImage, c *C, client *lhfake.Clientset) { + objList, err := client.LonghornV1beta2().BackingImages(TestNamespace).List(context.TODO(), metav1.ListOptions{}) + c.Assert(err, IsNil) + c.Assert(len(objList.Items), Equals, len(expectRestored)) + + exists := map[SystemRolloutCRName]longhorn.BackingImage{} + for _, obj := range objList.Items { + exists[SystemRolloutCRName(obj.Name)] = obj + } + + for name, restored := range expectRestored { + exist, found := exists[name] + c.Assert(found, Equals, true) + c.Assert(exist.Spec.SourceType, Equals, restored.Spec.SourceType) + } + + for name := range backups { + _, found := exists[name] + c.Assert(found, Equals, true) + } +} + func assertAnnotateSkippedLastSystemRestore(annos map[string]string, c *C) { _, ok := annos[types.GetLastSkippedSystemRestoreLabelKey()] c.Assert(ok, Equals, true) diff --git a/datastore/datastore.go b/datastore/datastore.go index 4dc709ecf8..a524158dbd 100644 --- a/datastore/datastore.go +++ b/datastore/datastore.go @@ -33,6 +33,9 @@ var ( // VolumeBackupTimeout is the timeout for volume backups VolumeBackupTimeout = 24 * time.Hour + + // BackingImageBackupTimeout is the timeout for backing image backups + BackingImageBackupTimeout = 24 * time.Hour ) // DataStore object diff --git a/datastore/longhorn.go b/datastore/longhorn.go index df75590446..7c3329f126 100644 --- a/datastore/longhorn.go +++ b/datastore/longhorn.go @@ -1936,6 +1936,7 @@ func (s *DataStore) ListBackingImages() (map[string]*longhorn.BackingImage, erro return itemMap, nil } +// ListBackingImagesRO returns object includes all BackingImage in namespace func (s *DataStore) ListBackingImagesRO() ([]*longhorn.BackingImage, error) { return s.backingImageLister.BackingImages(s.namespace).List(labels.Everything()) } diff --git a/datastore/uncached.go b/datastore/uncached.go index 59d1b5e907..771895f093 100644 --- a/datastore/uncached.go +++ b/datastore/uncached.go @@ -392,3 +392,12 @@ func (s *DataStore) GetConfigMapWithoutCache(namespace, name string) (*corev1.Co func (s *DataStore) GetLonghornSnapshotUncached(name string) (*longhorn.Snapshot, error) { return s.lhClient.LonghornV1beta2().Snapshots(s.namespace).Get(context.TODO(), name, metav1.GetOptions{}) } + +// GetAllBackingImages returns an uncached list of BackingImage in Longhorn +// namespace directly from the API server. +// Using cached informers should be preferred but current lister doesn't have a +// field selector. +// Direct retrieval from the API server should only be used for one-shot tasks. +func (s *DataStore) GetAllLonghornBackingImages() (runtime.Object, error) { + return s.lhClient.LonghornV1beta2().BackingImages(s.namespace).List(context.TODO(), metav1.ListOptions{}) +} diff --git a/k8s/pkg/apis/longhorn/v1beta2/systembackup.go b/k8s/pkg/apis/longhorn/v1beta2/systembackup.go index 6bcb9a8e94..a5a007a948 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/systembackup.go +++ b/k8s/pkg/apis/longhorn/v1beta2/systembackup.go @@ -5,14 +5,15 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" type SystemBackupState string const ( - SystemBackupStateDeleting = SystemBackupState("Deleting") - SystemBackupStateError = SystemBackupState("Error") - SystemBackupStateGenerating = SystemBackupState("Generating") - SystemBackupStateNone = SystemBackupState("") - SystemBackupStateReady = SystemBackupState("Ready") - SystemBackupStateSyncing = SystemBackupState("Syncing") - SystemBackupStateUploading = SystemBackupState("Uploading") - SystemBackupStateVolumeBackup = SystemBackupState("CreatingVolumeBackups") + SystemBackupStateDeleting = SystemBackupState("Deleting") + SystemBackupStateError = SystemBackupState("Error") + SystemBackupStateGenerating = SystemBackupState("Generating") + SystemBackupStateNone = SystemBackupState("") + SystemBackupStateReady = SystemBackupState("Ready") + SystemBackupStateSyncing = SystemBackupState("Syncing") + SystemBackupStateUploading = SystemBackupState("Uploading") + SystemBackupStateVolumeBackup = SystemBackupState("CreatingVolumeBackups") + SystemBackupStateBackingImageBackup = SystemBackupState("CreatingBackingImageBackups") SystemBackupConditionTypeError = "Error" diff --git a/types/types.go b/types/types.go index b1089d523e..deb9ccda58 100644 --- a/types/types.go +++ b/types/types.go @@ -47,6 +47,7 @@ const ( LonghornKindRecurringJobList = "RecurringJobList" LonghornKindSettingList = "SettingList" LonghornKindVolumeList = "VolumeList" + LonghornKindBackingImageList = "BackingImageList" KubernetesKindClusterRole = "ClusterRole" KubernetesKindClusterRoleBinding = "ClusterRoleBinding"