From 92e04a044b5f8d2d96fc97e039620de90962626b Mon Sep 17 00:00:00 2001 From: James Lu Date: Mon, 23 Sep 2024 21:54:28 +0800 Subject: [PATCH 01/29] feat(backup): new field of Backup CRD Introduce a new field `BackupTargetName` in `Spec` of Backup CRD to support multiple backup targets. Ref: 5411 Signed-off-by: James Lu --- k8s/crds.yaml | 3 +++ k8s/pkg/apis/longhorn/v1beta2/backup.go | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/k8s/crds.yaml b/k8s/crds.yaml index 783cd2ddd9..3e74373874 100644 --- a/k8s/crds.yaml +++ b/k8s/crds.yaml @@ -862,6 +862,9 @@ spec: - full - incremental - "" + backupTargetName: + description: The backup target name. + nullable: true type: string labels: additionalProperties: diff --git a/k8s/pkg/apis/longhorn/v1beta2/backup.go b/k8s/pkg/apis/longhorn/v1beta2/backup.go index 7578e3517d..01c5d7185f 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/backup.go +++ b/k8s/pkg/apis/longhorn/v1beta2/backup.go @@ -48,6 +48,10 @@ type BackupSpec struct { // Can be "full" or "incremental" // +optional BackupMode BackupMode `json:"backupMode"` + // The backup target name. + // +optional + // +nullable + BackupTargetName string `json:"backupTargetName"` } // BackupStatus defines the observed state of the Longhorn backup From f6062e2d9a377ccae02531f5eebbd0b64480a2cd Mon Sep 17 00:00:00 2001 From: James Lu Date: Mon, 23 Sep 2024 15:02:46 +0800 Subject: [PATCH 02/29] feat(backup): new fields of BackupVolume CRD Introduce new fields `BackupTargetName` and `VolumeName` in `Spec` of BackupVolume CRD to support multiple backup targets. Ref: 5411 Signed-off-by: James Lu --- k8s/crds.yaml | 7 +++++++ k8s/pkg/apis/longhorn/v1beta2/backupvolume.go | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/k8s/crds.yaml b/k8s/crds.yaml index 3e74373874..65587e7425 100644 --- a/k8s/crds.yaml +++ b/k8s/crds.yaml @@ -1274,11 +1274,18 @@ spec: description: BackupVolumeSpec defines the desired state of the Longhorn backup volume properties: + backupTargetName: + description: The backup target name that the backup volume was synced. + nullable: true + type: string syncRequestedAt: description: The time to request run sync the remote backup volume. format: date-time nullable: true type: string + volumeName: + description: The volume name that the backup volume was used to backup. + type: string type: object status: description: BackupVolumeStatus defines the observed state of the Longhorn diff --git a/k8s/pkg/apis/longhorn/v1beta2/backupvolume.go b/k8s/pkg/apis/longhorn/v1beta2/backupvolume.go index 47aeeb3a26..6a1fc73b60 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/backupvolume.go +++ b/k8s/pkg/apis/longhorn/v1beta2/backupvolume.go @@ -4,10 +4,17 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // BackupVolumeSpec defines the desired state of the Longhorn backup volume type BackupVolumeSpec struct { + // The backup target name that the backup volume was synced. + // +optional + // +nullable + BackupTargetName string `json:"backupTargetName"` // The time to request run sync the remote backup volume. // +optional // +nullable SyncRequestedAt metav1.Time `json:"syncRequestedAt"` + // The volume name that the backup volume was used to backup. + // +optional + VolumeName string `json:"volumeName"` } // BackupVolumeStatus defines the observed state of the Longhorn backup volume From 3c838fd4c5e981d4ec119cd40d1229a103fe13f2 Mon Sep 17 00:00:00 2001 From: James Lu Date: Mon, 23 Sep 2024 21:54:45 +0800 Subject: [PATCH 03/29] feat(backup): new field of BackupTarget CRD Introduce a new field `ReadOnly` in `Status` of BackupTarget CRD to support multiple backup targets. Ref: 5411 Signed-off-by: James Lu --- k8s/crds.yaml | 3 +++ k8s/pkg/apis/longhorn/v1beta2/backuptarget.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/k8s/crds.yaml b/k8s/crds.yaml index 65587e7425..718887bcc7 100644 --- a/k8s/crds.yaml +++ b/k8s/crds.yaml @@ -1103,6 +1103,9 @@ spec: description: The interval that the cluster needs to run sync with the backup target. type: string + readOnly: + description: ReadOnly indicates if it can create a backup on the remote backup target or not. + type: boolean syncRequestedAt: description: The time to request run sync the remote backup target. format: date-time diff --git a/k8s/pkg/apis/longhorn/v1beta2/backuptarget.go b/k8s/pkg/apis/longhorn/v1beta2/backuptarget.go index f92cf1ca6c..b76f00f358 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/backuptarget.go +++ b/k8s/pkg/apis/longhorn/v1beta2/backuptarget.go @@ -19,6 +19,9 @@ type BackupTargetSpec struct { // The interval that the cluster needs to run sync with the backup target. // +optional PollInterval metav1.Duration `json:"pollInterval"` + // ReadOnly indicates if it can create a backup on the remote backup target or not. + // +optional + ReadOnly bool `json:"readOnly"` // The time to request run sync the remote backup target. // +optional // +nullable From 29686be2b1bb363e5f312f81143bed3ed6b2a91d Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 25 Sep 2024 14:08:14 +0800 Subject: [PATCH 04/29] feat(backup): new filed of Volume CR Introduce a new filed `BackupTargetName` in `Volume.Spec` to point out where the volume will be backed up to. ref: 5411 Signed-off-by: James Lu --- client/generated_volume.go | 2 ++ k8s/crds.yaml | 3 +++ k8s/pkg/apis/longhorn/v1beta2/volume.go | 3 +++ 3 files changed, 8 insertions(+) diff --git a/client/generated_volume.go b/client/generated_volume.go index 30b96c6089..24831af848 100644 --- a/client/generated_volume.go +++ b/client/generated_volume.go @@ -15,6 +15,8 @@ type Volume struct { BackupStatus []BackupStatus `json:"backupStatus,omitempty" yaml:"backup_status,omitempty"` + BackupTargetName string `json:"backupTargetName,omitempty" yaml:"backup_target_name,omitempty"` + CloneStatus CloneStatus `json:"cloneStatus,omitempty" yaml:"clone_status,omitempty"` Conditions map[string]interface{} `json:"conditions,omitempty" yaml:"conditions,omitempty"` diff --git a/k8s/crds.yaml b/k8s/crds.yaml index 718887bcc7..530ce4e697 100644 --- a/k8s/crds.yaml +++ b/k8s/crds.yaml @@ -4191,6 +4191,9 @@ spec: - lz4 - gzip type: string + backupTargetName: + description: 'The backup target name that the volume will be backed up to or is synced.' + type: string dataEngine: enum: - v1 diff --git a/k8s/pkg/apis/longhorn/v1beta2/volume.go b/k8s/pkg/apis/longhorn/v1beta2/volume.go index 528a4d2870..4ed7bca262 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/volume.go +++ b/k8s/pkg/apis/longhorn/v1beta2/volume.go @@ -302,6 +302,9 @@ type VolumeSpec struct { // Setting that freezes the filesystem on the root partition before a snapshot is created. // +optional FreezeFilesystemForSnapshot FreezeFilesystemForSnapshot `json:"freezeFilesystemForSnapshot"` + // The backup target name that the volume will be backed up to or is synced. + // +optional + BackupTargetName string `json:"backupTargetName"` } // VolumeStatus defines the observed state of the Longhorn volume From 2d6bfb87507670c2ff5b23447c4f42cc647cb762 Mon Sep 17 00:00:00 2001 From: James Lu Date: Mon, 23 Sep 2024 15:42:06 +0800 Subject: [PATCH 05/29] feat(backup): methods to get backup volume CRs List backup volume CRs using a backup target name. List backup volume CRs using a volume name. Get a backup volume using a backup target name and a volume name. Ref: 5411 Signed-off-by: James Lu --- controller/volume_controller.go | 2 +- datastore/longhorn.go | 79 ++++++++++++++++++++++++++++++++- types/types.go | 14 ++++++ 3 files changed, 92 insertions(+), 3 deletions(-) diff --git a/controller/volume_controller.go b/controller/volume_controller.go index af658367e9..9794b3ee75 100644 --- a/controller/volume_controller.go +++ b/controller/volume_controller.go @@ -4742,7 +4742,7 @@ func (c *VolumeController) ReconcileBackupVolumeState(volume *longhorn.Volume) e backupVolumeName = volume.Name } - bv, err := c.ds.GetBackupVolumeRO(backupVolumeName) + bv, err := c.ds.GetBackupVolumeWithBackupTargetAndVolumeRO(volume.Spec.BackupTargetName, backupVolumeName) if err != nil && !apierrors.IsNotFound(err) { return errors.Wrapf(err, "failed to get backup volume %s for volume %v", backupVolumeName, volume.Name) } diff --git a/datastore/longhorn.go b/datastore/longhorn.go index e18d67a985..7fe79780dd 100644 --- a/datastore/longhorn.go +++ b/datastore/longhorn.go @@ -4079,9 +4079,78 @@ func (s *DataStore) ListBackupVolumes() (map[string]*longhorn.BackupVolume, erro return itemMap, nil } -func getBackupVolumeSelector(backupVolumeName string) (labels.Selector, error) { +// ListBackupVolumesWithBackupTargetNameRO returns an object contains all backup volumes in the cluster BackupVolumes CR +// of the given backup target name +func (s *DataStore) ListBackupVolumesWithBackupTargetNameRO(backupTargetName string) (map[string]*longhorn.BackupVolume, error) { + selector, err := getBackupTargetSelector(backupTargetName) + if err != nil { + return nil, err + } + + list, err := s.backupVolumeLister.BackupVolumes(s.namespace).List(selector) + if err != nil { + return nil, err + } + + itemMap := map[string]*longhorn.BackupVolume{} + for _, itemRO := range list { + itemMap[itemRO.Name] = itemRO + } + return itemMap, nil +} + +// ListBackupVolumesWithVolumeNameRO returns an object contains all backup volumes in the cluster BackupVolumes CR +// of the given volume name +func (s *DataStore) ListBackupVolumesWithVolumeNameRO(volumeName string) (map[string]*longhorn.BackupVolume, error) { + selector, err := getBackupVolumeSelector(volumeName) + if err != nil { + return nil, err + } + + list, err := s.backupVolumeLister.BackupVolumes(s.namespace).List(selector) + if err != nil { + return nil, err + } + + itemMap := map[string]*longhorn.BackupVolume{} + for _, itemRO := range list { + itemMap[itemRO.Name] = itemRO + } + return itemMap, nil +} + +// GetBackupVolumeWithBackupTargetAndVolumeRO returns a backup volume object using the given backup target and volume name in the cluster +func (s *DataStore) GetBackupVolumeWithBackupTargetAndVolumeRO(backupTargetName, volumeName string) (*longhorn.BackupVolume, error) { + selector, err := getBackupVolumeWithBackupTargetSelector(backupTargetName, volumeName) + if err != nil { + return nil, err + } + + list, err := s.backupVolumeLister.BackupVolumes(s.namespace).List(selector) + if err != nil { + return nil, err + } + + if len(list) >= 2 { + return nil, fmt.Errorf("datastore: found more than one backup volume with backup target %v and volume %v", backupTargetName, volumeName) + } + + for _, itemRO := range list { + return itemRO, nil + } + + return nil, nil +} + +func getBackupTargetSelector(backupTargetName string) (labels.Selector, error) { return metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ - MatchLabels: types.GetBackupVolumeLabels(backupVolumeName), + MatchLabels: types.GetBackupTargetLabels(backupTargetName), + }) +} + +func getBackupVolumeSelector(volumeName string) (labels.Selector, error) { + return metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + MatchLabels: types.GetBackupVolumeLabels(volumeName), }) } @@ -4090,6 +4159,12 @@ func (s *DataStore) GetBackupVolumeRO(backupVolumeName string) (*longhorn.Backup return s.backupVolumeLister.BackupVolumes(s.namespace).Get(backupVolumeName) } +func getBackupVolumeWithBackupTargetSelector(backupTargetName, volumeName string) (labels.Selector, error) { + return metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + MatchLabels: types.GetBackupVolumeWithBackupTargetLabels(backupTargetName, volumeName), + }) +} + // GetBackupVolume returns a copy of BackupVolume with the given backup volume name in the cluster func (s *DataStore) GetBackupVolume(name string) (*longhorn.BackupVolume, error) { resultRO, err := s.GetBackupVolumeRO(name) diff --git a/types/types.go b/types/types.go index 8acd372c58..a098aee221 100644 --- a/types/types.go +++ b/types/types.go @@ -156,6 +156,7 @@ const ( LonghornLabelManagedBy = "managed-by" LonghornLabelSnapshotForCloningVolume = "for-cloning-volume" LonghornLabelBackingImageDataSource = "backing-image-data-source" + LonghornLabelBackupTarget = "backup-target" LonghornLabelBackupVolume = "backup-volume" LonghornLabelRecurringJob = "job" LonghornLabelRecurringJobGroup = "job-group" @@ -562,6 +563,19 @@ func GetBackingImageDataSourceLabels(name, nodeID, diskUUID string) map[string]s return labels } +func GetBackupVolumeWithBackupTargetLabels(backupTargetName, volumeName string) map[string]string { + return map[string]string{ + LonghornLabelBackupVolume: volumeName, + LonghornLabelBackupTarget: backupTargetName, + } +} + +func GetBackupTargetLabels(backupTargetName string) map[string]string { + return map[string]string{ + LonghornLabelBackupTarget: backupTargetName, + } +} + func GetBackupVolumeLabels(volumeName string) map[string]string { return map[string]string{ LonghornLabelBackupVolume: volumeName, From a1f5a596669e7e3f0663423aa38bdb8d3867df08 Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 26 Sep 2024 11:42:55 +0800 Subject: [PATCH 06/29] feat(backup): backup target mutation Add the backup target mutator and add the finalizer into the backupTarget CR when mutating. ref: 5411 Signed-off-by: James Lu --- datastore/longhorn.go | 20 ++++++++ webhook/resources/backuptarget/mutator.go | 61 +++++++++++++++++++++++ webhook/server/mutation.go | 2 + 3 files changed, 83 insertions(+) create mode 100644 webhook/resources/backuptarget/mutator.go diff --git a/datastore/longhorn.go b/datastore/longhorn.go index 7fe79780dd..8f3b8e9a78 100644 --- a/datastore/longhorn.go +++ b/datastore/longhorn.go @@ -4042,6 +4042,26 @@ func (s *DataStore) DeleteBackupTarget(backupTargetName string) error { return s.lhClient.LonghornV1beta2().BackupTargets(s.namespace).Delete(context.TODO(), backupTargetName, metav1.DeleteOptions{}) } +// RemoveFinalizerForBackupTarget will result in deletion if DeletionTimestamp was set +func (s *DataStore) RemoveFinalizerForBackupTarget(backupTarget *longhorn.BackupTarget) error { + if !util.FinalizerExists(longhornFinalizerKey, backupTarget) { + // finalizer already removed + return nil + } + if err := util.RemoveFinalizer(longhornFinalizerKey, backupTarget); err != nil { + return err + } + _, err := s.lhClient.LonghornV1beta2().BackupTargets(s.namespace).Update(context.TODO(), backupTarget, metav1.UpdateOptions{}) + if err != nil { + // workaround `StorageError: invalid object, Code: 4` due to empty object + if backupTarget.DeletionTimestamp != nil { + return nil + } + return errors.Wrapf(err, "unable to remove finalizer for backup target %s", backupTarget.Name) + } + return nil +} + // CreateBackupVolume creates a Longhorn BackupVolumes CR and verifies creation func (s *DataStore) CreateBackupVolume(backupVolume *longhorn.BackupVolume) (*longhorn.BackupVolume, error) { ret, err := s.lhClient.LonghornV1beta2().BackupVolumes(s.namespace).Create(context.TODO(), backupVolume, metav1.CreateOptions{}) diff --git a/webhook/resources/backuptarget/mutator.go b/webhook/resources/backuptarget/mutator.go new file mode 100644 index 0000000000..684d04bf9e --- /dev/null +++ b/webhook/resources/backuptarget/mutator.go @@ -0,0 +1,61 @@ +package backuptarget + +import ( + "github.com/pkg/errors" + admissionregv1 "k8s.io/api/admissionregistration/v1" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/longhorn/longhorn-manager/datastore" + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" + "github.com/longhorn/longhorn-manager/webhook/admission" + common "github.com/longhorn/longhorn-manager/webhook/common" + werror "github.com/longhorn/longhorn-manager/webhook/error" +) + +type backupTargetMutator struct { + admission.DefaultMutator + ds *datastore.DataStore +} + +func NewMutator(ds *datastore.DataStore) admission.Mutator { + return &backupTargetMutator{ds: ds} +} + +func (b *backupTargetMutator) Resource() admission.Resource { + return admission.Resource{ + Name: "backuptargets", + Scope: admissionregv1.NamespacedScope, + APIGroup: longhorn.SchemeGroupVersion.Group, + APIVersion: longhorn.SchemeGroupVersion.Version, + ObjectType: &longhorn.BackupTarget{}, + OperationTypes: []admissionregv1.OperationType{ + admissionregv1.Create, + admissionregv1.Update, + }, + } +} + +func (b *backupTargetMutator) Create(request *admission.Request, newObj runtime.Object) (admission.PatchOps, error) { + return mutate(newObj) +} + +func (b *backupTargetMutator) Update(request *admission.Request, oldObj runtime.Object, newObj runtime.Object) (admission.PatchOps, error) { + return mutate(newObj) +} + +// mutate contains functionality shared by Create and Update. +func mutate(newObj runtime.Object) (admission.PatchOps, error) { + backupTarget := newObj.(*longhorn.BackupTarget) + var patchOps admission.PatchOps + + patchOp, err := common.GetLonghornFinalizerPatchOpIfNeeded(backupTarget) + if err != nil { + err := errors.Wrapf(err, "failed to get finalizer patch for backupTarget %v", backupTarget.Name) + return nil, werror.NewInvalidError(err.Error(), "") + } + if patchOp != "" { + patchOps = append(patchOps, patchOp) + } + + return patchOps, nil +} diff --git a/webhook/server/mutation.go b/webhook/server/mutation.go index d242ffdea2..55efff4ea1 100644 --- a/webhook/server/mutation.go +++ b/webhook/server/mutation.go @@ -12,6 +12,7 @@ import ( "github.com/longhorn/longhorn-manager/webhook/resources/backingimagemanager" "github.com/longhorn/longhorn-manager/webhook/resources/backup" "github.com/longhorn/longhorn-manager/webhook/resources/backupbackingimage" + "github.com/longhorn/longhorn-manager/webhook/resources/backuptarget" "github.com/longhorn/longhorn-manager/webhook/resources/backupvolume" "github.com/longhorn/longhorn-manager/webhook/resources/engine" "github.com/longhorn/longhorn-manager/webhook/resources/engineimage" @@ -42,6 +43,7 @@ func Mutation(ds *datastore.DataStore) (http.Handler, []admission.Resource, erro engineimage.NewMutator(ds), orphan.NewMutator(ds), sharemanager.NewMutator(ds), + backuptarget.NewMutator(ds), backupvolume.NewMutator(ds), snapshot.NewMutator(ds), replica.NewMutator(ds), From 906cc96e3085e3f128efdf8f905b85cb691192b2 Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 25 Sep 2024 15:48:37 +0800 Subject: [PATCH 07/29] feat(backup): volume mutation and validation for mutiple backup targets Add the volume mutation and validation to handle the field `Spec.BackupTargetName`. Add a new label "backup-target". Ref: 5411 Signed-off-by: James Lu --- webhook/resources/volume/mutator.go | 28 +++++++++++++++++++-------- webhook/resources/volume/validator.go | 21 ++++++++++++++++++++ 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/webhook/resources/volume/mutator.go b/webhook/resources/volume/mutator.go index 37fb9dafab..77b24b28f0 100644 --- a/webhook/resources/volume/mutator.go +++ b/webhook/resources/volume/mutator.go @@ -99,20 +99,22 @@ func (v *volumeMutator) Create(request *admission.Request, newObj runtime.Object moreLabels := map[string]string{} size := volume.Spec.Size + backupTargetName := volume.Spec.BackupTargetName if volume.Spec.FromBackup != "" { - bName, bvName, _, err := backupstore.DecodeBackupURL(volume.Spec.FromBackup) + bName, remoteBVName, _, err := backupstore.DecodeBackupURL(volume.Spec.FromBackup) if err != nil { return nil, werror.NewInvalidError(fmt.Sprintf("cannot get backup and volume name from backup URL %v: %v", volume.Spec.FromBackup, err), "") } - bv, err := v.ds.GetBackupVolumeRO(bvName) + backup, err := v.ds.GetBackupRO(bName) if err != nil { - return nil, werror.NewInvalidError(fmt.Sprintf("cannot get backup volume %s: %v", bvName, err), "") + return nil, werror.NewInvalidError(fmt.Sprintf("cannot get backup %s: %v", bName, err), "") } + backupTargetName = backup.Spec.BackupTargetName - backup, err := v.ds.GetBackupRO(bName) + bv, err := v.ds.GetBackupVolumeWithBackupTargetAndVolumeRO(backupTargetName, remoteBVName) if err != nil { - return nil, werror.NewInvalidError(fmt.Sprintf("cannot get backup %s: %v", bName, err), "") + return nil, werror.NewInvalidError(fmt.Sprintf("cannot get backup volume %s of backup target %s: %v", remoteBVName, backupTargetName, err), "") } if bv != nil && backup != nil && backup.Status.VolumeBackingImageName != "" { @@ -126,7 +128,7 @@ func (v *volumeMutator) Create(request *admission.Request, newObj runtime.Object patchOps = append(patchOps, fmt.Sprintf(`{"op": "replace", "path": "/spec/backingImage", "value": "%s"}`, bv.Status.BackingImageName)) logrus.Debugf("Since the backing image is not specified during the restore, "+ "the previous backing image %v used by backup volume %v will be set for volume %v creation", - bv.Status.BackingImageName, bvName, name) + bv.Status.BackingImageName, remoteBVName, name) } bi, err := v.ds.GetBackingImage(volume.Spec.BackingImage) if err != nil { @@ -150,8 +152,13 @@ func (v *volumeMutator) Create(request *admission.Request, newObj runtime.Object return nil, werror.NewInvalidError(fmt.Sprintf("get invalid size for volume %v: %v", backup.Status.VolumeSize, err), "") } - moreLabels[types.LonghornLabelBackupVolume] = bvName + moreLabels[types.LonghornLabelBackupVolume] = remoteBVName + } + if backupTargetName == "" { + backupTargetName = types.DefaultBackupTargetName } + moreLabels[types.LonghornLabelBackupTarget] = backupTargetName + patchOps = append(patchOps, fmt.Sprintf(`{"op": "replace", "path": "/spec/backupTargetName", "value": "%s"}`, backupTargetName)) // Round up the size to the unit in bytes newSize := util.RoundUpSize(size) @@ -266,9 +273,14 @@ func (v *volumeMutator) Update(request *admission.Request, oldObj runtime.Object patchOps = append(patchOps, fmt.Sprintf(`{"op": "replace", "path": "/spec/snapshotMaxSize", "value": "%s"}`, strconv.FormatInt(volume.Spec.Size*2, 10))) } + moreLabels := map[string]string{} + if oldVolume.Spec.BackupTargetName != volume.Spec.BackupTargetName { + moreLabels[types.LonghornLabelBackupTarget] = volume.Spec.BackupTargetName + } + var patchOpsInCommon admission.PatchOps var err error - if patchOpsInCommon, err = mutate(newObj, nil); err != nil { + if patchOpsInCommon, err = mutate(newObj, moreLabels); err != nil { return nil, err } patchOps = append(patchOps, patchOpsInCommon...) diff --git a/webhook/resources/volume/validator.go b/webhook/resources/volume/validator.go index 9078df91cd..eefcdc58de 100644 --- a/webhook/resources/volume/validator.go +++ b/webhook/resources/volume/validator.go @@ -145,6 +145,10 @@ func (v *volumeValidator) Create(request *admission.Request, newObj runtime.Obje return werror.NewInvalidError(err.Error(), "volume.spec.image") } + if err := v.validateBackupTarget("", volume.Spec.BackupTargetName); err != nil { + return werror.NewInvalidError(err.Error(), "volume.spec.backupTarget") + } + // TODO: remove this check when we support the following features for SPDK volumes if types.IsDataEngineV2(volume.Spec.DataEngine) { if volume.Spec.Encrypted { @@ -322,6 +326,10 @@ func (v *volumeValidator) Update(request *admission.Request, oldObj runtime.Obje return werror.NewInvalidError(err.Error(), "spec.snapshotMaxSize") } + if err := v.validateBackupTarget(oldVolume.Spec.BackupTargetName, newVolume.Spec.BackupTargetName); err != nil { + return werror.NewInvalidError(err.Error(), "spec.backupTarget") + } + if (oldVolume.Spec.SnapshotMaxCount != newVolume.Spec.SnapshotMaxCount) || (oldVolume.Spec.SnapshotMaxSize != newVolume.Spec.SnapshotMaxSize) { if err := v.validateUpdatingSnapshotMaxCountAndSize(oldVolume, newVolume); err != nil { @@ -480,6 +488,19 @@ func validateSnapshotMaxSize(size, snapshotMaxSize int64) error { return nil } +func (v *volumeValidator) validateBackupTarget(oldBackupTarget, newBackupTarget string) error { + if newBackupTarget == "" { + return fmt.Errorf("backup target name is required") + } + if oldBackupTarget == newBackupTarget { + return nil + } + if _, err := v.ds.GetBackupTarget(newBackupTarget); err != nil { + return err + } + return nil +} + func (v *volumeValidator) validateUpdatingSnapshotMaxCountAndSize(oldVolume, newVolume *longhorn.Volume) error { var ( currentSnapshotCount int From f18640247ec25d546d1515d7dd1767e0c1a74e7f Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 25 Sep 2024 09:33:31 +0800 Subject: [PATCH 08/29] feat(backup): new fields of BackupBackingImage CRD Add fields `Spec.BackingImage` and `Spec.BackupTargetName` in the BackupBackingImage CRD Ref: 5411 Signed-off-by: James Lu --- api/backupbackingimage.go | 13 +++++-- api/model.go | 4 ++ k8s/crds.yaml | 8 ++++ .../longhorn/v1beta2/backupbackingimage.go | 7 ++++ manager/backupbackingimage.go | 8 ++-- .../resources/backupbackingimage/mutator.go | 39 ++++++++++++++++++- 6 files changed, 72 insertions(+), 7 deletions(-) diff --git a/api/backupbackingimage.go b/api/backupbackingimage.go index 8df9502208..c9cf7a770f 100644 --- a/api/backupbackingimage.go +++ b/api/backupbackingimage.go @@ -63,9 +63,16 @@ func (s *Server) BackupBackingImageRestore(w http.ResponseWriter, req *http.Requ } func (s *Server) BackupBackingImageCreate(w http.ResponseWriter, req *http.Request) error { - backupBackingImageName := mux.Vars(req)["name"] - if err := s.m.CreateBackupBackingImage(backupBackingImageName); err != nil { - return errors.Wrapf(err, "failed to create backup backing image '%s'", backupBackingImageName) + var input BackupBackingImage + + apiContext := api.GetApiContext(req) + if err := apiContext.Read(&input); err != nil { + return err + } + + backingImageName := mux.Vars(req)["name"] + if err := s.m.CreateBackupBackingImage(input.Name, backingImageName, input.BackupTargetName); err != nil { + return errors.Wrapf(err, "failed to create backup backing image '%s'", input.Name) } return nil } diff --git a/api/model.go b/api/model.go index 60df599c22..60667d2fe2 100644 --- a/api/model.go +++ b/api/model.go @@ -187,6 +187,8 @@ type BackupBackingImage struct { CompressionMethod string `json:"compressionMethod"` Secret string `json:"secret"` SecretNamespace string `json:"secretNamespace"` + BackingImageName string `json:"backingImageName"` + BackupTargetName string `json:"backupTargetName"` } type Setting struct { @@ -1877,6 +1879,8 @@ func toBackupBackingImageResource(bbi *longhorn.BackupBackingImage, apiContext * CompressionMethod: string(bbi.Status.CompressionMethod), Secret: bbi.Status.Secret, SecretNamespace: bbi.Status.SecretNamespace, + BackingImageName: bbi.Spec.BackingImage, + BackupTargetName: bbi.Spec.BackupTargetName, } backupBackingImage.Actions = map[string]string{ diff --git a/k8s/crds.yaml b/k8s/crds.yaml index 530ce4e697..82ea39a155 100644 --- a/k8s/crds.yaml +++ b/k8s/crds.yaml @@ -644,6 +644,14 @@ spec: description: BackupBackingImageSpec defines the desired state of the Longhorn backing image backup properties: + backingImage: + description: The backing image name.. + nullable: true + type: string + backupTargetName: + description: The backup target name. + nullable: true + type: string labels: additionalProperties: type: string diff --git a/k8s/pkg/apis/longhorn/v1beta2/backupbackingimage.go b/k8s/pkg/apis/longhorn/v1beta2/backupbackingimage.go index 3a741e2c62..05b63c02e7 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/backupbackingimage.go +++ b/k8s/pkg/apis/longhorn/v1beta2/backupbackingimage.go @@ -4,6 +4,13 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // BackupBackingImageSpec defines the desired state of the Longhorn backing image backup type BackupBackingImageSpec struct { + // The backing image name. + // Required + BackingImage string `json:"backingImage"` + // The backup target name. + // +optional + // +nullable + BackupTargetName string `json:"backupTargetName"` // The time to request run sync the remote backing image backup. // +optional // +nullable diff --git a/manager/backupbackingimage.go b/manager/backupbackingimage.go index c029c33803..ef14945320 100644 --- a/manager/backupbackingimage.go +++ b/manager/backupbackingimage.go @@ -55,8 +55,8 @@ func (m *VolumeManager) RestoreBackupBackingImage(name string, secret, secretNam return m.restoreBackingImage(name, secret, secretNamespace) } -func (m *VolumeManager) CreateBackupBackingImage(name string) error { - _, err := m.ds.GetBackingImageRO(name) +func (m *VolumeManager) CreateBackupBackingImage(name, backingImageName, backupTargetName string) error { + _, err := m.ds.GetBackingImageRO(backingImageName) if err != nil { return errors.Wrapf(err, "failed to get backing image %v", name) } @@ -77,7 +77,9 @@ func (m *VolumeManager) CreateBackupBackingImage(name string) error { Name: name, }, Spec: longhorn.BackupBackingImageSpec{ - UserCreated: true, + UserCreated: true, + BackingImage: backingImageName, + BackupTargetName: backupTargetName, }, } if _, err = m.ds.CreateBackupBackingImage(backupBackingImage); err != nil && !apierrors.IsAlreadyExists(err) { diff --git a/webhook/resources/backupbackingimage/mutator.go b/webhook/resources/backupbackingimage/mutator.go index e74a92aaa7..b6479f02ec 100644 --- a/webhook/resources/backupbackingimage/mutator.go +++ b/webhook/resources/backupbackingimage/mutator.go @@ -6,10 +6,12 @@ import ( "github.com/pkg/errors" admissionregv1 "k8s.io/api/admissionregistration/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "github.com/longhorn/longhorn-manager/datastore" longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" + "github.com/longhorn/longhorn-manager/types" "github.com/longhorn/longhorn-manager/webhook/admission" common "github.com/longhorn/longhorn-manager/webhook/common" werror "github.com/longhorn/longhorn-manager/webhook/error" @@ -39,7 +41,42 @@ func (b *backupBackingImageMutator) Resource() admission.Resource { } func (b *backupBackingImageMutator) Create(request *admission.Request, newObj runtime.Object) (admission.PatchOps, error) { - return mutate(newObj) + backupBackingImage, ok := newObj.(*longhorn.BackupBackingImage) + if !ok { + return nil, werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.BackupBackingImage", newObj), "") + } + + var patchOps admission.PatchOps + + var err error + if patchOps, err = mutate(newObj); err != nil { + return nil, err + } + + backupTarget, err := b.ds.GetBackupTargetRO(backupBackingImage.Spec.BackupTargetName) + if err != nil { + if !apierrors.IsNotFound(err) { + return nil, werror.NewInvalidError(errors.Wrapf(err, "failed to get backup target of backup backing image").Error(), "") + } + if backupTarget == nil { + backupTarget, err = b.ds.GetDefaultBackupTargetRO() + if err != nil { + return nil, werror.NewInvalidError(errors.Wrapf(err, "failed to get default backup target").Error(), "") + } + } + patchOps = append(patchOps, fmt.Sprintf(`{"op": "replace", "path": "/spec/backupTargetName", "value": %s}`, string(backupTarget.Name))) + } + + moreLabels := backupBackingImage.Labels + moreLabels[types.LonghornLabelBackupTarget] = backupTarget.Name + moreLabels[types.LonghornLabelBackingImage] = backupBackingImage.Spec.BackingImage + patchOp, err := common.GetLonghornLabelsPatchOp(backupBackingImage, moreLabels, nil) + if err != nil { + return nil, werror.NewInvalidError(errors.Wrapf(err, "failed to get labels patch for backupBackingImage %v", backupBackingImage.Name).Error(), "") + } + patchOps = append(patchOps, patchOp) + + return patchOps, nil } func (b *backupBackingImageMutator) Update(request *admission.Request, oldObj runtime.Object, newObj runtime.Object) (admission.PatchOps, error) { From ad132bde8bc8a9e544791cdc60ba0004891be276 Mon Sep 17 00:00:00 2001 From: James Lu Date: Tue, 24 Sep 2024 17:59:34 +0800 Subject: [PATCH 09/29] feat(backup): modify the backup target controller 1. Add the deleting function. 2. Modify the synchronizing backup volumes and backup backing images from a backup target methods. 3. `sets.String` is deprecated and use generic `Set` instead Ref: 5411 Signed-off-by: James Lu --- controller/backup_target_controller.go | 190 +++++++++++++++++-------- datastore/longhorn.go | 21 +++ types/types.go | 8 ++ 3 files changed, 161 insertions(+), 58 deletions(-) diff --git a/controller/backup_target_controller.go b/controller/backup_target_controller.go index b1adc8c9d6..389daaaebd 100644 --- a/controller/backup_target_controller.go +++ b/controller/backup_target_controller.go @@ -84,6 +84,7 @@ func NewBackupTargetController( if _, err = ds.BackupTargetInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: btc.enqueueBackupTarget, UpdateFunc: func(old, cur interface{}) { btc.enqueueBackupTarget(cur) }, + DeleteFunc: btc.enqueueBackupTarget, }); err != nil { return nil, err } @@ -130,10 +131,17 @@ func (btc *BackupTargetController) enqueueEngineImage(obj interface{}) { if err != nil || ei.Spec.Image != defaultEngineImage || ei.Status.State != longhorn.EngineImageStateDeployed { return } - // For now, we only support a default backup target - // We've to enhance it once we support multiple backup targets - // https://github.com/longhorn/longhorn/issues/2317 - btc.queue.Add(ei.Namespace + "/" + types.DefaultBackupTargetName) + + backupTargetMap, err := btc.ds.ListBackupTargetsRO() + if err != nil { + return + } + for backupTargetName, backupTarget := range backupTargetMap { + // Enqueue the backup target only when the backup target is not started. + if backupTarget.Spec.PollInterval.Duration == time.Duration(0) { + btc.queue.Add(ei.Namespace + "/" + backupTargetName) + } + } } func (btc *BackupTargetController) Run(workers int, stopCh <-chan struct{}) { @@ -181,12 +189,6 @@ func (btc *BackupTargetController) syncHandler(key string) (err error) { // Not ours, skip it return nil } - if name != types.DefaultBackupTargetName { - // For now, we only support a default backup target - // We've to enhance it once we support multiple backup targets - // https://github.com/longhorn/longhorn/issues/2317 - return nil - } return btc.reconcile(name) } @@ -214,6 +216,7 @@ func getLoggerForBackupTarget(logger logrus.FieldLogger, backupTarget *longhorn. "url": backupTarget.Spec.BackupTargetURL, "cred": backupTarget.Spec.CredentialSecret, "interval": backupTarget.Spec.PollInterval.Duration, + "name": backupTarget.Name, }, ) } @@ -337,6 +340,18 @@ func (btc *BackupTargetController) reconcile(name string) (err error) { } } + if !backupTarget.DeletionTimestamp.IsZero() { + if err := btc.cleanupBackupVolumes(backupTarget.Name); err != nil { + return errors.Wrap(err, "failed to clean up BackupVolumes") + } + + if err := btc.cleanupBackupBackingImages(backupTarget.Name); err != nil { + return errors.Wrap(err, "failed to clean up BackupBackingImages") + } + + return btc.ds.RemoveFinalizerForBackupTarget(backupTarget) + } + // Check the controller should run synchronization if !backupTarget.Status.LastSyncedAt.IsZero() && !backupTarget.Spec.SyncRequestedAt.After(backupTarget.Status.LastSyncedAt.Time) { @@ -372,16 +387,19 @@ func (btc *BackupTargetController) reconcile(name string) (err error) { backupTarget.Status.Conditions = types.SetCondition(backupTarget.Status.Conditions, longhorn.BackupTargetConditionTypeUnavailable, longhorn.ConditionStatusTrue, longhorn.BackupTargetConditionReasonUnavailable, "backup target URL is empty") - if err := btc.cleanupBackupVolumes(); err != nil { + + if err := btc.cleanupBackupVolumes(backupTarget.Name); err != nil { return errors.Wrap(err, "failed to clean up BackupVolumes") } - if err := btc.cleanupSystemBackups(); err != nil { - return errors.Wrap(err, "failed to clean up SystemBackups") + if err := btc.cleanupBackupBackingImages(backupTarget.Name); err != nil { + return errors.Wrap(err, "failed to clean up BackupBackingImages") } - if err := btc.cleanupBackupBackingImages(); err != nil { - return errors.Wrap(err, "failed to clean up BackupBackingImages") + if backupTarget.Name == types.DefaultBackupTargetName { + if err := btc.cleanupSystemBackups(); err != nil { + return errors.Wrap(err, "failed to clean up SystemBackups") + } } return nil @@ -403,17 +421,20 @@ func (btc *BackupTargetController) reconcile(name string) (err error) { longhorn.BackupTargetConditionTypeUnavailable, longhorn.ConditionStatusFalse, "", "") - if err = btc.syncBackupVolume(info.backupStoreBackupVolumeNames, syncTime, log); err != nil { + if err = btc.syncBackupVolume(backupTarget.Name, info.backupStoreBackupVolumeNames, syncTime, log); err != nil { return err } - if err = btc.syncBackupBackingImage(info.backupStoreBackingImageNames, syncTime, log); err != nil { + if err = btc.syncBackupBackingImage(backupTarget.Name, info.backupStoreBackingImageNames, syncTime, log); err != nil { return err } - if err = btc.syncSystemBackup(info.backupStoreSystemBackups, log); err != nil { - return err + if backupTarget.Name == types.DefaultBackupTargetName { + if err = btc.syncSystemBackup(info.backupStoreSystemBackups, log); err != nil { + return err + } } + return nil } @@ -466,93 +487,139 @@ func (btc *BackupTargetController) getInfoFromBackupStore(backupTarget *longhorn return info, nil } -func (btc *BackupTargetController) syncBackupVolume(backupStoreBackupVolumeNames []string, syncTime metav1.Time, log logrus.FieldLogger) error { - backupStoreBackupVolumes := sets.NewString(backupStoreBackupVolumeNames...) +func (btc *BackupTargetController) syncBackupVolume(backupTargetName string, backupStoreBackupVolumeNames []string, syncTime metav1.Time, log logrus.FieldLogger) error { + backupStoreBackupVolumes := sets.New[string](backupStoreBackupVolumeNames...) // Get a list of all the backup volumes that exist as custom resources in the cluster - clusterBackupVolumes, err := btc.ds.ListBackupVolumes() + clusterBackupVolumes, err := btc.ds.ListBackupVolumesWithBackupTargetNameRO(backupTargetName) if err != nil { return err } - clusterBackupVolumesSet := sets.NewString() - for _, b := range clusterBackupVolumes { - clusterBackupVolumesSet.Insert(b.Name) + clusterVolumeBVMap := make(map[string]*longhorn.BackupVolume, len(clusterBackupVolumes)) + clusterBackupVolumesSet := sets.New[string]() + for _, bv := range clusterBackupVolumes { + clusterBackupVolumesSet.Insert(bv.Spec.VolumeName) + clusterVolumeBVMap[bv.Spec.VolumeName] = bv } - // TODO: add a unit test, separate to a function + // TODO: add a unit test // Get a list of backup volumes that *are* in the backup target and *aren't* in the cluster // and create the BackupVolume CR in the cluster + if err := btc.pullBackupVolumeFromBackupTarget(backupTargetName, backupStoreBackupVolumes, clusterBackupVolumesSet, log); err != nil { + log.WithError(err).Error("Failed to pull backup volumes that do not exist in the cluster") + return err + } + + // TODO: add a unit test + // Get a list of backup volumes that *are* in the cluster and *aren't* in the backup target + // and delete the BackupVolume CR in the cluster + if err := btc.cleanupBackupVolumeNotExistOnBackupTarget(clusterVolumeBVMap, backupStoreBackupVolumes, clusterBackupVolumesSet, log); err != nil { + log.WithError(err).Error("Failed to clean up backup volumes that do not exist on the backup target server") + return err + } + + // Update the BackupVolume CR spec.syncRequestAt to request the + // backup_volume_controller to reconcile the BackupVolume CR + for backupVolumeName, backupVolume := range clusterBackupVolumes { + backupVolume.Spec.SyncRequestedAt = syncTime + if _, err = btc.ds.UpdateBackupVolume(backupVolume); err != nil && !apierrors.IsConflict(errors.Cause(err)) { + log.WithError(err).Errorf("Failed to update backup volume %s spec", backupVolumeName) + } + } + + return nil +} + +func (btc *BackupTargetController) pullBackupVolumeFromBackupTarget(backupTargetName string, backupStoreBackupVolumes, clusterBackupVolumesSet sets.Set[string], log logrus.FieldLogger) (err error) { backupVolumesToPull := backupStoreBackupVolumes.Difference(clusterBackupVolumesSet) if count := backupVolumesToPull.Len(); count > 0 { log.Infof("Found %d backup volumes in the backup target that do not exist in the cluster and need to be pulled", count) } - for backupVolumeName := range backupVolumesToPull { + for volumeName := range backupVolumesToPull { + backupVolumeName := types.GetBackupVolumeNameFromVolumeName(volumeName) backupVolume := &longhorn.BackupVolume{ ObjectMeta: metav1.ObjectMeta{ Name: backupVolumeName, + Labels: map[string]string{ + types.LonghornLabelBackupTarget: backupTargetName, + types.LonghornLabelBackupVolume: volumeName, + }, + }, + Spec: longhorn.BackupVolumeSpec{ + BackupTargetName: backupTargetName, + VolumeName: volumeName, }, } if _, err = btc.ds.CreateBackupVolume(backupVolume); err != nil && !apierrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to create backup volume %s in the cluster", backupVolumeName) } } + return nil +} - // TODO: add a unit test, separate to a function - // Get a list of backup volumes that *are* in the cluster and *aren't* in the backup target - // and delete the BackupVolume CR in the cluster +func (btc *BackupTargetController) cleanupBackupVolumeNotExistOnBackupTarget(clusterVolumeBVMap map[string]*longhorn.BackupVolume, backupStoreBackupVolumes, clusterBackupVolumesSet sets.Set[string], log logrus.FieldLogger) (err error) { backupVolumesToDelete := clusterBackupVolumesSet.Difference(backupStoreBackupVolumes) if count := backupVolumesToDelete.Len(); count > 0 { log.Infof("Found %d backup volumes in the backup target that do not exist in the cluster and need to be deleted from the cluster", count) } - for backupVolumeName := range backupVolumesToDelete { - log.WithField("backupVolume", backupVolumeName).Info("Deleting backup volume from cluster") - if err = btc.ds.DeleteBackupVolume(backupVolumeName); err != nil { - return errors.Wrapf(err, "failed to delete backup volume %s from cluster", backupVolumeName) + + for volumeName := range backupVolumesToDelete { + bv, exists := clusterVolumeBVMap[volumeName] + if !exists { + log.WithField("volume", volumeName).Warnf("Can not find the BackupVolume CR in the cluster backup volume map") + continue } - } - // Update the BackupVolume CR spec.syncRequestAt to request the - // backup_volume_controller to reconcile the BackupVolume CR - for backupVolumeName, backupVolume := range clusterBackupVolumes { - backupVolume.Spec.SyncRequestedAt = syncTime - if _, err = btc.ds.UpdateBackupVolume(backupVolume); err != nil && !apierrors.IsConflict(errors.Cause(err)) { - log.WithError(err).Errorf("Failed to update backup volume %s spec", backupVolumeName) + backupVolumeName := bv.Name + log.WithField("backupVolume", backupVolumeName).Info("Deleting backup volume from cluster") + if err = btc.ds.DeleteBackupVolume(backupVolumeName); err != nil { + log.WithError(err).Errorf("Failed to delete backup volume %s from cluster", backupVolumeName) + return err } } return nil } -func (btc *BackupTargetController) syncBackupBackingImage(backupStoreBackingImageNames []string, syncTime metav1.Time, log logrus.FieldLogger) error { - backupStoreBackupBackingImages := sets.NewString(backupStoreBackingImageNames...) +func (btc *BackupTargetController) syncBackupBackingImage(backupTargetName string, backupStoreBackingImageNames []string, syncTime metav1.Time, log logrus.FieldLogger) error { + backupStoreBackupBackingImages := sets.New[string](backupStoreBackingImageNames...) // Get a list of all the backup volumes that exist as custom resources in the cluster - clusterBackupBackingImages, err := btc.ds.ListBackupBackingImages() + clusterBackupBackingImages, err := btc.ds.ListBackupBackingImagesWithBackupTargetNameRO(backupTargetName) if err != nil { return err } - clusterBackupBackingImagesSet := sets.NewString() - for _, b := range clusterBackupBackingImages { - clusterBackupBackingImagesSet.Insert(b.Name) + clusterBackingImageBBIMap := make(map[string]*longhorn.BackupBackingImage, len(clusterBackupBackingImages)) + clusterBackupBackingImagesSet := sets.New[string]() + for _, bbi := range clusterBackupBackingImages { + clusterBackupBackingImagesSet.Insert(bbi.Spec.BackingImage) + clusterBackingImageBBIMap[bbi.Spec.BackingImage] = bbi } backupBackingImagesToPull := backupStoreBackupBackingImages.Difference(clusterBackupBackingImagesSet) if count := backupBackingImagesToPull.Len(); count > 0 { log.Infof("Found %d backup backing images in the backup target that do not exist in the cluster and need to be pulled", count) } - for backupBackingImageName := range backupBackingImagesToPull { + for backingImageName := range backupBackingImagesToPull { + backupBackingImageName := types.GetBackupBackingImageNameFromBIName(backingImageName) backupBackingImage := &longhorn.BackupBackingImage{ ObjectMeta: metav1.ObjectMeta{ Name: backupBackingImageName, + Labels: map[string]string{ + types.LonghornLabelBackupTarget: backupTargetName, + types.LonghornLabelBackingImage: backingImageName, + }, }, Spec: longhorn.BackupBackingImageSpec{ - UserCreated: false, + UserCreated: false, + BackupTargetName: backupTargetName, + BackingImage: backupBackingImageName, }, } if _, err = btc.ds.CreateBackupBackingImage(backupBackingImage); err != nil && !apierrors.IsAlreadyExists(err) { - return errors.Wrapf(err, "failed to create backup backing image %s in the cluster", backupBackingImageName) + return errors.Wrapf(err, "failed to create backup backing image %s/%s from backup target %s in the cluster", backupBackingImageName, backingImageName, backupTargetName) } } @@ -560,7 +627,14 @@ func (btc *BackupTargetController) syncBackupBackingImage(backupStoreBackingImag if count := backupBackingImagesToDelete.Len(); count > 0 { log.Infof("Found %d backup backing images in the cluster that do not exist in the backup target and need to be deleted", count) } - for backupBackingImageName := range backupBackingImagesToDelete { + for backingImageName := range backupBackingImagesToDelete { + bbi, exists := clusterBackingImageBBIMap[backingImageName] + if !exists { + log.WithField("backing image", backingImageName).Warnf("Can not find the BackupBackingImage CR in the cluster backup backing image map") + continue + } + + backupBackingImageName := bbi.Name log.WithField("backupBackingImage", backupBackingImageName).Info("Deleting backup backing image from cluster") if err = btc.ds.DeleteBackupBackingImage(backupBackingImageName); err != nil { return errors.Wrapf(err, "failed to delete backup backing image %s from cluster", backupBackingImageName) @@ -576,7 +650,7 @@ func (btc *BackupTargetController) syncSystemBackup(backupStoreSystemBackups sys return errors.Wrap(err, "failed to list SystemBackups") } - clusterReadySystemBackupNames := sets.NewString() + clusterReadySystemBackupNames := sets.New[string]() for _, systemBackup := range clusterSystemBackups { if systemBackup.Status.State != longhorn.SystemBackupStateReady { continue @@ -584,7 +658,7 @@ func (btc *BackupTargetController) syncSystemBackup(backupStoreSystemBackups sys clusterReadySystemBackupNames.Insert(systemBackup.Name) } - backupstoreSystemBackupNames := sets.NewString(util.GetSortedKeysFromMap(backupStoreSystemBackups)...) + backupstoreSystemBackupNames := sets.New[string](util.GetSortedKeysFromMap(backupStoreSystemBackups)...) // Create SystemBackup from the system backups in the backup store if not already exist in the cluster. addSystemBackupsToCluster := backupstoreSystemBackupNames.Difference(clusterReadySystemBackupNames) @@ -658,8 +732,8 @@ func (btc *BackupTargetController) isResponsibleFor(bt *longhorn.BackupTarget, d } // cleanupBackupVolumes deletes all BackupVolume CRs -func (btc *BackupTargetController) cleanupBackupVolumes() error { - clusterBackupVolumes, err := btc.ds.ListBackupVolumes() +func (btc *BackupTargetController) cleanupBackupVolumes(backupTargetName string) error { + clusterBackupVolumes, err := btc.ds.ListBackupVolumesWithBackupTargetNameRO(backupTargetName) if err != nil { return err } @@ -677,9 +751,9 @@ func (btc *BackupTargetController) cleanupBackupVolumes() error { return nil } -// cleanupSystemBackups deletes all BackupBackingImage CRs -func (btc *BackupTargetController) cleanupBackupBackingImages() error { - clusterBackupBackingImages, err := btc.ds.ListBackupBackingImages() +// cleanupBackupBackingImages deletes all BackupBackingImage CRs +func (btc *BackupTargetController) cleanupBackupBackingImages(backupTargetName string) error { + clusterBackupBackingImages, err := btc.ds.ListBackupBackingImagesWithBackupTargetNameRO(backupTargetName) if err != nil { return err } diff --git a/datastore/longhorn.go b/datastore/longhorn.go index 8f3b8e9a78..f58bd74fec 100644 --- a/datastore/longhorn.go +++ b/datastore/longhorn.go @@ -5561,6 +5561,27 @@ func (s *DataStore) ListBackupBackingImages() (map[string]*longhorn.BackupBackin return itemMap, nil } +// ListBackupBackingImagesWithBackupTargetNameRO returns object includes all BackupBackingImage in namespace +func (s *DataStore) ListBackupBackingImagesWithBackupTargetNameRO(backupTargetName string) (map[string]*longhorn.BackupBackingImage, error) { + itemMap := map[string]*longhorn.BackupBackingImage{} + + selector, err := getBackupTargetSelector(backupTargetName) + if err != nil { + return nil, err + } + + list, err := s.backupBackingImageLister.BackupBackingImages(s.namespace).List(selector) + if err != nil { + return nil, err + } + + for _, itemRO := range list { + // Cannot use cached object from lister + itemMap[itemRO.Name] = itemRO + } + return itemMap, nil +} + func (s *DataStore) ListBackupBackingImagesRO() ([]*longhorn.BackupBackingImage, error) { return s.backupBackingImageLister.BackupBackingImages(s.namespace).List(labels.Everything()) } diff --git a/types/types.go b/types/types.go index a098aee221..e5380fdce1 100644 --- a/types/types.go +++ b/types/types.go @@ -720,6 +720,14 @@ func GetShareManagerNameFromShareManagerPodName(podName string) string { return strings.TrimPrefix(podName, shareManagerPrefix) } +func GetBackupVolumeNameFromVolumeName(volumeName string) string { + return util.GetStringChecksum(strings.TrimSpace(volumeName))[:util.RandomIDLenth] + "-" + util.RandomID() +} + +func GetBackupBackingImageNameFromBIName(backingImageName string) string { + return util.GetStringChecksum(strings.TrimSpace(backingImageName))[:util.RandomIDLenth] + "-" + util.RandomID() +} + func ValidateEngineImageChecksumName(name string) bool { matched, _ := regexp.MatchString(fmt.Sprintf("^%s[a-fA-F0-9]{%d}$", engineImagePrefix, ImageChecksumNameLength), name) return matched From ac791c1d21193910822a3901cdc6021349d2819d Mon Sep 17 00:00:00 2001 From: James Lu Date: Mon, 23 Sep 2024 22:37:58 +0800 Subject: [PATCH 10/29] feat(backup): move out backup target logic from setting controller Move backupTimer from setting controller to backup target controller Move AWS IAM Role Annotation logic to datastore/kubernetes.go Ref: 5411, 6947 Signed-off-by: James Lu --- controller/backup_target_controller.go | 101 ++++++++++-- controller/setting_controller.go | 215 ++----------------------- datastore/kubernetes.go | 89 ++++++++++ 3 files changed, 192 insertions(+), 213 deletions(-) diff --git a/controller/backup_target_controller.go b/controller/backup_target_controller.go index 389daaaebd..dc167d4c0d 100644 --- a/controller/backup_target_controller.go +++ b/controller/backup_target_controller.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "reflect" "strings" @@ -44,6 +45,9 @@ type BackupTargetController struct { kubeClient clientset.Interface eventRecorder record.EventRecorder + // backup store timer map is responsible for updating the backupTarget.spec.syncRequestAt + bsTimerMap map[string]*BackupStoreTimer + ds *datastore.DataStore cacheSyncs []cache.InformerSynced @@ -51,6 +55,17 @@ type BackupTargetController struct { proxyConnCounter util.Counter } +type BackupStoreTimer struct { + logger logrus.FieldLogger + controllerID string + ds *datastore.DataStore + + btName string + pollInterval time.Duration + ctx context.Context + cancel context.CancelFunc +} + func NewBackupTargetController( logger logrus.FieldLogger, ds *datastore.DataStore, @@ -72,6 +87,8 @@ func NewBackupTargetController( namespace: namespace, controllerID: controllerID, + bsTimerMap: map[string]*BackupStoreTimer{}, + ds: ds, kubeClient: kubeClient, @@ -340,6 +357,15 @@ func (btc *BackupTargetController) reconcile(name string) (err error) { } } + // start a BackupStoreTimer and keep it in a map[string]*BackupStoreTimer + stopTimer := func(backupTargetName string) { + _, exists := btc.bsTimerMap[backupTargetName] + if exists { + btc.bsTimerMap[backupTargetName].Stop() + delete(btc.bsTimerMap, backupTargetName) + } + } + if !backupTarget.DeletionTimestamp.IsZero() { if err := btc.cleanupBackupVolumes(backupTarget.Name); err != nil { return errors.Wrap(err, "failed to clean up BackupVolumes") @@ -349,9 +375,29 @@ func (btc *BackupTargetController) reconcile(name string) (err error) { return errors.Wrap(err, "failed to clean up BackupBackingImages") } + stopTimer(backupTarget.Name) return btc.ds.RemoveFinalizerForBackupTarget(backupTarget) } + if backupTarget.Spec.PollInterval.Duration == time.Duration(0) || (btc.bsTimerMap[name] != nil && btc.bsTimerMap[name].pollInterval != backupTarget.Spec.PollInterval.Duration) { + stopTimer(backupTarget.Name) + } + if btc.bsTimerMap[name] == nil && backupTarget.Spec.PollInterval.Duration != time.Duration(0) && backupTarget.Spec.BackupTargetURL != "" { + // Start backup store sync timer + ctx, cancel := context.WithCancel(context.Background()) + btc.bsTimerMap[name] = &BackupStoreTimer{ + logger: log.WithField("component", "backup-store-timer"), + controllerID: btc.controllerID, + ds: btc.ds, + + btName: name, + pollInterval: backupTarget.Spec.PollInterval.Duration, + ctx: ctx, + cancel: cancel, + } + go btc.bsTimerMap[name].Start() + } + // Check the controller should run synchronization if !backupTarget.Status.LastSyncedAt.IsZero() && !backupTarget.Spec.SyncRequestedAt.After(backupTarget.Status.LastSyncedAt.Time) { @@ -402,6 +448,7 @@ func (btc *BackupTargetController) reconcile(name string) (err error) { } } + stopTimer(backupTarget.Name) return nil } @@ -536,23 +583,23 @@ func (btc *BackupTargetController) pullBackupVolumeFromBackupTarget(backupTarget if count := backupVolumesToPull.Len(); count > 0 { log.Infof("Found %d backup volumes in the backup target that do not exist in the cluster and need to be pulled", count) } - for volumeName := range backupVolumesToPull { - backupVolumeName := types.GetBackupVolumeNameFromVolumeName(volumeName) + for remoteVolumeName := range backupVolumesToPull { + backupVolumeName := types.GetBackupVolumeNameFromVolumeName(remoteVolumeName) backupVolume := &longhorn.BackupVolume{ ObjectMeta: metav1.ObjectMeta{ Name: backupVolumeName, Labels: map[string]string{ types.LonghornLabelBackupTarget: backupTargetName, - types.LonghornLabelBackupVolume: volumeName, + types.LonghornLabelBackupVolume: remoteVolumeName, }, }, Spec: longhorn.BackupVolumeSpec{ BackupTargetName: backupTargetName, - VolumeName: volumeName, + VolumeName: remoteVolumeName, }, } if _, err = btc.ds.CreateBackupVolume(backupVolume); err != nil && !apierrors.IsAlreadyExists(err) { - return errors.Wrapf(err, "failed to create backup volume %s in the cluster", backupVolumeName) + return errors.Wrapf(err, "failed to create backup volume %s/%s in the cluster from backup target %s", backupVolumeName, remoteVolumeName, backupTargetName) } } return nil @@ -602,14 +649,14 @@ func (btc *BackupTargetController) syncBackupBackingImage(backupTargetName strin if count := backupBackingImagesToPull.Len(); count > 0 { log.Infof("Found %d backup backing images in the backup target that do not exist in the cluster and need to be pulled", count) } - for backingImageName := range backupBackingImagesToPull { - backupBackingImageName := types.GetBackupBackingImageNameFromBIName(backingImageName) + for remoteBackingImageName := range backupBackingImagesToPull { + backupBackingImageName := types.GetBackupBackingImageNameFromBIName(remoteBackingImageName) backupBackingImage := &longhorn.BackupBackingImage{ ObjectMeta: metav1.ObjectMeta{ Name: backupBackingImageName, Labels: map[string]string{ types.LonghornLabelBackupTarget: backupTargetName, - types.LonghornLabelBackingImage: backingImageName, + types.LonghornLabelBackingImage: remoteBackingImageName, }, }, Spec: longhorn.BackupBackingImageSpec{ @@ -619,7 +666,7 @@ func (btc *BackupTargetController) syncBackupBackingImage(backupTargetName strin }, } if _, err = btc.ds.CreateBackupBackingImage(backupBackingImage); err != nil && !apierrors.IsAlreadyExists(err) { - return errors.Wrapf(err, "failed to create backup backing image %s/%s from backup target %s in the cluster", backupBackingImageName, backingImageName, backupTargetName) + return errors.Wrapf(err, "failed to create backup backing image %s/%s from backup target %s in the cluster", backupBackingImageName, remoteBackingImageName, backupTargetName) } } @@ -801,3 +848,39 @@ func parseSystemBackupURI(uri string) (version, name string, err error) { return split[len(split)-2], split[len(split)-1], nil } + +func (bst *BackupStoreTimer) Start() { + if bst == nil { + return + } + log := bst.logger.WithFields(logrus.Fields{ + "interval": bst.pollInterval, + }) + log.Info("Starting backup store timer") + + if err := wait.PollUntilContextCancel(bst.ctx, bst.pollInterval, false, func(context.Context) (done bool, err error) { + backupTarget, err := bst.ds.GetBackupTarget(bst.btName) + if err != nil { + bst.logger.WithError(err).Errorf("Failed to get %s backup target", bst.btName) + return false, err + } + + bst.logger.Debug("Triggering sync backup target") + backupTarget.Spec.SyncRequestedAt = metav1.Time{Time: time.Now().UTC()} + if _, err = bst.ds.UpdateBackupTarget(backupTarget); err != nil && !apierrors.IsConflict(errors.Cause(err)) { + bst.logger.WithError(err).Warn("Failed to updating backup target") + } + return false, nil + }); err != nil { + log.WithError(err).Error("Failed to sync backup target") + } + + bst.logger.Infof("Stopped backup store timer") +} + +func (bst *BackupStoreTimer) Stop() { + if bst == nil { + return + } + bst.cancel() +} diff --git a/controller/setting_controller.go b/controller/setting_controller.go index 160c82aff4..5b45bb84f3 100644 --- a/controller/setting_controller.go +++ b/controller/setting_controller.go @@ -73,19 +73,6 @@ type SettingController struct { // upgrade checker lastUpgradeCheckedTimestamp time.Time version string - - // backup store timer is responsible for updating the backupTarget.spec.syncRequestAt - bsTimer *BackupStoreTimer -} - -type BackupStoreTimer struct { - logger logrus.FieldLogger - controllerID string - ds *datastore.DataStore - - pollInterval time.Duration - ctx context.Context - cancel context.CancelFunc } type Version struct { @@ -155,13 +142,6 @@ func NewSettingController( } sc.cacheSyncs = append(sc.cacheSyncs, ds.NodeInformer.HasSynced) - if _, err = ds.BackupTargetInformer.AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{ - DeleteFunc: sc.enqueueSettingForBackupTarget, - }, 0); err != nil { - return nil, err - } - sc.cacheSyncs = append(sc.cacheSyncs, ds.BackupTargetInformer.HasSynced) - return sc, nil } @@ -262,7 +242,7 @@ func (sc *SettingController) syncNonDangerZoneSettingsForManagedComponents(setti return err } case types.SettingNameBackupTarget, types.SettingNameBackupTargetCredentialSecret, types.SettingNameBackupstorePollInterval: - if err := sc.syncBackupTarget(); err != nil { + if err := sc.syncDefaultBackupTarget(); err != nil { return err } case types.SettingNameKubernetesClusterAutoscalerEnabled: @@ -421,28 +401,12 @@ func getResponsibleNodeID(ds *datastore.DataStore) (string, error) { return responsibleNodes[0], nil } -func (sc *SettingController) syncBackupTarget() (err error) { +func (sc *SettingController) syncDefaultBackupTarget() (err error) { defer func() { err = errors.Wrap(err, "failed to sync backup target") }() - stopTimer := func() { - if sc.bsTimer != nil { - sc.bsTimer.Stop() - sc.bsTimer = nil - } - } - - responsibleNodeID, err := getResponsibleNodeID(sc.ds) - if err != nil { - return errors.Wrap(err, "failed to select node for sync backup target") - } - if responsibleNodeID != sc.controllerID { - stopTimer() - return nil - } - - // Get settings + // Get default backup target settings targetSetting, err := sc.ds.GetSettingWithAutoFillingRO(types.SettingNameBackupTarget) if err != nil { return err @@ -482,138 +446,17 @@ func (sc *SettingController) syncBackupTarget() (err error) { } existingBackupTarget := backupTarget.DeepCopy() - defer func() { - backupTarget.Spec.BackupTargetURL = targetSetting.Value - backupTarget.Spec.CredentialSecret = secretSetting.Value - backupTarget.Spec.PollInterval = metav1.Duration{Duration: pollInterval} - if !reflect.DeepEqual(existingBackupTarget.Spec, backupTarget.Spec) { - // Force sync backup target once the BackupTarget spec be updated - backupTarget.Spec.SyncRequestedAt = metav1.Time{Time: time.Now().UTC()} - if _, err = sc.ds.UpdateBackupTarget(backupTarget); err != nil && !apierrors.IsConflict(errors.Cause(err)) { - sc.logger.WithError(err).Warn("Failed to update backup target") - } - if err = sc.handleSecretsForAWSIAMRoleAnnotation(backupTarget.Spec.BackupTargetURL, existingBackupTarget.Spec.CredentialSecret, secretSetting.Value, existingBackupTarget.Spec.BackupTargetURL != targetSetting.Value); err != nil { - sc.logger.WithError(err).Warn("Failed to update secrets for AWSIAMRoleAnnotation") - } - } - }() - - noNeedMonitor := targetSetting.Value == "" || pollInterval == time.Duration(0) - if noNeedMonitor { - stopTimer() - return nil - } - - if sc.bsTimer != nil { - if sc.bsTimer.pollInterval == pollInterval { - // No need to start a new timer if there was one - return - } - // Stop the timer if the poll interval changes - stopTimer() - } - - ctx, cancel := context.WithCancel(context.Background()) - // Start backup store timer - sc.bsTimer = &BackupStoreTimer{ - logger: sc.logger.WithField("component", "backup-store-timer"), - controllerID: sc.controllerID, - ds: sc.ds, - - pollInterval: pollInterval, - ctx: ctx, - cancel: cancel, - } - go sc.bsTimer.Start() - return nil -} - -func (sc *SettingController) handleSecretsForAWSIAMRoleAnnotation(backupTargetURL, oldSecretName, newSecretName string, isBackupTargetURLChanged bool) (err error) { - isSameSecretName := oldSecretName == newSecretName - if isSameSecretName && !isBackupTargetURLChanged { - return nil - } - - isArnExists := false - if !isSameSecretName { - isArnExists, _, err = sc.updateSecretForAWSIAMRoleAnnotation(backupTargetURL, oldSecretName, true) - if err != nil { - return err - } - } - _, isValidSecret, err := sc.updateSecretForAWSIAMRoleAnnotation(backupTargetURL, newSecretName, false) - if err != nil { - return err - } - // kubernetes_secret_controller will not reconcile the secret that does not exist such as named "", so we remove AWS IAM role annotation of pods if new secret name is "". - if !isValidSecret && isArnExists { - if err = sc.removePodsAWSIAMRoleAnnotation(); err != nil { - return err - } - } - return nil -} - -// updateSecretForAWSIAMRoleAnnotation adds the AWS IAM Role annotation to make an update to reconcile in kubernetes secret controller and returns -// -// isArnExists = true if annotation had been added to the secret for first parameter, -// isValidSecret = true if this secret is valid for second parameter. -// err != nil if there is an error occurred. -func (sc *SettingController) updateSecretForAWSIAMRoleAnnotation(backupTargetURL, secretName string, isOldSecret bool) (isArnExists bool, isValidSecret bool, err error) { - if secretName == "" { - return false, false, nil - } - - secret, err := sc.ds.GetSecret(sc.namespace, secretName) - if err != nil { - if apierrors.IsNotFound(err) { - return false, false, nil - } - return false, false, err - } - - if isOldSecret { - delete(secret.Annotations, types.GetLonghornLabelKey(string(types.SettingNameBackupTarget))) - isArnExists = secret.Data[types.AWSIAMRoleArn] != nil - } else { - if secret.Annotations == nil { - secret.Annotations = make(map[string]string) + backupTarget.Spec.BackupTargetURL = targetSetting.Value + backupTarget.Spec.CredentialSecret = secretSetting.Value + backupTarget.Spec.PollInterval = metav1.Duration{Duration: pollInterval} + if !reflect.DeepEqual(existingBackupTarget.Spec, backupTarget.Spec) { + // Force sync backup target once the BackupTarget spec be updated + backupTarget.Spec.SyncRequestedAt = metav1.Time{Time: time.Now().UTC()} + if _, err = sc.ds.UpdateBackupTarget(backupTarget); err != nil && !apierrors.IsConflict(errors.Cause(err)) { + sc.logger.WithError(err).Warn("Failed to update backup target") } - secret.Annotations[types.GetLonghornLabelKey(string(types.SettingNameBackupTarget))] = backupTargetURL } - if _, err = sc.ds.UpdateSecret(sc.namespace, secret); err != nil { - return false, false, err - } - return isArnExists, true, nil -} - -func (sc *SettingController) removePodsAWSIAMRoleAnnotation() error { - managerPods, err := sc.ds.ListManagerPods() - if err != nil { - return err - } - - instanceManagerPods, err := sc.ds.ListInstanceManagerPods() - if err != nil { - return err - } - pods := append(managerPods, instanceManagerPods...) - - for _, pod := range pods { - _, exist := pod.Annotations[types.AWSIAMRoleAnnotation] - if !exist { - continue - } - delete(pod.Annotations, types.AWSIAMRoleAnnotation) - sc.logger.Infof("Removing AWS IAM role for pod %v", pod.Name) - if _, err := sc.ds.UpdatePod(pod); err != nil { - if apierrors.IsNotFound(err) { - continue - } - return err - } - } return nil } @@ -1284,42 +1127,6 @@ func getNotUpdatedNodeSelectorList(newNodeSelector map[string]string, objs ...ru return notUpdatedObjsList, nil } -func (bst *BackupStoreTimer) Start() { - if bst == nil { - return - } - log := bst.logger.WithFields(logrus.Fields{ - "interval": bst.pollInterval, - }) - log.Info("Starting backup store timer") - - if err := wait.PollUntilContextCancel(bst.ctx, bst.pollInterval, false, func(context.Context) (done bool, err error) { - backupTarget, err := bst.ds.GetBackupTarget(types.DefaultBackupTargetName) - if err != nil { - log.WithError(err).Errorf("Failed to get %s backup target", types.DefaultBackupTargetName) - return false, err - } - - log.Debug("Triggering sync backup target") - backupTarget.Spec.SyncRequestedAt = metav1.Time{Time: time.Now().UTC()} - if _, err = bst.ds.UpdateBackupTarget(backupTarget); err != nil && !apierrors.IsConflict(errors.Cause(err)) { - log.WithError(err).Warn("Failed to updating backup target") - } - return false, nil - }); err != nil { - log.WithError(err).Error("Failed to sync backup target") - } - - log.Infof("Stopped backup store timer") -} - -func (bst *BackupStoreTimer) Stop() { - if bst == nil { - return - } - bst.cancel() -} - func (sc *SettingController) syncUpgradeChecker() error { upgradeCheckerEnabled, err := sc.ds.GetSettingAsBool(types.SettingNameUpgradeChecker) if err != nil { diff --git a/datastore/kubernetes.go b/datastore/kubernetes.go index 2142c40501..dd0ff0332b 100644 --- a/datastore/kubernetes.go +++ b/datastore/kubernetes.go @@ -890,6 +890,95 @@ func (s *DataStore) DeleteSecret(namespace, name string) error { return s.kubeClient.CoreV1().Secrets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) } +// HandleSecretsForAWSIAMRoleAnnotation handles AWS IAM Role Annotation when a BackupTarget is created or updated with a Secret. +func (s *DataStore) HandleSecretsForAWSIAMRoleAnnotation(backupTargetURL, oldSecretName, newSecretName string, isBackupTargetURLChanged bool) (err error) { + isSameSecretName := oldSecretName == newSecretName + if isSameSecretName && !isBackupTargetURLChanged { + return nil + } + + isArnExists := false + if !isSameSecretName { + isArnExists, _, err = s.updateSecretForAWSIAMRoleAnnotation(backupTargetURL, oldSecretName, true) + if err != nil { + return err + } + } + _, isValidSecret, err := s.updateSecretForAWSIAMRoleAnnotation(backupTargetURL, newSecretName, false) + if err != nil { + return err + } + // kubernetes_secret_controller will not reconcile the secret that does not exist such as named "", so we remove AWS IAM role annotation of pods if new secret name is "". + if !isValidSecret && isArnExists { + if err = s.removePodsAWSIAMRoleAnnotation(); err != nil { + return err + } + } + return nil +} + +// updateSecretForAWSIAMRoleAnnotation adds the AWS IAM Role annotation to make an update to reconcile in kubernetes secret controller and returns +// +// isArnExists = true if annotation had been added to the secret for first parameter, +// isValidSecret = true if this secret is valid for second parameter. +// err != nil if there is an error occurred. +func (s *DataStore) updateSecretForAWSIAMRoleAnnotation(backupTargetURL, secretName string, isOldSecret bool) (isArnExists bool, isValidSecret bool, err error) { + if secretName == "" { + return false, false, nil + } + + secret, err := s.GetSecret(s.namespace, secretName) + if err != nil { + if apierrors.IsNotFound(err) { + return false, false, nil + } + return false, false, err + } + + if isOldSecret { + delete(secret.Annotations, types.GetLonghornLabelKey(string(types.LonghornLabelBackupTarget))) + isArnExists = secret.Data[types.AWSIAMRoleArn] != nil + } else { + if secret.Annotations == nil { + secret.Annotations = make(map[string]string) + } + secret.Annotations[types.GetLonghornLabelKey(string(types.LonghornLabelBackupTarget))] = backupTargetURL + } + + if _, err = s.UpdateSecret(s.namespace, secret); err != nil { + return false, false, err + } + return isArnExists, true, nil +} + +func (s *DataStore) removePodsAWSIAMRoleAnnotation() error { + managerPods, err := s.ListManagerPods() + if err != nil { + return err + } + + instanceManagerPods, err := s.ListInstanceManagerPods() + if err != nil { + return err + } + pods := append(managerPods, instanceManagerPods...) + + for _, pod := range pods { + _, exist := pod.Annotations[types.AWSIAMRoleAnnotation] + if !exist { + continue + } + delete(pod.Annotations, types.AWSIAMRoleAnnotation) + if _, err := s.UpdatePod(pod); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return err + } + } + return nil +} + // GetPriorityClass gets the PriorityClass from the index for the // given name func (s *DataStore) GetPriorityClass(pcName string) (*schedulingv1.PriorityClass, error) { From 01dbdf57c555ced2c35552f1c7935df325087f7c Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 25 Sep 2024 18:30:02 +0800 Subject: [PATCH 11/29] feat(backup): add backup target validator Ref: 5411 Signed-off-by: James Lu --- controller/uninstall_controller.go | 7 + datastore/longhorn.go | 103 ++++++++++ types/types.go | 4 + webhook/resources/backuptarget/validator.go | 199 ++++++++++++++++++++ webhook/server/validation.go | 4 + 5 files changed, 317 insertions(+) create mode 100644 webhook/resources/backuptarget/validator.go diff --git a/controller/uninstall_controller.go b/controller/uninstall_controller.go index 2b7251367c..64dba2a5cf 100644 --- a/controller/uninstall_controller.go +++ b/controller/uninstall_controller.go @@ -804,7 +804,14 @@ func (c *UninstallController) deleteBackupTargets(backupTargets map[string]*long }() for _, bt := range backupTargets { log := getLoggerForBackupTarget(c.logger, bt) + if bt.Annotations == nil { + bt.Annotations = make(map[string]string) + } if bt.DeletionTimestamp == nil { + bt.Annotations[types.GetLonghornLabelKey(types.DeleteBackupTargetFromLonghorn)] = "" + if _, err := c.ds.UpdateBackupTarget(bt); err != nil { + return errors.Wrap(err, "failed to update backup target annotations to mark for deletion") + } if errDelete := c.ds.DeleteBackupTarget(bt.Name); errDelete != nil { if datastore.ErrorIsNotFound(errDelete) { log.Info("BackupTarget is not found") diff --git a/datastore/longhorn.go b/datastore/longhorn.go index f58bd74fec..7fd37f071d 100644 --- a/datastore/longhorn.go +++ b/datastore/longhorn.go @@ -5,6 +5,7 @@ import ( "fmt" "math/bits" "math/rand" + "net/url" "regexp" "runtime" "strconv" @@ -299,6 +300,9 @@ func (s *DataStore) ValidateSetting(name, value string) (err error) { } return fmt.Errorf("cannot modify BackupTarget since there are existing standby volumes: %v", standbyVolumeNames) } + if err := s.ValidateBackupTargetURL(value); err != nil { + return err + } case types.SettingNameBackupTargetCredentialSecret: secret, err := s.GetSecretRO(s.namespace, value) if err != nil { @@ -3979,6 +3983,20 @@ func (s *DataStore) CreateBackupTarget(backupTarget *longhorn.BackupTarget) (*lo return ret.DeepCopy(), nil } +// ListBackupTargetsRO returns all BackupTargets in the cluster +func (s *DataStore) ListBackupTargetsRO() (map[string]*longhorn.BackupTarget, error) { + list, err := s.backupTargetLister.BackupTargets(s.namespace).List(labels.Everything()) + if err != nil { + return nil, err + } + + itemMap := map[string]*longhorn.BackupTarget{} + for _, itemRO := range list { + itemMap[itemRO.Name] = itemRO + } + return itemMap, nil +} + // ListBackupTargets returns an object contains all backup targets in the cluster BackupTargets CR func (s *DataStore) ListBackupTargets() (map[string]*longhorn.BackupTarget, error) { list, err := s.backupTargetLister.BackupTargets(s.namespace).List(labels.Everything()) @@ -4015,6 +4033,10 @@ func (s *DataStore) GetBackupTarget(name string) (*longhorn.BackupTarget, error) // UpdateBackupTarget updates the given Longhorn backup target in the cluster BackupTargets CR and verifies update func (s *DataStore) UpdateBackupTarget(backupTarget *longhorn.BackupTarget) (*longhorn.BackupTarget, error) { + if backupTarget.Annotations == nil { + backupTarget.Annotations = make(map[string]string) + } + backupTarget.Annotations[types.GetLonghornLabelKey(types.UpdateBackupTargetFromLonghorn)] = "" obj, err := s.lhClient.LonghornV1beta2().BackupTargets(s.namespace).Update(context.TODO(), backupTarget, metav1.UpdateOptions{}) if err != nil { return nil, err @@ -4062,6 +4084,87 @@ func (s *DataStore) RemoveFinalizerForBackupTarget(backupTarget *longhorn.Backup return nil } +// ValidateBackupTargetURL checks if the given backup target URL is already used by other backup targets +func (s *DataStore) ValidateBackupTargetURL(backupTargetURL string) error { + if backupTargetURL == "" { + return nil + } + u, err := url.Parse(backupTargetURL) + if err != nil { + return errors.Wrapf(err, "failed to parse %v as url", backupTargetURL) + } + + if err := checkBackupTargetURLFormat(u); err != nil { + return err + } + if err := s.validateBackupTargetURLExisting(u); err != nil { + return err + } + + return nil +} + +func checkBackupTargetURLFormat(u *url.URL) error { + // Check whether have $ or , have been set in BackupTarget path + regStr := `[\$\,]` + if u.Scheme == "cifs" { + // The $ in SMB/CIFS URIs means that the share is hidden. + regStr = `[\,]` + } + + reg := regexp.MustCompile(regStr) + findStr := reg.FindAllString(u.Path, -1) + if len(findStr) != 0 { + return fmt.Errorf("url %s, contains %v", u.String(), strings.Join(findStr, " or ")) + } + return nil +} + +// validateBackupTargetURLExisting checks if the given backup target URL is already used by other backup targets +func (s *DataStore) validateBackupTargetURLExisting(u *url.URL) error { + newBackupTargetPath, err := getBackupTargetPath(u) + if err != nil { + return err + } + + bts, err := s.ListBackupTargetsRO() + if err != nil { + return err + } + for _, bt := range bts { + existingURL, err := url.Parse(bt.Spec.BackupTargetURL) + if err != nil { + return errors.Wrapf(err, "failed to parse %v as url", bt.Spec.BackupTargetURL) + } + if existingURL.Scheme != u.Scheme { + continue + } + + oldBackupTargetPath, err := getBackupTargetPath(existingURL) + if err != nil { + return err + } + if oldBackupTargetPath == newBackupTargetPath { + return fmt.Errorf("url %s is the same to backup target %v", u.String(), bt.Name) + } + } + + return nil +} + +func getBackupTargetPath(u *url.URL) (backupTargetPath string, err error) { + switch u.Scheme { + case types.BackupStoreTypeAZBlob, types.BackupStoreTypeS3: + backupTargetPath = u.String() + case types.BackupStoreTypeCIFS, types.BackupStoreTypeNFS: + backupTargetPath = strings.TrimRight(u.Host+u.Path, "/") + default: + return "", fmt.Errorf("url %s with the unsupported protocol %v", u.String(), u.Scheme) + } + + return backupTargetPath, nil +} + // CreateBackupVolume creates a Longhorn BackupVolumes CR and verifies creation func (s *DataStore) CreateBackupVolume(backupVolume *longhorn.BackupVolume) (*longhorn.BackupVolume, error) { ret, err := s.lhClient.LonghornV1beta2().BackupVolumes(s.namespace).Create(context.TODO(), backupVolume, metav1.CreateOptions{}) diff --git a/types/types.go b/types/types.go index e5380fdce1..535ed30e16 100644 --- a/types/types.go +++ b/types/types.go @@ -128,6 +128,9 @@ const ( ConfigMapResourceVersionKey = "configmap-resource-version" UpdateSettingFromLonghorn = "update-setting-from-longhorn" + DeleteBackupTargetFromLonghorn = "delete-backup-target-from-longhorn" + UpdateBackupTargetFromLonghorn = "update-backup-target-from-longhorn" + KubernetesStatusLabel = "KubernetesStatus" KubernetesReplicaSet = "ReplicaSet" KubernetesStatefulSet = "StatefulSet" @@ -238,6 +241,7 @@ const ( BackupStoreTypeS3 = "s3" BackupStoreTypeCIFS = "cifs" + BackupStoreTypeNFS = "nfs" BackupStoreTypeAZBlob = "azblob" AWSIAMRoleAnnotation = "iam.amazonaws.com/role" diff --git a/webhook/resources/backuptarget/validator.go b/webhook/resources/backuptarget/validator.go new file mode 100644 index 0000000000..e9329d11d8 --- /dev/null +++ b/webhook/resources/backuptarget/validator.go @@ -0,0 +1,199 @@ +package backuptarget + +import ( + "fmt" + "net/url" + "strings" + + "github.com/pkg/errors" + + admissionregv1 "k8s.io/api/admissionregistration/v1" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/longhorn/longhorn-manager/datastore" + "github.com/longhorn/longhorn-manager/types" + "github.com/longhorn/longhorn-manager/util" + "github.com/longhorn/longhorn-manager/webhook/admission" + + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" + werror "github.com/longhorn/longhorn-manager/webhook/error" +) + +type backupTargetValidator struct { + admission.DefaultValidator + ds *datastore.DataStore +} + +func NewValidator(ds *datastore.DataStore) admission.Validator { + return &backupTargetValidator{ds: ds} +} + +func (b *backupTargetValidator) Resource() admission.Resource { + return admission.Resource{ + Name: "backuptargets", + Scope: admissionregv1.NamespacedScope, + APIGroup: longhorn.SchemeGroupVersion.Group, + APIVersion: longhorn.SchemeGroupVersion.Version, + ObjectType: &longhorn.BackupTarget{}, + OperationTypes: []admissionregv1.OperationType{ + admissionregv1.Create, + admissionregv1.Update, + admissionregv1.Delete, + }, + } +} + +func (b *backupTargetValidator) Create(request *admission.Request, newObj runtime.Object) error { + backupTarget := newObj.(*longhorn.BackupTarget) + + if !util.ValidateName(backupTarget.Name) { + return werror.NewInvalidError(fmt.Sprintf("invalid name %v", backupTarget.Name), "") + } + + if err := b.ds.ValidateBackupTargetURL(backupTarget.Spec.BackupTargetURL); err != nil { + return werror.NewInvalidError(err.Error(), "") + } + + if err := b.validateCredentialSecret(backupTarget.Spec.CredentialSecret); err != nil { + return werror.NewInvalidError(err.Error(), "") + } + + if err := b.handleAWSIAMRoleAnnotation(backupTarget, nil); err != nil { + return werror.NewInvalidError(err.Error(), "") + } + + return nil +} + +func (b *backupTargetValidator) handleAWSIAMRoleAnnotation(newBackupTarget, oldBackupTarget *longhorn.BackupTarget) error { + oldBackupTargetURL := "" + oldBackupTargetSecret := "" + if oldBackupTarget != nil { + oldBackupTargetURL = oldBackupTarget.Spec.BackupTargetURL + oldBackupTargetSecret = oldBackupTarget.Spec.CredentialSecret + } + uOld, err := url.Parse(oldBackupTargetURL) + if err != nil { + return errors.Wrapf(err, "failed to parse %v as url", oldBackupTargetURL) + } + newBackupTargetURL := newBackupTarget.Spec.BackupTargetURL + u, err := url.Parse(newBackupTargetURL) + if err != nil { + return errors.Wrapf(err, "failed to parse %v as url", newBackupTargetURL) + } + if u.Scheme == types.BackupStoreTypeS3 || (uOld.Scheme == types.BackupStoreTypeS3 && newBackupTargetURL == "") { + if err := b.ds.HandleSecretsForAWSIAMRoleAnnotation(newBackupTargetURL, oldBackupTargetSecret, newBackupTarget.Spec.CredentialSecret, oldBackupTargetURL != newBackupTargetURL); err != nil { + return err + } + } + return nil +} + +func (b *backupTargetValidator) Update(request *admission.Request, oldObj runtime.Object, newObj runtime.Object) error { + oldBackupTarget := oldObj.(*longhorn.BackupTarget) + newBackupTarget := newObj.(*longhorn.BackupTarget) + + urlChanged := oldBackupTarget.Spec.BackupTargetURL != newBackupTarget.Spec.BackupTargetURL + secretChanged := oldBackupTarget.Spec.CredentialSecret != newBackupTarget.Spec.CredentialSecret + + if urlChanged { + if err := b.ds.ValidateBackupTargetURL(newBackupTarget.Spec.BackupTargetURL); err != nil { + return werror.NewInvalidError(err.Error(), "") + } + } + + if secretChanged { + if err := b.validateCredentialSecret(newBackupTarget.Spec.CredentialSecret); err != nil { + return werror.NewInvalidError(err.Error(), "") + } + } + + if urlChanged || secretChanged { + if err := b.validateDRVolume(newBackupTarget); err != nil { + return werror.NewInvalidError(err.Error(), "") + } + } + + return nil +} + +func (b *backupTargetValidator) validateCredentialSecret(secretName string) error { + namespace, err := b.ds.GetLonghornNamespace() + if err != nil { + return errors.Wrapf(err, "failed to get Longhorn namespace") + } + secret, err := b.ds.GetSecretRO(namespace.Name, secretName) + if err != nil { + if !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "failed to get the secret before modifying backup target credential secret") + } + return nil + } + checkKeyList := []string{ + types.AWSAccessKey, + types.AWSIAMRoleAnnotation, + types.AWSIAMRoleArn, + types.AWSAccessKey, + types.AWSSecretKey, + types.AWSEndPoint, + types.AWSCert, + types.CIFSUsername, + types.CIFSPassword, + types.AZBlobAccountName, + types.AZBlobAccountKey, + types.AZBlobEndpoint, + types.AZBlobCert, + types.HTTPSProxy, + types.HTTPProxy, + types.NOProxy, + types.VirtualHostedStyle, + } + for _, checkKey := range checkKeyList { + if value, ok := secret.Data[checkKey]; ok { + if strings.TrimSpace(string(value)) != string(value) { + return fmt.Errorf("there is space or new line in %s", checkKey) + } + } + } + return nil +} + +func (b *backupTargetValidator) validateDRVolume(backupTarget *longhorn.BackupTarget) error { + vs, err := b.ds.ListDRVolumesRO() + if err != nil { + return errors.Wrapf(err, "failed to list standby volume when modifying BackupTarget") + } + + backupTargetURL := backupTarget.Spec.BackupTargetURL + if len(vs) != 0 { + standbyVolumeNames := sets.NewString() + for k, v := range vs { + specCharIndex := strings.Index(v.Spec.FromBackup, "?") + fromBackupURL := v.Spec.FromBackup[:specCharIndex] + fromBackupTargetName, isFound := v.Labels[types.LonghornLabelBackupTarget] + if (isFound && fromBackupTargetName == backupTarget.Name) || fromBackupURL == backupTargetURL { + standbyVolumeNames.Insert(k) + } + } + if standbyVolumeNames.Len() != 0 { + return fmt.Errorf("cannot modify BackupTarget since there are existing standby volumes: %v", standbyVolumeNames) + } + } + return nil +} + +func (b *backupTargetValidator) Delete(request *admission.Request, oldObj runtime.Object) error { + backupTarget := oldObj.(*longhorn.BackupTarget) + + if backupTarget.Annotations == nil { + backupTarget.Annotations = make(map[string]string) + } + _, exists := backupTarget.Annotations[types.GetLonghornLabelKey(types.DeleteBackupTargetFromLonghorn)] + if backupTarget.Name == types.DefaultBackupTargetName && !exists { + return werror.NewInvalidError("prohibit deleting a default backup target directly", "") + } + return nil +} diff --git a/webhook/server/validation.go b/webhook/server/validation.go index a664004eaf..695935af3d 100644 --- a/webhook/server/validation.go +++ b/webhook/server/validation.go @@ -10,6 +10,8 @@ import ( "github.com/longhorn/longhorn-manager/util" "github.com/longhorn/longhorn-manager/webhook/admission" "github.com/longhorn/longhorn-manager/webhook/resources/backingimage" + "github.com/longhorn/longhorn-manager/webhook/resources/backup" + "github.com/longhorn/longhorn-manager/webhook/resources/backuptarget" "github.com/longhorn/longhorn-manager/webhook/resources/engine" "github.com/longhorn/longhorn-manager/webhook/resources/instancemanager" "github.com/longhorn/longhorn-manager/webhook/resources/node" @@ -37,6 +39,8 @@ func Validation(ds *datastore.DataStore) (http.Handler, []admission.Resource, er setting.NewValidator(ds), recurringjob.NewValidator(ds), backingimage.NewValidator(ds), + backup.NewValidator(ds), + backuptarget.NewValidator(ds), volume.NewValidator(ds, currentNodeID), orphan.NewValidator(ds), snapshot.NewValidator(ds), From be817a7588eca8322de0d3d410a0b1f14d61c056 Mon Sep 17 00:00:00 2001 From: James Lu Date: Mon, 23 Sep 2024 23:11:58 +0800 Subject: [PATCH 12/29] feat(backup): modify setting and uninstall controllers Remove adding `backup-target` back when node is updated. Remove backup targets when uninstalling. ref: 5411 Signed-off-by: James Lu --- controller/setting_controller.go | 8 -------- controller/uninstall_controller.go | 13 +++++-------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/controller/setting_controller.go b/controller/setting_controller.go index 5b45bb84f3..0dd89252d9 100644 --- a/controller/setting_controller.go +++ b/controller/setting_controller.go @@ -1275,14 +1275,6 @@ func (sc *SettingController) enqueueSettingForNode(obj interface{}) { } sc.queue.Add(sc.namespace + "/" + string(types.SettingNameGuaranteedInstanceManagerCPU)) - sc.queue.Add(sc.namespace + "/" + string(types.SettingNameBackupTarget)) -} - -func (sc *SettingController) enqueueSettingForBackupTarget(obj interface{}) { - if _, ok := obj.(*longhorn.BackupTarget); !ok { - return - } - sc.queue.Add(sc.namespace + "/" + string(types.SettingNameBackupTarget)) } // updateInstanceManagerCPURequest deletes all instance manager pods immediately with the updated CPU request. diff --git a/controller/uninstall_controller.go b/controller/uninstall_controller.go index 64dba2a5cf..2f3de6df37 100644 --- a/controller/uninstall_controller.go +++ b/controller/uninstall_controller.go @@ -527,15 +527,12 @@ func (c *UninstallController) deleteCRs() (bool, error) { // Remove the setting CRs and add OwnerReferences on BackupTarget CR. // After that when deleting setting CRs, the default BackupTarget CR // be cascading deleted automatically. - targetSetting, err := c.ds.GetSetting(types.SettingNameBackupTarget) - if err != nil { + // Delete the BackupTarget CRs + if backupTargets, err := c.ds.ListBackupTargets(); err != nil { return true, err - } - if targetSetting.Value != "" { - targetSetting.Value = "" - if _, err := c.ds.UpdateSetting(targetSetting); err != nil { - return true, err - } + } else if len(backupTargets) > 0 { + c.logger.Infof("Found %d backuptargets remaining", len(backupTargets)) + return true, c.deleteBackupTargets(backupTargets) } // Waits the BackupVolume CRs be clean up by backup_target_controller From 4ad096a04e4db3f83ea65fb623f4c6a55430ce0e Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 25 Sep 2024 15:55:22 +0800 Subject: [PATCH 13/29] feat(backup): new filed of Volume CR Introduce a new filed `BackupTargetName` in `Volume.Spec` to point out where the volume will be backed up to. ref: 5411 Signed-off-by: James Lu --- api/model.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/model.go b/api/model.go index 60667d2fe2..9d7e300f79 100644 --- a/api/model.go +++ b/api/model.go @@ -61,6 +61,7 @@ type Volume struct { SnapshotMaxCount int `json:"snapshotMaxCount"` SnapshotMaxSize string `json:"snapshotMaxSize"` FreezeFilesystemForSnapshot longhorn.FreezeFilesystemForSnapshot `json:"freezeFilesystemForSnapshot"` + BackupTargetName string `json:"backupTargetName"` DiskSelector []string `json:"diskSelector"` NodeSelector []string `json:"nodeSelector"` @@ -1564,6 +1565,7 @@ func toVolumeResource(v *longhorn.Volume, ves []*longhorn.Engine, vrs []*longhor NodeSelector: v.Spec.NodeSelector, RestoreVolumeRecurringJob: v.Spec.RestoreVolumeRecurringJob, FreezeFilesystemForSnapshot: v.Spec.FreezeFilesystemForSnapshot, + BackupTargetName: v.Spec.BackupTargetName, State: v.Status.State, Robustness: v.Status.Robustness, From fa1a4ddbcb95b8f6732c8273b8b8dc56ad729fe5 Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 25 Sep 2024 16:18:33 +0800 Subject: [PATCH 14/29] feat(backup): modify backup volume controller Modify backup volume controller to support multiple backup stores support. Synchronize and handle the backup volumes information from different backup targets with the same backup volume name or not. Ref: 5411 Signed-off-by: James Lu --- api/volume.go | 2 +- controller/backup_target_controller.go | 5 ++++ controller/backup_volume_controller.go | 39 +++++++++++++++----------- datastore/longhorn.go | 18 ++++++------ manager/engine.go | 6 +++- 5 files changed, 43 insertions(+), 27 deletions(-) diff --git a/api/volume.go b/api/volume.go index 7168eaef55..c1b15cd886 100644 --- a/api/volume.go +++ b/api/volume.go @@ -104,7 +104,7 @@ func (s *Server) responseWithVolume(rw http.ResponseWriter, req *http.Request, i if err != nil { return err } - backups, err := s.m.ListBackupsForVolumeSorted(id) + backups, err := s.m.ListBackupsForVolumeSorted(v.Name) if err != nil { return err } diff --git a/controller/backup_target_controller.go b/controller/backup_target_controller.go index dc167d4c0d..6789a610a7 100644 --- a/controller/backup_target_controller.go +++ b/controller/backup_target_controller.go @@ -884,3 +884,8 @@ func (bst *BackupStoreTimer) Stop() { } bst.cancel() } + +// IsBackupTargetAvailable returns a boolean that the backup target is available for true and not available for false +func IsBackupTargetAvailable(backupTarget *longhorn.BackupTarget) bool { + return backupTarget != nil && backupTarget.DeletionTimestamp == nil && backupTarget.Spec.BackupTargetURL != "" && backupTarget.Status.Available +} diff --git a/controller/backup_volume_controller.go b/controller/backup_volume_controller.go index cf64b39152..70ec665f30 100644 --- a/controller/backup_volume_controller.go +++ b/controller/backup_volume_controller.go @@ -212,24 +212,25 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error log := getLoggerForBackupVolume(bvc.logger, backupVolume) - // Get default backup target - backupTarget, err := bvc.ds.GetBackupTargetRO(types.DefaultBackupTargetName) - if err != nil { - if apierrors.IsNotFound(err) { - return nil - } - return errors.Wrapf(err, "failed to get %s backup target", types.DefaultBackupTargetName) + // Get the backup target of the backup volume + backupTarget, err := bvc.ds.GetBackupTargetRO(backupVolume.Spec.BackupTargetName) + if err != nil && !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "failed to get %s backup target", backupVolume.Spec.BackupTargetName) + } + if backupTarget == nil && backupVolume.DeletionTimestamp == nil { + return fmt.Errorf("Failed to find the backup target %v for the backup volume %v", backupVolume.Spec.BackupTargetName, backupVolume.Name) } + backupTargetName := backupTarget.Name + remoteBackupVolumeName := backupVolume.Spec.VolumeName // Examine DeletionTimestamp to determine if object is under deletion if !backupVolume.DeletionTimestamp.IsZero() { - - if err := bvc.ds.DeleteAllBackupsForBackupVolume(backupVolumeName); err != nil { + if err := bvc.ds.DeleteAllBackupsForBackupVolumeWithBackupTarget(backupTargetName, remoteBackupVolumeName); err != nil { return errors.Wrap(err, "failed to delete backups") } // Delete the backup volume from the remote backup target - if backupTarget.Spec.BackupTargetURL != "" { + if IsBackupTargetAvailable(backupTarget) { engineClientProxy, backupTargetClient, err := getBackupTarget(bvc.controllerID, backupTarget, bvc.ds, log, bvc.proxyConnCounter) if err != nil || engineClientProxy == nil { log.WithError(err).Error("Failed to init backup target clients") @@ -237,7 +238,7 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error } defer engineClientProxy.Close() - if err := backupTargetClient.BackupVolumeDelete(backupTargetClient.URL, backupVolumeName, backupTargetClient.Credential); err != nil { + if err := backupTargetClient.BackupVolumeDelete(backupTargetClient.URL, remoteBackupVolumeName, backupTargetClient.Credential); err != nil { return errors.Wrap(err, "failed to delete remote backup volume") } } @@ -273,7 +274,7 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error defer engineClientProxy.Close() // Get a list of all the backups that are stored in the backup target - res, err := backupTargetClient.BackupNameList(backupTargetClient.URL, backupVolumeName, backupTargetClient.Credential) + res, err := backupTargetClient.BackupNameList(backupTargetClient.URL, remoteBackupVolumeName, backupTargetClient.Credential) if err != nil { log.WithError(err).Error("Failed to list backups from backup target") return nil // Ignore error to prevent enqueue @@ -281,7 +282,7 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error backupStoreBackups := sets.NewString(res...) // Get a list of all the backups that exist as custom resources in the cluster - clusterBackups, err := bvc.ds.ListBackupsWithBackupVolumeName(backupVolumeName) + clusterBackups, err := bvc.ds.ListBackupsWithBackupVolumeNameRO(backupTargetName, remoteBackupVolumeName) if err != nil { log.WithError(err).Error("Failed to list backups in the cluster") return err @@ -319,7 +320,7 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error for backupName := range backupsToPull { backupLabelMap := map[string]string{} - backupURL := backupstore.EncodeBackupURL(backupName, backupVolumeName, backupTargetClient.URL) + backupURL := backupstore.EncodeBackupURL(backupName, remoteBackupVolumeName, backupTargetClient.URL) if backupInfo, err := backupTargetClient.BackupGet(backupURL, backupTargetClient.Credential); err != nil { log.WithError(err).WithFields(logrus.Fields{ "backup": backupName, @@ -336,12 +337,16 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error backup := &longhorn.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: backupName, + Labels: map[string]string{ + types.LonghornLabelBackupTarget: backupTarget.Name, + }, }, Spec: longhorn.BackupSpec{ - Labels: backupLabelMap, + Labels: backupLabelMap, + BackupTargetName: backupVolume.Spec.BackupTargetName, }, } - if _, err = bvc.ds.CreateBackup(backup, backupVolumeName); err != nil && !apierrors.IsAlreadyExists(err) { + if _, err = bvc.ds.CreateBackup(backup, remoteBackupVolumeName); err != nil && !apierrors.IsAlreadyExists(err) { log.WithError(err).Errorf("Failed to create backup %s in the cluster", backupName) return err } @@ -359,7 +364,7 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error } } - backupVolumeMetadataURL := backupstore.EncodeBackupURL("", backupVolumeName, backupTargetClient.URL) + backupVolumeMetadataURL := backupstore.EncodeBackupURL("", remoteBackupVolumeName, backupTargetClient.URL) configMetadata, err := backupTargetClient.BackupConfigMetaGet(backupVolumeMetadataURL, backupTargetClient.Credential) if err != nil { log.WithError(err).Error("Failed to get backup volume config metadata from backup target") diff --git a/datastore/longhorn.go b/datastore/longhorn.go index 7fd37f071d..02d8f7ac9f 100644 --- a/datastore/longhorn.go +++ b/datastore/longhorn.go @@ -4373,10 +4373,10 @@ func (s *DataStore) CreateBackup(backup *longhorn.Backup, backupVolumeName strin return ret.DeepCopy(), nil } -// ListBackupsWithBackupVolumeName returns an object contains all backups in the cluster Backups CR -// of the given backup volume name -func (s *DataStore) ListBackupsWithBackupVolumeName(backupVolumeName string) (map[string]*longhorn.Backup, error) { - selector, err := getBackupVolumeSelector(backupVolumeName) +// ListBackupsWithVolumeNameAndBackupTargetRO returns an object contains all backups in the cluster Backups CR +// of the given volume name and backup target name +func (s *DataStore) ListBackupsWithBackupVolumeNameRO(backupTargetName, volumeName string) (map[string]*longhorn.Backup, error) { + selector, err := getBackupVolumeWithBackupTargetSelector(backupTargetName, volumeName) if err != nil { return nil, err } @@ -4388,7 +4388,7 @@ func (s *DataStore) ListBackupsWithBackupVolumeName(backupVolumeName string) (ma itemMap := map[string]*longhorn.Backup{} for _, itemRO := range list { - itemMap[itemRO.Name] = itemRO.DeepCopy() + itemMap[itemRO.Name] = itemRO } return itemMap, nil } @@ -4456,9 +4456,11 @@ func (s *DataStore) DeleteBackup(backupName string) error { return s.lhClient.LonghornV1beta2().Backups(s.namespace).Delete(context.TODO(), backupName, metav1.DeleteOptions{}) } -// DeleteAllBackupsForBackupVolume won't result in immediately deletion since finalizer was set by default -func (s *DataStore) DeleteAllBackupsForBackupVolume(backupVolumeName string) error { - return s.lhClient.LonghornV1beta2().Backups(s.namespace).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", types.LonghornLabelBackupVolume, backupVolumeName)}) +// DeleteAllBackupsForBackupVolumeWithBackupTarget won't result in immediately deletion since finalizer was set by default +func (s *DataStore) DeleteAllBackupsForBackupVolumeWithBackupTarget(backuptargetName, volumeName string) error { + return s.lhClient.LonghornV1beta2().Backups(s.namespace).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{ + LabelSelector: labels.Set(types.GetBackupVolumeWithBackupTargetLabels(backuptargetName, volumeName)).String(), + }) } // RemoveFinalizerForBackup will result in deletion if DeletionTimestamp was set diff --git a/manager/engine.go b/manager/engine.go index f1f45e1806..3e72af4495 100644 --- a/manager/engine.go +++ b/manager/engine.go @@ -415,7 +415,11 @@ func (m *VolumeManager) ListAllBackupsSorted() ([]*longhorn.Backup, error) { } func (m *VolumeManager) ListBackupsForVolume(volumeName string) (map[string]*longhorn.Backup, error) { - return m.ds.ListBackupsWithBackupVolumeName(volumeName) + v, err := m.ds.GetVolumeRO(volumeName) + if err != nil { + return nil, err + } + return m.ds.ListBackupsWithBackupVolumeNameRO(v.Spec.BackupTargetName, volumeName) } func (m *VolumeManager) ListBackupsForVolumeSorted(volumeName string) ([]*longhorn.Backup, error) { From 6e7a401bb5197742485a9ba38af57afbb288e33f Mon Sep 17 00:00:00 2001 From: James Lu Date: Tue, 24 Sep 2024 21:17:05 +0800 Subject: [PATCH 15/29] feat(backup): modify backup controller Modify backup controller to support multiple backup stores support. Synchronize and handle the backup information from different backup targets with the same backup volume name or not. Ref: 5411 Signed-off-by: James Lu --- controller/backup_controller.go | 57 ++++++++++++++++++++------------- datastore/longhorn.go | 10 ++++++ 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/controller/backup_controller.go b/controller/backup_controller.go index 04c0c7bd09..eec3a558e5 100644 --- a/controller/backup_controller.go +++ b/controller/backup_controller.go @@ -251,17 +251,16 @@ func (bc *BackupController) reconcile(backupName string) (err error) { log := getLoggerForBackup(bc.logger, backup) - // Get default backup target - backupTarget, err := bc.ds.GetBackupTargetRO(types.DefaultBackupTargetName) + // Get BackupTarget CR + backupTarget, err := bc.getBackupTarget(backup) if err != nil { - if apierrors.IsNotFound(err) { - return nil - } - return errors.Wrapf(err, "failed to get the backup target %v", types.DefaultBackupTargetName) + return err } + backupTargetName := backupTarget.Name // Find the backup volume name from label - backupVolumeName, err := bc.getBackupVolumeName(backup) + + remoteBackupVolumeName, err := bc.getBackupVolumeName(backup) if err != nil { if types.ErrorIsNotFound(err) { return nil // Ignore error to prevent enqueue @@ -271,16 +270,16 @@ func (bc *BackupController) reconcile(backupName string) (err error) { // Examine DeletionTimestamp to determine if object is under deletion if !backup.DeletionTimestamp.IsZero() { - if err := bc.handleAttachmentTicketDeletion(backup, backupVolumeName); err != nil { + if err := bc.handleAttachmentTicketDeletion(backup, remoteBackupVolumeName); err != nil { return err } - backupVolume, err := bc.ds.GetBackupVolume(backupVolumeName) + backupVolume, err := bc.ds.GetBackupVolumeWithBackupTargetAndVolume(backupTargetName, remoteBackupVolumeName) if err != nil && !apierrors.IsNotFound(err) { return err } - if backupTarget.Spec.BackupTargetURL != "" && + if IsBackupTargetAvailable(backupTarget) && backupVolume != nil && backupVolume.DeletionTimestamp == nil { backupTargetClient, err := newBackupTargetClientFromDefaultEngineImage(bc.ds, backupTarget) if err != nil { @@ -288,22 +287,22 @@ func (bc *BackupController) reconcile(backupName string) (err error) { return nil // Ignore error to prevent enqueue } - if unused, err := bc.isBackupNotBeingUsedForVolumeRestore(backup.Name, backupVolumeName); !unused { + if unused, err := bc.isBackupNotBeingUsedForVolumeRestore(backup.Name, remoteBackupVolumeName); !unused { log.WithError(err).Warn("Failed to delete remote backup") return nil } - backupURL := backupstore.EncodeBackupURL(backup.Name, backupVolumeName, backupTargetClient.URL) + backupURL := backupstore.EncodeBackupURL(backup.Name, remoteBackupVolumeName, backupTargetClient.URL) if err := backupTargetClient.BackupDelete(backupURL, backupTargetClient.Credential); err != nil { return errors.Wrap(err, "failed to delete remote backup") } } // Request backup_volume_controller to reconcile BackupVolume immediately if it's the last backup - if backupVolume != nil && backupVolume.Status.LastBackupName == backup.Name { + if backupVolume != nil && backupVolume.DeletionTimestamp == nil && backupVolume.Status.LastBackupName == backup.Name { backupVolume.Spec.SyncRequestedAt = metav1.Time{Time: time.Now().UTC()} if _, err = bc.ds.UpdateBackupVolume(backupVolume); err != nil && !apierrors.IsConflict(errors.Cause(err)) { - log.WithError(err).Errorf("Failed to update backup volume %s spec", backupVolumeName) + log.WithError(err).Errorf("Failed to update backup volume %s/%s spec", backupVolume.Name, remoteBackupVolumeName) // Do not return err to enqueue since backup_controller is responsible to // reconcile Backup CR spec, waits the backup_volume_controller next reconcile time // to update it's BackupVolume CR status @@ -344,7 +343,7 @@ func (bc *BackupController) reconcile(backupName string) (err error) { } if bc.backupInFinalState(backup) && (!backup.Status.LastSyncedAt.IsZero() || backup.Spec.SnapshotName == "") { - err = bc.handleAttachmentTicketDeletion(backup, backupVolumeName) + err = bc.handleAttachmentTicketDeletion(backup, remoteBackupVolumeName) } if reflect.DeepEqual(existingBackup.Status, backup.Status) { return @@ -356,8 +355,8 @@ func (bc *BackupController) reconcile(backupName string) (err error) { return } if backup.Status.State == longhorn.BackupStateCompleted && existingBackupState != backup.Status.State { - if err := bc.syncBackupVolume(backupVolumeName); err != nil { - log.Warnf("failed to sync Backup Volume: %v", backupVolumeName) + if err := bc.syncBackupVolume(backupTargetName, remoteBackupVolumeName); err != nil { + log.Warnf("failed to sync Backup Volume: %v", remoteBackupVolumeName) return } } @@ -371,7 +370,7 @@ func (bc *BackupController) reconcile(backupName string) (err error) { // exists in the remote backup target before the CR creation. // What the controller needs to do for this case is retrieve the info from the remote backup target. if backup.Status.LastSyncedAt.IsZero() && backup.Spec.SnapshotName != "" && !bc.backupInFinalState(backup) { - volume, err := bc.ds.GetVolume(backupVolumeName) + volume, err := bc.ds.GetVolume(remoteBackupVolumeName) if err != nil { if !apierrors.IsNotFound(err) { return err @@ -384,7 +383,7 @@ func (bc *BackupController) reconcile(backupName string) (err error) { return nil // Ignore error to prevent enqueue } - if err := bc.handleAttachmentTicketCreation(backup, backupVolumeName); err != nil { + if err := bc.handleAttachmentTicketCreation(backup, remoteBackupVolumeName); err != nil { return err } @@ -442,7 +441,7 @@ func (bc *BackupController) reconcile(backupName string) (err error) { return nil // Ignore error to prevent enqueue } - backupURL := backupstore.EncodeBackupURL(backup.Name, backupVolumeName, backupTargetClient.URL) + backupURL := backupstore.EncodeBackupURL(backup.Name, remoteBackupVolumeName, backupTargetClient.URL) backupInfo, err := backupTargetClient.BackupGet(backupURL, backupTargetClient.Credential) if err != nil { if !strings.Contains(err.Error(), "in progress") { @@ -588,6 +587,18 @@ func (bc *BackupController) isResponsibleFor(b *longhorn.Backup, defaultEngineIm return isPreferredOwner || continueToBeOwner || requiresNewOwner, nil } +func (bc *BackupController) getBackupTarget(backup *longhorn.Backup) (*longhorn.BackupTarget, error) { + backupTarget, err := bc.ds.GetBackupTargetRO(backup.Spec.BackupTargetName) + if err != nil && !apierrors.IsNotFound(err) { + return nil, errors.Wrapf(err, "failed to get the backup target %v", backup.Spec.BackupTargetName) + } + if backupTarget == nil && backup.DeletionTimestamp == nil { + return nil, fmt.Errorf("Failed to find the backup target %v for the backup %v", backup.Spec.BackupTargetName, backup.Name) + } + + return backupTarget, nil +} + func (bc *BackupController) getBackupVolumeName(backup *longhorn.Backup) (string, error) { backupVolumeName, ok := backup.Labels[types.LonghornLabelBackupVolume] if !ok { @@ -800,9 +811,9 @@ func (bc *BackupController) syncWithMonitor(backup *longhorn.Backup, volume *lon // syncBackupVolume triggers the backup_volume_controller/backup_target_controller // to run reconcile immediately -func (bc *BackupController) syncBackupVolume(volumeName string) error { +func (bc *BackupController) syncBackupVolume(backupTargetName, volumeName string) error { syncTime := metav1.Time{Time: time.Now().UTC()} - backupVolume, err := bc.ds.GetBackupVolume(volumeName) + backupVolume, err := bc.ds.GetBackupVolumeWithBackupTargetAndVolume(backupTargetName, volumeName) if err == nil { // Request backup_volume_controller to reconcile BackupVolume immediately. backupVolume.Spec.SyncRequestedAt = syncTime @@ -811,7 +822,7 @@ func (bc *BackupController) syncBackupVolume(volumeName string) error { } } else if err != nil && apierrors.IsNotFound(err) { // Request backup_target_controller to reconcile BackupTarget immediately. - backupTarget, err := bc.ds.GetBackupTarget(types.DefaultBackupTargetName) + backupTarget, err := bc.ds.GetBackupTarget(backupTargetName) if err != nil { return errors.Wrap(err, "failed to get backup target") } diff --git a/datastore/longhorn.go b/datastore/longhorn.go index 02d8f7ac9f..4492c2224c 100644 --- a/datastore/longhorn.go +++ b/datastore/longhorn.go @@ -4265,6 +4265,16 @@ func (s *DataStore) GetBackupVolumeWithBackupTargetAndVolumeRO(backupTargetName, return nil, nil } +// GetBackupVolumeWithBackupTargetAndVolume returns a copy of BackupVolume with the given backup target and volume name in the cluster +func (s *DataStore) GetBackupVolumeWithBackupTargetAndVolume(backupTargetName, volumeName string) (*longhorn.BackupVolume, error) { + resultRO, err := s.GetBackupVolumeWithBackupTargetAndVolumeRO(backupTargetName, volumeName) + if err != nil { + return nil, err + } + // Cannot use cached object from lister + return resultRO.DeepCopy(), nil +} + func getBackupTargetSelector(backupTargetName string) (labels.Selector, error) { return metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ MatchLabels: types.GetBackupTargetLabels(backupTargetName), From d8971ca81e96a426456aabe2c9a280ea9f6a339f Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 26 Sep 2024 12:01:41 +0800 Subject: [PATCH 16/29] feat(backup): add backup target label when creating a backup Ref: 5411 Signed-off-by: James Lu --- webhook/resources/backup/mutator.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/webhook/resources/backup/mutator.go b/webhook/resources/backup/mutator.go index b554b3dd45..d0ed6b085d 100644 --- a/webhook/resources/backup/mutator.go +++ b/webhook/resources/backup/mutator.go @@ -85,6 +85,28 @@ func (b *backupMutator) Create(request *admission.Request, newObj runtime.Object patchOps = append(patchOps, fmt.Sprintf(`{"op": "replace", "path": "/spec/backupMode", "value": "%s"}`, string(longhorn.BackupModeIncremental))) } + if backup.Spec.BackupTargetName == "" { + backupTargetName := types.DefaultBackupTargetName + if volume, err := b.ds.GetVolumeRO(volumeName); err == nil { + backupTargetName = volume.Spec.BackupTargetName + } + backup.Spec.BackupTargetName = backupTargetName + patchOps = append(patchOps, fmt.Sprintf(`{"op": "replace", "path": "/spec/backupTargetName", "value": "%s"}`, backupTargetName)) + } + + labels := backup.Labels + if labels == nil { + labels = map[string]string{} + } + labels[types.LonghornLabelBackupTarget] = backup.Spec.BackupTargetName + + patchOp, err := common.GetLonghornLabelsPatchOp(backup, labels, nil) + if err != nil { + err := errors.Wrapf(err, "failed to get label patch for backup %v", backup.Name) + return nil, werror.NewInvalidError(err.Error(), "") + } + patchOps = append(patchOps, patchOp) + return patchOps, nil } From a04677051022441c72e532a2f0139308b0b616e0 Mon Sep 17 00:00:00 2001 From: James Lu Date: Tue, 24 Sep 2024 11:45:10 +0800 Subject: [PATCH 17/29] feat(backup): handle the restoration Ref: 5411 Signed-off-by: James Lu --- controller/engine_controller.go | 13 +++++++++---- controller/volume_controller.go | 17 +++++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/controller/engine_controller.go b/controller/engine_controller.go index 92da3d22d2..63301f4a09 100644 --- a/controller/engine_controller.go +++ b/controller/engine_controller.go @@ -1414,12 +1414,17 @@ func checkSizeBeforeRestoration(log logrus.FieldLogger, engine *longhorn.Engine, } func (m *EngineMonitor) restoreBackup(engine *longhorn.Engine, rsMap map[string]*longhorn.RestoreStatus, cliAPIVersion int, engineClientProxy engineapi.EngineClientProxy) error { - backupTarget, err := m.ds.GetBackupTargetRO(types.DefaultBackupTargetName) + backupVolume, err := m.ds.GetBackupVolumeRO(engine.Spec.BackupVolume) + if err != nil { + return errors.Wrapf(err, "failed to get backup volume %v for backup restoration of engine %v", engine.Spec.BackupVolume, engine.Name) + } + + backupTarget, err := m.ds.GetBackupTargetRO(backupVolume.Spec.BackupTargetName) if err != nil { if !apierrors.IsNotFound(err) { return err } - return fmt.Errorf("cannot find the backup target %s", types.DefaultBackupTargetName) + return fmt.Errorf("cannot find the backup target %s", backupVolume.Spec.BackupTargetName) } backupTargetClient, err := newBackupTargetClientFromDefaultEngineImage(m.ds, backupTarget) @@ -1442,13 +1447,13 @@ func (m *EngineMonitor) restoreBackup(engine *longhorn.Engine, rsMap map[string] mlog.Info("Restoring backup") if cliAPIVersion < engineapi.CLIVersionFour { // For compatible engines, `LastRestoredBackup` is required to indicate if the restore is incremental restore - if err = engineClientProxy.BackupRestore(engine, backupTargetClient.URL, engine.Spec.RequestedBackupRestore, engine.Spec.BackupVolume, engine.Status.LastRestoredBackup, backupTargetClient.Credential, int(concurrentLimit)); err != nil { + if err = engineClientProxy.BackupRestore(engine, backupTargetClient.URL, engine.Spec.RequestedBackupRestore, backupVolume.Spec.VolumeName, engine.Status.LastRestoredBackup, backupTargetClient.Credential, int(concurrentLimit)); err != nil { if extraErr := handleRestoreErrorForCompatibleEngine(mlog, engine, rsMap, m.restoreBackoff, err); extraErr != nil { return extraErr } } } else { - if err = engineClientProxy.BackupRestore(engine, backupTargetClient.URL, engine.Spec.RequestedBackupRestore, engine.Spec.BackupVolume, "", backupTargetClient.Credential, int(concurrentLimit)); err != nil { + if err = engineClientProxy.BackupRestore(engine, backupTargetClient.URL, engine.Spec.RequestedBackupRestore, backupVolume.Spec.VolumeName, "", backupTargetClient.Credential, int(concurrentLimit)); err != nil { if extraErr := handleRestoreError(mlog, engine, rsMap, m.restoreBackoff, err); extraErr != nil { return extraErr } diff --git a/controller/volume_controller.go b/controller/volume_controller.go index 9794b3ee75..dc28469872 100644 --- a/controller/volume_controller.go +++ b/controller/volume_controller.go @@ -3615,15 +3615,24 @@ func (c *VolumeController) createEngine(v *longhorn.Volume, currentEngineName st } if v.Spec.FromBackup != "" && v.Status.RestoreRequired { - backupVolumeName, backupName, err := c.getInfoFromBackupURL(v) + remoteBackupVolumeName, backupName, err := c.getInfoFromBackupURL(v) if err != nil { return nil, errors.Wrapf(err, "failed to get backup volume when creating engine object of restored volume %v", v.Name) } - engine.Spec.BackupVolume = backupVolumeName + backupRO, err := c.ds.GetBackupRO(backupName) + if err != nil { + return nil, errors.Wrapf(err, "failed to get backup when creating engine object of restored volume %v", v.Name) + } + backupVolumeRO, err := c.ds.GetBackupVolumeWithBackupTargetAndVolumeRO(backupRO.Spec.BackupTargetName, remoteBackupVolumeName) + if err != nil { + return nil, errors.Wrapf(err, "failed to get backup volume when creating engine object of restored volume %v", v.Name) + } + + engine.Spec.BackupVolume = backupVolumeRO.Name engine.Spec.RequestedBackupRestore = backupName - log.Infof("Creating engine %v for restored volume, BackupVolume is %v, RequestedBackupRestore is %v", - engine.Name, engine.Spec.BackupVolume, engine.Spec.RequestedBackupRestore) + log.Infof("Creating engine %v for restored volume %v, BackupVolume is %v, Remote BackupVolume is %v, RequestedBackupRestore is %v", + engine.Name, v.Name, engine.Spec.BackupVolume, remoteBackupVolumeName, engine.Spec.RequestedBackupRestore) } unmapMarkEnabled, err := c.isUnmapMarkSnapChainRemovedEnabled(v) From c8eef1f52614f5189829ab1bb7ab8eaa9ecd3bfd Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 26 Sep 2024 17:31:27 +0800 Subject: [PATCH 18/29] feat(backup): handle triggering the backup volume synchronization Ref: 5411 Signed-off-by: James Lu --- manager/volume.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/manager/volume.go b/manager/volume.go index 97b65dce0c..0da196f475 100644 --- a/manager/volume.go +++ b/manager/volume.go @@ -427,17 +427,22 @@ func (m *VolumeManager) Activate(volumeName string, frontend string) (v *longhor } func (m *VolumeManager) triggerBackupVolumeToSync(volume *longhorn.Volume) error { + backupTargetName, exists := volume.Labels[types.LonghornLabelBackupTarget] + if !exists || backupTargetName == "" { + return errors.Errorf("cannot find the backup target label for volume: %v", volume.Name) + } + backupVolumeName, isExist := volume.Labels[types.LonghornLabelBackupVolume] if !isExist || backupVolumeName == "" { return errors.Errorf("cannot find the backup volume label for volume: %v", volume.Name) } - backupVolume, err := m.ds.GetBackupVolume(backupVolumeName) + backupVolume, err := m.ds.GetBackupVolumeWithBackupTargetAndVolume(backupTargetName, backupVolumeName) if err != nil { // The backup volume may be deleted already. // hence it's better not to block the caller to continue the handlings like DR volume activation. if apierrors.IsNotFound(err) { - logrus.Infof("Cannot find backup volume %v to trigger the sync-up, will skip it", backupVolumeName) + logrus.Infof("Cannot find backup volume %v of backup target %s to trigger the sync-up, will skip it", backupVolumeName, backupTargetName) return nil } return errors.Wrapf(err, "failed to get backup volume: %v", backupVolumeName) From 31f4e5fc3951742ea8977ad75b917c898a95af9f Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 25 Sep 2024 15:51:11 +0800 Subject: [PATCH 19/29] feat(backup): recurringjob for multiple backupstores add the method to get the correct BackupVolume with the volume name and backup target name for the recurring job and related data strucurtes of APIs. Ref: 5411 Signed-off-by: James Lu --- api/snapshot.go | 2 +- app/recurring_job.go | 25 +++++++++++++++++++++++-- controller/volume_controller.go | 10 ++++++---- manager/engine.go | 12 ++++++++---- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/api/snapshot.go b/api/snapshot.go index f627a4a381..907f636ec8 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -195,7 +195,7 @@ func (s *Server) SnapshotBackup(w http.ResponseWriter, req *http.Request) (err e labels[types.KubernetesStatusLabel] = string(kubeStatus) } - if err := s.m.BackupSnapshot(bsutil.GenerateName("backup"), volName, input.Name, labels, input.BackupMode); err != nil { + if err := s.m.BackupSnapshot(bsutil.GenerateName("backup"), vol.Spec.BackupTargetName, volName, input.Name, labels, input.BackupMode); err != nil { return err } diff --git a/app/recurring_job.go b/app/recurring_job.go index 1dd2296ed7..eecf1977fa 100644 --- a/app/recurring_job.go +++ b/app/recurring_job.go @@ -693,7 +693,7 @@ func (job *Job) doRecurringBackup() (err error) { } }() - backupVolume, err := job.api.BackupVolume.ById(job.volumeName) + backupVolume, err := job.getBackupVolume(volume.BackupTargetName) if err != nil { return err } @@ -717,6 +717,27 @@ func (job *Job) doRecurringBackup() (err error) { return nil } +func (job *Job) getBackupVolume(backupTargetName string) (*longhornclient.BackupVolume, error) { + list, err := job.api.BackupVolume.List(&longhornclient.ListOpts{ + Filters: map[string]interface{}{ + types.LonghornLabelBackupTarget: backupTargetName, + types.LonghornLabelBackupVolume: job.volumeName, + }}) + if err != nil { + return nil, err + } + + if len(list.Data) >= 2 { + return nil, fmt.Errorf("found more than one backup volume %v of the backup target %v", job.volumeName, backupTargetName) + } + + for _, backupVolume := range list.Data { + return &backupVolume, nil + } + + return nil, fmt.Errorf("failed to find backup volume %s of the backup target %s: %v", job.volumeName, backupTargetName, err) +} + func (job *Job) doRecurringFilesystemTrim(volume *longhornclient.Volume) (err error) { defer func() { err = errors.Wrapf(err, "failed to complete filesystem-trim for %v", volume.Name) @@ -770,7 +791,7 @@ func (job *Job) getLastBackup() (*longhornclient.Backup, error) { if volume.LastBackup == "" { return nil, nil } - backupVolume, err := job.api.BackupVolume.ById(job.volumeName) + backupVolume, err := job.getBackupVolume(volume.BackupTargetName) if err != nil { return nil, err } diff --git a/controller/volume_controller.go b/controller/volume_controller.go index dc28469872..a558502754 100644 --- a/controller/volume_controller.go +++ b/controller/volume_controller.go @@ -3861,15 +3861,17 @@ func (c *VolumeController) restoreVolumeRecurringJobs(v *longhorn.Volume) error log := getLoggerForVolume(c.logger, v) backupVolumeRecurringJobsInfo := make(map[string]longhorn.VolumeRecurringJobInfo) - bvName, exist := v.Labels[types.LonghornLabelBackupVolume] + btName := v.Spec.BackupTargetName + + volName, exist := v.Labels[types.LonghornLabelBackupVolume] if !exist { log.Warn("Failed to find the backup volume label") return nil } - bv, err := c.ds.GetBackupVolumeRO(bvName) + bv, err := c.ds.GetBackupVolumeWithBackupTargetAndVolumeRO(btName, volName) if err != nil { - return errors.Wrapf(err, "failed to get the backup volume %v info", bvName) + return errors.Wrapf(err, "failed to get the backup volume %v of the backup target %s info", volName, btName) } volumeRecurringJobInfoStr, exist := bv.Status.Labels[types.VolumeRecurringJobInfoLabel] @@ -3878,7 +3880,7 @@ func (c *VolumeController) restoreVolumeRecurringJobs(v *longhorn.Volume) error } if err := json.Unmarshal([]byte(volumeRecurringJobInfoStr), &backupVolumeRecurringJobsInfo); err != nil { - return errors.Wrapf(err, "failed to unmarshal information of volume recurring jobs, backup volume %v", bvName) + return errors.Wrapf(err, "failed to unmarshal information of volume recurring jobs, backup volume %v of the backup target %s", volName, btName) } for jobName, job := range backupVolumeRecurringJobsInfo { diff --git a/manager/engine.go b/manager/engine.go index 3e72af4495..e54a3fadcf 100644 --- a/manager/engine.go +++ b/manager/engine.go @@ -252,7 +252,7 @@ func (m *VolumeManager) PurgeSnapshot(volumeName string) error { return nil } -func (m *VolumeManager) BackupSnapshot(backupName, volumeName, snapshotName string, labels map[string]string, backupMode string) error { +func (m *VolumeManager) BackupSnapshot(backupName, backupTargetName, volumeName, snapshotName string, labels map[string]string, backupMode string) error { if volumeName == "" || snapshotName == "" { return fmt.Errorf("volume and snapshot name required") } @@ -264,11 +264,15 @@ func (m *VolumeManager) BackupSnapshot(backupName, volumeName, snapshotName stri backupCR := &longhorn.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: backupName, + Labels: map[string]string{ + types.LonghornLabelBackupTarget: backupTargetName, + }, }, Spec: longhorn.BackupSpec{ - SnapshotName: snapshotName, - Labels: labels, - BackupMode: longhorn.BackupMode(backupMode), + SnapshotName: snapshotName, + Labels: labels, + BackupMode: longhorn.BackupMode(backupMode), + BackupTargetName: backupTargetName, }, } _, err := m.ds.CreateBackup(backupCR, volumeName) From b99190d71b6b7751c5f296f5bebccdc49c77b711 Mon Sep 17 00:00:00 2001 From: James Lu Date: Tue, 24 Sep 2024 16:56:14 +0800 Subject: [PATCH 20/29] feat(backup): add backup target APIs Ref: 5411 Signed-off-by: James Lu --- api/backup.go | 121 ++++++++++++++++++++++++++++++ api/model.go | 44 ++++++++++- api/router.go | 6 +- client/generated_backup.go | 2 + client/generated_backup_target.go | 4 + client/generated_backup_volume.go | 4 + engineapi/types.go | 4 + manager/engine.go | 65 +++++++++++++++- 8 files changed, 243 insertions(+), 7 deletions(-) diff --git a/api/backup.go b/api/backup.go index 510434a75d..36a6c41971 100644 --- a/api/backup.go +++ b/api/backup.go @@ -3,6 +3,10 @@ package api import ( "fmt" "net/http" + "strconv" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/gorilla/mux" "github.com/pkg/errors" @@ -10,8 +14,27 @@ import ( "github.com/rancher/go-rancher/api" "github.com/rancher/go-rancher/client" + + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" + "github.com/longhorn/longhorn-manager/util" ) +const ( + BackupTargetDefaultPollInterval = 300 +) + +func (s *Server) BackupTargetGet(w http.ResponseWriter, req *http.Request) error { + apiContext := api.GetApiContext(req) + + backupTargetName := mux.Vars(req)["name"] + backupTarget, err := s.m.GetBackupTarget(backupTargetName) + if err != nil { + return errors.Wrap(err, "failed to list backup targets") + } + apiContext.Write(toBackupTargetResource(backupTarget, apiContext)) + return nil +} + func (s *Server) BackupTargetList(w http.ResponseWriter, req *http.Request) error { apiContext := api.GetApiContext(req) @@ -23,6 +46,80 @@ func (s *Server) BackupTargetList(w http.ResponseWriter, req *http.Request) erro return nil } +func (s *Server) BackupTargetCreate(rw http.ResponseWriter, req *http.Request) error { + var input BackupTarget + apiContext := api.GetApiContext(req) + + if err := apiContext.Read(&input); err != nil { + return err + } + + backupTargetSpec, err := newBackupTarget(input) + if err != nil { + return err + } + + obj, err := s.m.CreateBackupTarget(input.Name, backupTargetSpec) + if err != nil { + return errors.Wrapf(err, "failed to create backup target %v", input.Name) + } + apiContext.Write(toBackupTargetResource(obj, apiContext)) + return nil +} + +func newBackupTarget(input BackupTarget) (*longhorn.BackupTargetSpec, error) { + pollInterval, err := strconv.ParseInt(input.PollInterval, 10, 64) + if err != nil { + pollInterval = BackupTargetDefaultPollInterval + } + + return &longhorn.BackupTargetSpec{ + BackupTargetURL: input.BackupTargetURL, + CredentialSecret: input.CredentialSecret, + PollInterval: metav1.Duration{Duration: time.Duration(pollInterval) * time.Second}, + ReadOnly: input.ReadyOnly}, nil +} + +func (s *Server) BackupTargetUpdate(rw http.ResponseWriter, req *http.Request) error { + var input BackupTarget + + apiContext := api.GetApiContext(req) + if err := apiContext.Read(&input); err != nil { + return err + } + + name := mux.Vars(req)["name"] + + backupTargetSpec, err := newBackupTarget(input) + if err != nil { + return err + } + + obj, err := util.RetryOnConflictCause(func() (interface{}, error) { + return s.m.UpdateBackupTarget(input.Name, backupTargetSpec) + }) + if err != nil { + return err + } + + backupTarget, ok := obj.(*longhorn.BackupTarget) + if !ok { + return fmt.Errorf("failed to convert %v to backup target", name) + } + + apiContext.Write(toBackupTargetResource(backupTarget, apiContext)) + return nil +} + +func (s *Server) BackupTargetDelete(rw http.ResponseWriter, req *http.Request) error { + backupTargetName := mux.Vars(req)["name"] + if err := s.m.DeleteBackupTarget(backupTargetName); err != nil { + return errors.Wrapf(err, "failed to delete backup target %v", backupTargetName) + } + + return nil +} + func (s *Server) BackupTargetSyncAll(w http.ResponseWriter, req *http.Request) error { var input SyncBackupResource @@ -174,6 +271,30 @@ func (s *Server) BackupVolumeDelete(w http.ResponseWriter, req *http.Request) er func (s *Server) BackupList(w http.ResponseWriter, req *http.Request) error { apiContext := api.GetApiContext(req) + bs, err := s.m.ListAllBackupsSorted() + if err != nil { + return errors.Wrapf(err, "failed to list all backups") + } + apiContext.Write(toBackupCollection(bs)) + return nil +} + +func (s *Server) BackupListByVolumeName(w http.ResponseWriter, req *http.Request) error { + apiContext := api.GetApiContext(req) + + volName := mux.Vars(req)["volName"] + + bs, err := s.m.ListBackupsForVolumeSorted(volName) + if err != nil { + return errors.Wrapf(err, "failed to list backups for volume '%s'", volName) + } + apiContext.Write(toBackupCollection(bs)) + return nil +} + +func (s *Server) BackupListByBVName(w http.ResponseWriter, req *http.Request) error { + apiContext := api.GetApiContext(req) + volName := mux.Vars(req)["volName"] bs, err := s.m.ListBackupsForVolumeSorted(volName) diff --git a/api/model.go b/api/model.go index 9d7e300f79..feee223ac8 100644 --- a/api/model.go +++ b/api/model.go @@ -140,6 +140,8 @@ type BackupVolume struct { BackingImageName string `json:"backingImageName"` BackingImageChecksum string `json:"backingImageChecksum"` StorageClassName string `json:"storageClassName"` + BackupTargetName string `json:"backupTargetName"` + VolumeName string `json:"volumeName"` } // SyncBackupResource is used for the Backup*Sync* actions @@ -172,6 +174,7 @@ type Backup struct { CompressionMethod string `json:"compressionMethod"` NewlyUploadedDataSize string `json:"newlyUploadDataSize"` ReUploadedDataSize string `json:"reUploadedDataSize"` + BackupTargetName string `json:"backupTargetName"` } type BackupBackingImage struct { @@ -874,14 +877,44 @@ func kubernetesStatusSchema(status *client.Schema) { } func backupTargetSchema(backupTarget *client.Schema) { - backupTarget.CollectionMethods = []string{"GET"} - backupTarget.ResourceMethods = []string{"GET", "PUT"} + backupTarget.CollectionMethods = []string{"GET", "POST"} + backupTarget.ResourceMethods = []string{"GET", "PUT", "DELETE"} + + backupTargetName := backupTarget.ResourceFields["name"] + backupTargetName.Required = true + backupTargetName.Unique = true + backupTargetName.Create = true + backupTarget.ResourceFields["name"] = backupTargetName + + backupTargetURL := backupTarget.ResourceFields["backupTargetURL"] + backupTargetURL.Create = true + backupTargetURL.Default = "" + backupTarget.ResourceFields["backupTargetURL"] = backupTargetURL + + credentialSecret := backupTarget.ResourceFields["credentialSecret"] + credentialSecret.Create = true + credentialSecret.Default = "" + backupTarget.ResourceFields["credentialSecret"] = credentialSecret + + backupTargetPollInterval := backupTarget.ResourceFields["pollInterval"] + backupTargetPollInterval.Create = true + backupTargetPollInterval.Default = "300" + backupTarget.ResourceFields["pollInterval"] = backupTargetPollInterval + + backupTargetReadOnly := backupTarget.ResourceFields["readOnly"] + backupTargetReadOnly.Create = true + backupTargetReadOnly.Default = false + backupTarget.ResourceFields["readOnly"] = backupTargetReadOnly backupTarget.ResourceActions = map[string]client.Action{ "backupTargetSync": { Input: "syncBackupResource", Output: "backupTargetListOutput", }, + "backupTargetUpdate": { + Input: "BackupTarget", + Output: "backupTargetListOutput", + }, } } @@ -1801,11 +1834,13 @@ func toBackupTargetResource(bt *longhorn.BackupTarget, apiContext *api.ApiContex CredentialSecret: bt.Spec.CredentialSecret, PollInterval: bt.Spec.PollInterval.Duration.String(), Available: bt.Status.Available, + ReadyOnly: bt.Spec.ReadOnly, Message: types.GetCondition(bt.Status.Conditions, longhorn.BackupTargetConditionTypeUnavailable).Message, }, } res.Actions = map[string]string{ - "backupTargetSync": apiContext.UrlBuilder.ActionLink(res.Resource, "backupTargetSync"), + "backupTargetSync": apiContext.UrlBuilder.ActionLink(res.Resource, "backupTargetSync"), + "backupTargetUpdate": apiContext.UrlBuilder.ActionLink(res.Resource, "backupTargetUpdate"), } return res @@ -1832,6 +1867,8 @@ func toBackupVolumeResource(bv *longhorn.BackupVolume, apiContext *api.ApiContex BackingImageName: bv.Status.BackingImageName, BackingImageChecksum: bv.Status.BackingImageChecksum, StorageClassName: bv.Status.StorageClassName, + BackupTargetName: bv.Spec.BackupTargetName, + VolumeName: bv.Spec.VolumeName, } b.Actions = map[string]string{ "backupList": apiContext.UrlBuilder.ActionLink(b.Resource, "backupList"), @@ -1938,6 +1975,7 @@ func toBackupResource(b *longhorn.Backup) *Backup { CompressionMethod: string(b.Status.CompressionMethod), NewlyUploadedDataSize: b.Status.NewlyUploadedDataSize, ReUploadedDataSize: b.Status.ReUploadedDataSize, + BackupTargetName: b.Spec.BackupTargetName, } // Set the volume name from backup CR's label if it's empty. // This field is empty probably because the backup state is not Ready diff --git a/api/router.go b/api/router.go index 2a89fbaf31..e5c7953a04 100644 --- a/api/router.go +++ b/api/router.go @@ -122,11 +122,15 @@ func NewRouter(s *Server) *mux.Router { r.Methods("GET").Path("/v1/backuptargets").Handler(f(schemas, s.BackupTargetList)) r.Methods("PUT").Path("/v1/backuptargets").Handler(f(schemas, s.BackupTargetSyncAll)) backupTargetActions := map[string]func(http.ResponseWriter, *http.Request) error{ - "backupTargetSync": s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupTargetSync), + "backupTargetSync": s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupTargetSync), + "backupTargetUpdate": s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupTargetUpdate), } for name, action := range backupTargetActions { r.Methods("POST").Path("/v1/backuptargets/{backupTargetName}").Queries("action", name).Handler(f(schemas, action)) } + r.Methods("GET").Path("/v1/backuptargets/{name}").Handler(f(schemas, s.BackupTargetGet)) + r.Methods("POST").Path("/v1/backuptargets").Handler(f(schemas, s.BackupTargetCreate)) + r.Methods("DELETE").Path("/v1/backuptargets/{name}").Handler(f(schemas, s.BackupTargetDelete)) r.Methods("GET").Path("/v1/backupvolumes").Handler(f(schemas, s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupVolumeList))) r.Methods("PUT").Path("/v1/backupvolumes").Handler(f(schemas, s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupVolumeSyncAll))) r.Methods("GET").Path("/v1/backupvolumes/{volName}").Handler(f(schemas, s.fwd.Handler(s.fwd.HandleProxyRequestByNodeID, s.fwd.GetHTTPAddressByNodeID(NodeHasDefaultEngineImage(s.m)), s.BackupVolumeGet))) diff --git a/client/generated_backup.go b/client/generated_backup.go index ffa76a9f62..846c2455d4 100644 --- a/client/generated_backup.go +++ b/client/generated_backup.go @@ -7,6 +7,8 @@ const ( type Backup struct { Resource `yaml:"-"` + BackupTargetName string `json:"backupTargetName,omitempty" yaml:"backup_target_name,omitempty"` + CompressionMethod string `json:"compressionMethod,omitempty" yaml:"compression_method,omitempty"` Created string `json:"created,omitempty" yaml:"created,omitempty"` diff --git a/client/generated_backup_target.go b/client/generated_backup_target.go index 4393d39462..d36b4b1893 100644 --- a/client/generated_backup_target.go +++ b/client/generated_backup_target.go @@ -15,7 +15,11 @@ type BackupTarget struct { Message string `json:"message,omitempty" yaml:"message,omitempty"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` + PollInterval string `json:"pollInterval,omitempty" yaml:"poll_interval,omitempty"` + + ReadOnly bool `json:"readOnly,omitempty" yaml:"read_only,omitempty"` } type BackupTargetCollection struct { diff --git a/client/generated_backup_volume.go b/client/generated_backup_volume.go index b1936e1988..5727a19ee4 100644 --- a/client/generated_backup_volume.go +++ b/client/generated_backup_volume.go @@ -11,6 +11,8 @@ type BackupVolume struct { BackingImageName string `json:"backingImageName,omitempty" yaml:"backing_image_name,omitempty"` + BackupTargetName string `json:"backupTargetName,omitempty" yaml:"backup_target_name,omitempty"` + Created string `json:"created,omitempty" yaml:"created,omitempty"` DataStored string `json:"dataStored,omitempty" yaml:"data_stored,omitempty"` @@ -28,6 +30,8 @@ type BackupVolume struct { Size string `json:"size,omitempty" yaml:"size,omitempty"` StorageClassName string `json:"storageClassName,omitempty" yaml:"storage_class_name,omitempty"` + + VolumeName string `json:"volumeName,omitempty" yaml:"volume_name,omitempty"` } type BackupVolumeCollection struct { diff --git a/engineapi/types.go b/engineapi/types.go index 7854bffd0d..4cc4501a2d 100644 --- a/engineapi/types.go +++ b/engineapi/types.go @@ -149,6 +149,7 @@ type BackupTarget struct { PollInterval string `json:"pollInterval"` Available bool `json:"available"` Message string `json:"message"` + ReadyOnly bool `json:"readOnly"` } type BackupVolume struct { @@ -164,6 +165,8 @@ type BackupVolume struct { BackingImageName string `json:"backingImageName"` BackingImageChecksum string `json:"backingImageChecksum"` StorageClassName string `json:"storageClassName"` + BackupTargetName string `json:"backupTargetName"` + VolumeName string `json:"volumeName"` } type Backup struct { @@ -184,6 +187,7 @@ type Backup struct { Parameters map[string]string `json:"parameters"` NewlyUploadedDataSize string `json:"newlyUploadedDataSize"` ReUploadedDataSize string `json:"reUploadedDataSize"` + BackupTargetName string `json:"backupTargetName"` } type ConfigMetadata struct { diff --git a/manager/engine.go b/manager/engine.go index e54a3fadcf..8e46409a6e 100644 --- a/manager/engine.go +++ b/manager/engine.go @@ -10,10 +10,12 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/longhorn/longhorn-manager/datastore" "github.com/longhorn/longhorn-manager/engineapi" - longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" "github.com/longhorn/longhorn-manager/types" "github.com/longhorn/longhorn-manager/util" + + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ) const ( @@ -326,7 +328,7 @@ func (m *VolumeManager) GetRunningEngineByVolume(name string) (e *longhorn.Engin } func (m *VolumeManager) ListBackupTargetsSorted() ([]*longhorn.BackupTarget, error) { - backupTargetMap, err := m.ds.ListBackupTargets() + backupTargetMap, err := m.ds.ListBackupTargetsRO() if err != nil { return []*longhorn.BackupTarget{}, err } @@ -342,7 +344,64 @@ func (m *VolumeManager) ListBackupTargetsSorted() ([]*longhorn.BackupTarget, err } func (m *VolumeManager) GetBackupTarget(backupTargetName string) (*longhorn.BackupTarget, error) { - return m.ds.GetBackupTarget(backupTargetName) + backupTarget, err := m.ds.GetBackupTargetRO(backupTargetName) + if err != nil { + if apierrors.IsNotFound(err) { + // If the BackupTarget CR is not found, return succeeded result + return &longhorn.BackupTarget{ObjectMeta: metav1.ObjectMeta{Name: backupTargetName}}, nil + } + return nil, err + } + return backupTarget, nil +} + +func (m *VolumeManager) CreateBackupTarget(backupTargetName string, backupTargetSpec *longhorn.BackupTargetSpec) (*longhorn.BackupTarget, error) { + backupTarget, err := m.ds.GetBackupTarget(backupTargetName) + if err != nil { + if !datastore.ErrorIsNotFound(err) { + return nil, err + } + + // Create the default BackupTarget CR if not present + backupTarget, err = m.ds.CreateBackupTarget(&longhorn.BackupTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: backupTargetName, + }, + Spec: *backupTargetSpec, + }) + if err != nil { + return nil, err + } + } + return backupTarget, nil +} + +func (m *VolumeManager) UpdateBackupTarget(backupTargetName string, backupTargetSpec *longhorn.BackupTargetSpec) (*longhorn.BackupTarget, error) { + existingBackupTarget, err := m.ds.GetBackupTarget(backupTargetName) + if err != nil { + return nil, err + } + + if backupTargetSpec.BackupTargetURL != existingBackupTarget.Spec.BackupTargetURL || + backupTargetSpec.CredentialSecret != existingBackupTarget.Spec.CredentialSecret || + backupTargetSpec.PollInterval != existingBackupTarget.Spec.PollInterval || + backupTargetSpec.ReadOnly != existingBackupTarget.Spec.ReadOnly { + existingBackupTarget.Spec.BackupTargetURL = backupTargetSpec.BackupTargetURL + existingBackupTarget.Spec.CredentialSecret = backupTargetSpec.CredentialSecret + existingBackupTarget.Spec.PollInterval = backupTargetSpec.PollInterval + existingBackupTarget.Spec.ReadOnly = backupTargetSpec.ReadOnly + existingBackupTarget.Spec.SyncRequestedAt = metav1.Time{Time: time.Now().UTC()} + existingBackupTarget, err = m.ds.UpdateBackupTarget(existingBackupTarget) + if err != nil { + return nil, errors.Wrap(err, "failed to update backup target spec") + } + } + + return existingBackupTarget, nil +} + +func (m *VolumeManager) DeleteBackupTarget(backupTargetName string) error { + return m.ds.DeleteBackupTarget(backupTargetName) } func (m *VolumeManager) SyncBackupTarget(backupTarget *longhorn.BackupTarget) (*longhorn.BackupTarget, error) { From 9e2eea62bd56a39d6ffac9028ba361cdf9a29320 Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 26 Sep 2024 12:07:29 +0800 Subject: [PATCH 21/29] feat(backup): backup validation Ref: 5411 Signed-off-by: James Lu --- webhook/resources/backup/validator.go | 42 +++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/webhook/resources/backup/validator.go b/webhook/resources/backup/validator.go index 12d8019347..991fba5dfe 100644 --- a/webhook/resources/backup/validator.go +++ b/webhook/resources/backup/validator.go @@ -7,8 +7,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "github.com/longhorn/longhorn-manager/datastore" + "github.com/longhorn/longhorn-manager/util" "github.com/longhorn/longhorn-manager/webhook/admission" werror "github.com/longhorn/longhorn-manager/webhook/error" + "github.com/pkg/errors" longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ) @@ -24,12 +26,14 @@ func NewValidator(ds *datastore.DataStore) admission.Validator { func (b *backupValidator) Resource() admission.Resource { return admission.Resource{ - Name: "backups", - Scope: admissionregv1.NamespacedScope, - APIGroup: longhorn.SchemeGroupVersion.Group, - APIVersion: longhorn.SchemeGroupVersion.Version, - ObjectType: &longhorn.Backup{}, - OperationTypes: []admissionregv1.OperationType{}, + Name: "backups", + Scope: admissionregv1.NamespacedScope, + APIGroup: longhorn.SchemeGroupVersion.Group, + APIVersion: longhorn.SchemeGroupVersion.Version, + ObjectType: &longhorn.Backup{}, + OperationTypes: []admissionregv1.OperationType{ + admissionregv1.Create, + }, } } @@ -39,10 +43,36 @@ func (b *backupValidator) Create(request *admission.Request, newObj runtime.Obje return werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.Backup", newObj), "") } + if !util.ValidateName(backup.Name) { + return werror.NewInvalidError(fmt.Sprintf("invalid name %v", backup.Name), "") + } + if backup.Spec.BackupMode != longhorn.BackupModeFull && backup.Spec.BackupMode != longhorn.BackupModeIncremental { return werror.NewInvalidError(fmt.Sprintf("BackupMode %v is not a valid option", backup.Spec.BackupMode), "") } + if backup.Spec.SnapshotName != "" { + snap, err := b.ds.GetSnapshotRO(backup.Spec.SnapshotName) + if err != nil { + return werror.NewInvalidError(errors.Wrapf(err, "failed to get snapshot %v of backup %v", backup.Spec.SnapshotName, backup.Name).Error(), "") + } + volume, err := b.ds.GetVolumeRO(snap.Spec.Volume) + if err != nil { + return werror.NewInvalidError(errors.Wrapf(err, "failed to get volume %v of snapshot %v", snap.Spec.Volume, backup.Spec.SnapshotName).Error(), "") + } + if backup.Spec.BackupTargetName != volume.Spec.BackupTargetName { + return werror.NewInvalidError(fmt.Sprintf("failed to create a new backup %v on a different backup target %v from the volume %v (%v)", backup.Name, backup.Spec.BackupTargetName, volume.Name, volume.Spec.BackupTargetName), "") + } + + backupTarget, err := b.ds.GetBackupTargetRO(backup.Spec.BackupTargetName) + if err != nil { + return werror.NewInvalidError(errors.Wrapf(err, "failed to get backup target").Error(), "") + } + if backupTarget.Spec.ReadOnly { + return werror.NewInvalidError(fmt.Sprintf("failed to create a new backup %v on a read-only backup target %v ", backup.Name, backupTarget.Name), "") + } + } + return nil } From 84dc92ad78152c1c615346276feba5a905f29268 Mon Sep 17 00:00:00 2001 From: James Lu Date: Tue, 24 Sep 2024 17:06:10 +0800 Subject: [PATCH 22/29] feat(backup): modify volume controller unit tests for backup targets Add more parameters in creating BackupVolume for testing volume_controller.go Ref: 5411 Signed-off-by: James Lu --- controller/controller_test.go | 1 + controller/volume_controller_test.go | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/controller/controller_test.go b/controller/controller_test.go index eefcaea71a..7ced95d408 100644 --- a/controller/controller_test.go +++ b/controller/controller_test.go @@ -88,6 +88,7 @@ const ( TestDeploymentName = "test-deployment" TestBackupTarget = "s3://backupbucket@us-east-1/backupstore" + TestBackupTargetName = "default" TestBackupVolumeName = "test-backup-volume-for-restoration" TestBackupName = "test-backup-for-restoration" diff --git a/controller/volume_controller_test.go b/controller/volume_controller_test.go index 6e1eab29f0..b9977dd515 100644 --- a/controller/volume_controller_test.go +++ b/controller/volume_controller_test.go @@ -1383,6 +1383,14 @@ func (s *TestSuite) runTestCases(c *C, testCases map[string]*VolumeTestCase) { Finalizers: []string{ longhorn.SchemeGroupVersion.Group, }, + Labels: map[string]string{ + types.LonghornLabelBackupTarget: TestBackupTargetName, + types.LonghornLabelBackupVolume: bvName, + }, + }, + Spec: longhorn.BackupVolumeSpec{ + BackupTargetName: TestBackupTargetName, + VolumeName: bvName, }, Status: longhorn.BackupVolumeStatus{ LastBackupName: bName, From 0a18099e171b0aa758d4d90d76d4c7f16121b35b Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 26 Sep 2024 10:03:39 +0800 Subject: [PATCH 23/29] feat(backup): modify volume controller unit tests for backup targets Add more parameters in creating BackupVolume for testing volume_controller.go Ref: 5411 Signed-off-by: James Lu --- controller/volume_controller_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/controller/volume_controller_test.go b/controller/volume_controller_test.go index b9977dd515..0717c775c5 100644 --- a/controller/volume_controller_test.go +++ b/controller/volume_controller_test.go @@ -1137,6 +1137,7 @@ func newVolume(name string, replicaCount int) *longhorn.Volume { StaleReplicaTimeout: TestVolumeStaleTimeout, Image: TestEngineImage, DataEngine: longhorn.DataEngineTypeV1, + BackupTargetName: TestBackupTargetName, }, Status: longhorn.VolumeStatus{ OwnerID: TestOwnerID1, From 3b067ba6544fce22b04d2fa20a280096b420f94e Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 25 Sep 2024 15:01:15 +0800 Subject: [PATCH 24/29] feat(backup): modify getting BackupVolume for csi backup Ref: 5411 Signed-off-by: James Lu --- csi/controller_server.go | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/csi/controller_server.go b/csi/controller_server.go index b58f7fd3e8..528b261aed 100644 --- a/csi/controller_server.go +++ b/csi/controller_server.go @@ -127,7 +127,7 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, status.Errorf(codes.NotFound, "volume source snapshot %v is not found", snapshot.SnapshotId) } backupVolume, backupName := sourceVolumeName, id - bv, err := cs.apiClient.BackupVolume.ById(backupVolume) + bv, err := cs.getBackupVolume(backupVolume) if err != nil { return nil, status.Errorf(codes.NotFound, "failed to restore CSI snapshot %s backup volume %s unavailable", snapshot.SnapshotId, backupVolume) } @@ -265,6 +265,32 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol }, nil } +func (cs *ControllerServer) getBackupVolume(volumeName string) (*longhornclient.BackupVolume, error) { + vol, err := cs.apiClient.Volume.ById(volumeName) + if err != nil { + return nil, fmt.Errorf("waitForSnapshotToBeReady: fail to get source volume %v: %v", volumeName, err) + } + + list, err := cs.apiClient.BackupVolume.List(&longhornclient.ListOpts{ + Filters: map[string]interface{}{ + types.LonghornLabelBackupTarget: vol.BackupTargetName, + types.LonghornLabelBackupVolume: volumeName, + }}) + if err != nil { + return nil, err + } + + if len(list.Data) >= 2 { + return nil, fmt.Errorf("found more than one backup volume %v of the backup target %v", volumeName, vol.BackupTargetName) + } + + for _, backupVolume := range list.Data { + return &backupVolume, nil + } + + return nil, fmt.Errorf("failed to find backup volume %v of the backup target %s: %v", volumeName, vol.BackupTargetName, err) +} + func (cs *ControllerServer) checkAndPrepareBackingImage(volumeName, backingImageName string, volumeParameters map[string]string) error { if backingImageName == "" { return nil @@ -1042,7 +1068,7 @@ func (cs *ControllerServer) cleanupSnapshot(sourceVolumeName, id string) error { func (cs *ControllerServer) cleanupBackupVolume(sourceVolumeName, id string) error { backupVolumeName, backupName := sourceVolumeName, id - backupVolume, err := cs.apiClient.BackupVolume.ById(backupVolumeName) + backupVolume, err := cs.getBackupVolume(backupVolumeName) if err != nil { return err } @@ -1258,7 +1284,7 @@ func (cs *ControllerServer) waitForBackupControllerSync(volumeName, snapshotName // (and in particular, its name) as quickly as possible and in any state. func (cs *ControllerServer) getBackup(volumeName, snapshotName string) (*longhornclient.Backup, error) { // Successfully returns an empty BackupVolume with volumeName even if one doesn't exist. - backupVolume, err := cs.apiClient.BackupVolume.ById(volumeName) + backupVolume, err := cs.getBackupVolume(volumeName) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } From cf734d6551a3ba65967fa9a5bd038aa3aee5c24f Mon Sep 17 00:00:00 2001 From: James Lu Date: Tue, 24 Sep 2024 18:59:35 +0800 Subject: [PATCH 25/29] feat(backup): modify controllers related to backup backing image Ref: 5411 Signed-off-by: James Lu --- controller/backup_backing_image_controller.go | 10 +++--- controller/backup_controller.go | 16 +++++++--- controller/backup_target_controller.go | 2 +- datastore/longhorn.go | 32 ++++++++++++++++++- engineapi/backup_backing_image.go | 7 ++-- types/types.go | 7 ++++ 6 files changed, 61 insertions(+), 13 deletions(-) diff --git a/controller/backup_backing_image_controller.go b/controller/backup_backing_image_controller.go index c8cc99db07..14cc7f0d4f 100644 --- a/controller/backup_backing_image_controller.go +++ b/controller/backup_backing_image_controller.go @@ -203,18 +203,18 @@ func (bc *BackupBackingImageController) reconcile(backupBackingImageName string) log := getLoggerForBackupBackingImage(bc.logger, bbi) - // Get default backup target - backupTarget, err := bc.ds.GetBackupTargetRO(types.DefaultBackupTargetName) + // Get backup target with the backup backing image spec backup target name + backupTarget, err := bc.ds.GetBackupTargetRO(bbi.Spec.BackupTargetName) if err != nil { if apierrors.IsNotFound(err) { return nil } - return errors.Wrapf(err, "failed to get the backup target %v", types.DefaultBackupTargetName) + return errors.Wrapf(err, "failed to get the backup target %v", bbi.Spec.BackupTargetName) } // Examine DeletionTimestamp to determine if object is under deletion if !bbi.DeletionTimestamp.IsZero() { - if backupTarget.Spec.BackupTargetURL != "" { + if IsBackupTargetAvailable(backupTarget) { backupTargetClient, err := newBackupTargetClientFromDefaultEngineImage(bc.ds, backupTarget) if err != nil { log.WithError(err).Warn("Failed to init backup target clients") @@ -256,7 +256,7 @@ func (bc *BackupBackingImageController) reconcile(backupBackingImageName string) }() if bbi.Status.LastSyncedAt.IsZero() && bbi.Spec.UserCreated && bc.backupNotInFinalState(bbi) { - backingImage, err := bc.ds.GetBackingImage(backupBackingImageName) + backingImage, err := bc.ds.GetBackingImage(bbi.Spec.BackingImage) if err != nil { if !apierrors.IsNotFound(err) { return err diff --git a/controller/backup_controller.go b/controller/backup_controller.go index eec3a558e5..94d963bb11 100644 --- a/controller/backup_controller.go +++ b/controller/backup_controller.go @@ -659,7 +659,8 @@ func (bc *BackupController) backupBackingImage(volume *longhorn.Volume) error { return errors.Wrapf(err, "failed to get backing image %v", biName) } - bbi, err := bc.ds.GetBackupBackingImage(biName) + backupTargetName := volume.Spec.BackupTargetName + bbi, err := bc.ds.GetBackupBackingImagesWithBackupTargetNameRO(backupTargetName, biName) if err != nil { if !apierrors.IsNotFound(err) { return errors.Wrapf(err, "failed to get backup backing image %v", biName) @@ -667,16 +668,23 @@ func (bc *BackupController) backupBackingImage(volume *longhorn.Volume) error { } if bbi == nil { + bbiName := types.GetBackupBackingImageNameFromBIName(biName) backupBackingImage := &longhorn.BackupBackingImage{ ObjectMeta: metav1.ObjectMeta{ - Name: biName, + Name: bbiName, + Labels: map[string]string{ + types.LonghornLabelBackupTarget: backupTargetName, + types.LonghornLabelBackingImage: biName, + }, }, Spec: longhorn.BackupBackingImageSpec{ - UserCreated: true, + UserCreated: true, + BackingImage: biName, + BackupTargetName: backupTargetName, }, } if _, err = bc.ds.CreateBackupBackingImage(backupBackingImage); err != nil && !apierrors.IsAlreadyExists(err) { - return errors.Wrapf(err, "failed to create backup backing image %s in the cluster", biName) + return errors.Wrapf(err, "failed to create backup backing image %s for backing image %s of backup target %s in the cluster", bbiName, biName, backupTargetName) } } diff --git a/controller/backup_target_controller.go b/controller/backup_target_controller.go index 6789a610a7..a0d40137ea 100644 --- a/controller/backup_target_controller.go +++ b/controller/backup_target_controller.go @@ -662,7 +662,7 @@ func (btc *BackupTargetController) syncBackupBackingImage(backupTargetName strin Spec: longhorn.BackupBackingImageSpec{ UserCreated: false, BackupTargetName: backupTargetName, - BackingImage: backupBackingImageName, + BackingImage: remoteBackingImageName, }, } if _, err = btc.ds.CreateBackupBackingImage(backupBackingImage); err != nil && !apierrors.IsAlreadyExists(err) { diff --git a/datastore/longhorn.go b/datastore/longhorn.go index 4492c2224c..bc54d41683 100644 --- a/datastore/longhorn.go +++ b/datastore/longhorn.go @@ -19,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation" k8sruntime "k8s.io/apimachinery/pkg/runtime" @@ -4262,7 +4263,7 @@ func (s *DataStore) GetBackupVolumeWithBackupTargetAndVolumeRO(backupTargetName, return itemRO, nil } - return nil, nil + return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "backupVolumes"}, "") } // GetBackupVolumeWithBackupTargetAndVolume returns a copy of BackupVolume with the given backup target and volume name in the cluster @@ -5660,6 +5661,35 @@ func (s *DataStore) GetBackupBackingImage(name string) (*longhorn.BackupBackingI return resultRO.DeepCopy(), nil } +// GetBackupBackingImagesWithBackupTargetBackingImageRO returns a new BackupBackingImage object for the given backup target and backing image name +func (s *DataStore) GetBackupBackingImagesWithBackupTargetNameRO(backupTargetName, backingImageName string) (*longhorn.BackupBackingImage, error) { + selector, err := getBackingImageWithBackupTargetSelector(backupTargetName, backingImageName) + if err != nil { + return nil, err + } + + list, err := s.backupBackingImageLister.BackupBackingImages(s.namespace).List(selector) + if err != nil { + return nil, err + } + + if len(list) >= 2 { + return nil, fmt.Errorf("datastore: found more than one backup backing image with backup target %v and backing image %v", backupTargetName, backingImageName) + } + + for _, itemRO := range list { + return itemRO, nil + } + + return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "backupBackingImages"}, "") +} + +func getBackingImageWithBackupTargetSelector(backupTargetName, backingImageName string) (labels.Selector, error) { + return metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + MatchLabels: types.GetBackingImageWithBackupTargetLabels(backupTargetName, backingImageName), + }) +} + // ListBackupBackingImages returns object includes all BackupBackingImage in namespace func (s *DataStore) ListBackupBackingImages() (map[string]*longhorn.BackupBackingImage, error) { itemMap := map[string]*longhorn.BackupBackingImage{} diff --git a/engineapi/backup_backing_image.go b/engineapi/backup_backing_image.go index 46c4b48428..88b1c362e3 100644 --- a/engineapi/backup_backing_image.go +++ b/engineapi/backup_backing_image.go @@ -89,6 +89,7 @@ type BackupBackingImageMonitor struct { client *BackingImageManagerClient backupBackingImageStatus longhorn.BackupBackingImageStatus + backingImageName string backupStatusLock sync.RWMutex syncCallback func(key string) @@ -101,10 +102,12 @@ type BackupBackingImageMonitor struct { func NewBackupBackingImageMonitor(logger logrus.FieldLogger, ds *datastore.DataStore, bbi *longhorn.BackupBackingImage, backingImage *longhorn.BackingImage, backupTargetClient *BackupTargetClient, compressionMethod longhorn.BackupCompressionMethod, concurrentLimit int, bimClient *BackingImageManagerClient, syncCallback func(key string)) (*BackupBackingImageMonitor, error) { ctx, quit := context.WithCancel(context.Background()) + biName := backingImage.Name m := &BackupBackingImageMonitor{ logger: logger.WithFields(logrus.Fields{"backupBackingImage": bbi.Name}), backupBackingImageName: bbi.Name, + backingImageName: biName, client: bimClient, backupStatusLock: sync.RWMutex{}, @@ -120,7 +123,7 @@ func NewBackupBackingImageMonitor(logger logrus.FieldLogger, ds *datastore.DataS // Call backing image manager API snapshot backup if bbi.Status.State == longhorn.BackupStateNew { - err := m.client.BackupCreate(bbi.Name, backingImage.Status.UUID, bbi.Status.Checksum, + err := m.client.BackupCreate(biName, backingImage.Status.UUID, bbi.Status.Checksum, backupTargetClient.URL, bbi.Spec.Labels, backupTargetClient.Credential, string(compressionMethod), concurrentLimit, backupBackingImageParameters) if err != nil { if !strings.Contains(err.Error(), "DeadlineExceeded") { @@ -243,7 +246,7 @@ func (m *BackupBackingImageMonitor) syncBackupStatusFromBackingImageManager() (c m.backupBackingImageStatus.DeepCopyInto(¤tBackupStatus) m.backupStatusLock.RUnlock() - backupBackingImageStatus, err = m.client.BackupStatus(m.backupBackingImageName) + backupBackingImageStatus, err = m.client.BackupStatus(m.backingImageName) if err != nil { return currentBackupStatus, err } diff --git a/types/types.go b/types/types.go index 535ed30e16..a13b845394 100644 --- a/types/types.go +++ b/types/types.go @@ -540,6 +540,13 @@ func GetBackingImageLabels() map[string]string { return labels } +func GetBackingImageWithBackupTargetLabels(backupTargetName, backingImageName string) map[string]string { + return map[string]string{ + LonghornLabelBackingImage: backingImageName, + LonghornLabelBackupTarget: backupTargetName, + } +} + func GetBackingImageManagerLabels(nodeID, diskUUID string) map[string]string { labels := GetBaseLabelsForSystemManagedComponent() labels[GetLonghornLabelComponentKey()] = LonghornLabelBackingImageManager From 2158d2be792cece5766f8995d5c160644728df26 Mon Sep 17 00:00:00 2001 From: James Lu Date: Tue, 24 Sep 2024 19:37:04 +0800 Subject: [PATCH 26/29] feat(backup): modify backing image data source controller Ref: 5411 Signed-off-by: James Lu --- .../backing_image_data_source_controller.go | 4 ++-- .../longhorn/v1beta2/backingimagedatasource.go | 1 + manager/backupbackingimage.go | 10 ++++++++-- manager/volume.go | 15 ++++++++++----- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/controller/backing_image_data_source_controller.go b/controller/backing_image_data_source_controller.go index a160f5b10a..c3d318702b 100644 --- a/controller/backing_image_data_source_controller.go +++ b/controller/backing_image_data_source_controller.go @@ -711,12 +711,12 @@ func (c *BackingImageDataSourceController) generateBackingImageDataSourcePodMani if bids.Spec.SourceType == longhorn.BackingImageDataSourceTypeRestore { var credential map[string]string - backupTarget, err := c.ds.GetBackupTargetRO(types.DefaultBackupTargetName) + backupTarget, err := c.ds.GetBackupTargetRO(bids.Spec.Parameters[longhorn.DataSourceTypeRestoreParameterBackupTargetName]) if err != nil { if apierrors.IsNotFound(err) { return nil, err } - return nil, errors.Wrapf(err, "failed to get the backup target %v", types.DefaultBackupTargetName) + return nil, errors.Wrapf(err, "failed to get the backup target %v", bids.Spec.Parameters[longhorn.DataSourceTypeRestoreParameterBackupTargetName]) } backupType, err := util.CheckBackupType(backupTarget.Spec.BackupTargetURL) diff --git a/k8s/pkg/apis/longhorn/v1beta2/backingimagedatasource.go b/k8s/pkg/apis/longhorn/v1beta2/backingimagedatasource.go index a592795234..320209db77 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/backingimagedatasource.go +++ b/k8s/pkg/apis/longhorn/v1beta2/backingimagedatasource.go @@ -23,6 +23,7 @@ const ( DataSourceTypeExportFromVolumeParameterSnapshotName = "snapshot-name" DataSourceTypeExportFromVolumeParameterSenderAddress = "sender-address" DataSourceTypeExportFromVolumeParameterFileSyncHTTPClientTimeout = "file-sync-http-client-timeout" + DataSourceTypeRestoreParameterBackupTargetName = "backup-target-name" DataSourceTypeRestoreParameterBackupURL = "backup-url" DataSourceTypeRestoreParameterConcurrentLimit = "concurrent-limit" DataSourceTypeCloneParameterBackingImage = "backing-image" diff --git a/manager/backupbackingimage.go b/manager/backupbackingimage.go index ef14945320..188ec824d7 100644 --- a/manager/backupbackingimage.go +++ b/manager/backupbackingimage.go @@ -41,7 +41,13 @@ func (m *VolumeManager) RestoreBackupBackingImage(name string, secret, secretNam if name == "" { return fmt.Errorf("restore backing image name is not given") } - bi, err := m.ds.GetBackingImageRO(name) + + bbi, err := m.ds.GetBackupBackingImageRO(name) + if err != nil { + return err + } + biName := bbi.Spec.BackingImage + bi, err := m.ds.GetBackingImageRO(biName) if err != nil { if !apierrors.IsNotFound(err) { return errors.Wrapf(err, "failed to get backing image %v to check if it exists", name) @@ -52,7 +58,7 @@ func (m *VolumeManager) RestoreBackupBackingImage(name string, secret, secretNam return errors.Wrapf(err, "backing image %v already exists", name) } - return m.restoreBackingImage(name, secret, secretNamespace) + return m.restoreBackingImage(bbi.Spec.BackupTargetName, biName, secret, secretNamespace) } func (m *VolumeManager) CreateBackupBackingImage(name, backingImageName, backupTargetName string) error { diff --git a/manager/volume.go b/manager/volume.go index 0da196f475..3f05a842ba 100644 --- a/manager/volume.go +++ b/manager/volume.go @@ -154,7 +154,7 @@ func (m *VolumeManager) Create(name string, spec *longhorn.VolumeSpec, recurring // TODO: We should record the secret and secret namespace in the backing image // so we can auto fill in the secret and namespace when auto restore the backing image in volume creation api. // Currently, if the backing image is encrypted, users need to restore it first so they can specifically assign the secret and namespace - if err := m.restoreBackingImage(spec.BackingImage, "", ""); err != nil { + if err := m.restoreBackingImage(spec.BackupTargetName, spec.BackingImage, "", ""); err != nil { return nil, errors.Wrapf(err, "failed to restore backing image %v when create volume %v", spec.BackingImage, v.Name) } @@ -192,6 +192,7 @@ func (m *VolumeManager) Create(name string, spec *longhorn.VolumeSpec, recurring ReplicaDiskSoftAntiAffinity: spec.ReplicaDiskSoftAntiAffinity, DataEngine: spec.DataEngine, FreezeFilesystemForSnapshot: spec.FreezeFilesystemForSnapshot, + BackupTargetName: spec.BackupTargetName, }, } @@ -1176,7 +1177,7 @@ func (m *VolumeManager) UpdateSnapshotMaxSize(name string, snapshotMaxSize int64 return v, nil } -func (m *VolumeManager) restoreBackingImage(biName, secret, secretNamespace string) error { +func (m *VolumeManager) restoreBackingImage(backupTargetName, biName, secret, secretNamespace string) error { if secret != "" || secretNamespace != "" { _, err := m.ds.GetSecretRO(secretNamespace, secret) if err != nil { @@ -1199,7 +1200,10 @@ func (m *VolumeManager) restoreBackingImage(biName, secret, secretNamespace stri } // try find the backup backing image - bbi, err := m.ds.GetBackupBackingImageRO(biName) + if backupTargetName == "" { + backupTargetName = types.DefaultBackupTargetName + } + bbi, err := m.ds.GetBackupBackingImagesWithBackupTargetNameRO(backupTargetName, biName) if err != nil { return errors.Wrapf(err, "failed to get backup backing image %v", biName) } @@ -1218,8 +1222,9 @@ func (m *VolumeManager) restoreBackingImage(biName, secret, secretNamespace stri Checksum: bbi.Status.Checksum, SourceType: longhorn.BackingImageDataSourceTypeRestore, SourceParameters: map[string]string{ - longhorn.DataSourceTypeRestoreParameterBackupURL: bbi.Status.URL, - longhorn.DataSourceTypeRestoreParameterConcurrentLimit: strconv.FormatInt(concurrentLimit, 10), + longhorn.DataSourceTypeRestoreParameterBackupTargetName: backupTargetName, + longhorn.DataSourceTypeRestoreParameterBackupURL: bbi.Status.URL, + longhorn.DataSourceTypeRestoreParameterConcurrentLimit: strconv.FormatInt(concurrentLimit, 10), }, Secret: secret, SecretNamespace: secretNamespace, From 1ceae2e2de55d07237386bacd102962bef7879d3 Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 26 Sep 2024 11:33:20 +0800 Subject: [PATCH 27/29] feat(backup): modify system backup controller Ref: 5411 Signed-off-by: James Lu --- controller/system_backup_controller.go | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/controller/system_backup_controller.go b/controller/system_backup_controller.go index 3b6245088f..a3a83973a6 100644 --- a/controller/system_backup_controller.go +++ b/controller/system_backup_controller.go @@ -849,12 +849,17 @@ func (c *SystemBackupController) createVolumeBackup(volume *longhorn.Volume, sys return nil, errors.Wrapf(err, "failed to create Volume %v snapshot %s", volume.Name, snapshot.Name) } + backupTargetName := volume.Spec.BackupTargetName backup = &longhorn.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: volumeBackupName, + Labels: map[string]string{ + types.LonghornLabelBackupTarget: backupTargetName, + }, }, Spec: longhorn.BackupSpec{ - SnapshotName: snapshot.Name, + SnapshotName: snapshot.Name, + BackupTargetName: backupTargetName, }, } backup, err = c.ds.CreateBackup(backup, volume.Name) @@ -946,25 +951,34 @@ func (c *SystemBackupController) WaitForBackingImageBackupToComplete(backupBacki } func (c *SystemBackupController) createBackingImageBackup(backingImage *longhorn.BackingImage) (backupBackingImage *longhorn.BackupBackingImage, err error) { - backupBackingImage, err = c.ds.GetBackupBackingImage(backingImage.Name) + backupTargetName := types.DefaultBackupTargetName + backingImageName := backingImage.Name + backupBackingImage, err = c.ds.GetBackupBackingImagesWithBackupTargetNameRO(backupTargetName, backingImageName) if err != nil { if !apierrors.IsNotFound(err) { - return nil, errors.Wrapf(err, "failed to get backup backing image %v", backingImage.Name) + return nil, errors.Wrapf(err, "failed to get backup backing image %v", backingImageName) } } if backupBackingImage == nil { + backupBackingImageName := types.GetBackupBackingImageNameFromBIName(backingImageName) backupBackingImage = &longhorn.BackupBackingImage{ ObjectMeta: metav1.ObjectMeta{ - Name: backingImage.Name, + Name: backupBackingImageName, + Labels: map[string]string{ + types.LonghornLabelBackingImage: backingImageName, + types.LonghornLabelBackupTarget: backupTargetName, + }, }, Spec: longhorn.BackupBackingImageSpec{ - UserCreated: true, + UserCreated: true, + BackingImage: backingImageName, + BackupTargetName: backupTargetName, }, } backupBackingImage, err = c.ds.CreateBackupBackingImage(backupBackingImage) if err != nil && !apierrors.IsAlreadyExists(err) { - return nil, errors.Wrapf(err, "failed to create backup backing image %s", backingImage.Name) + return nil, errors.Wrapf(err, "failed to create backup backing image %s", backingImageName) } } return backupBackingImage, nil From 99e9b3500086424065cfce6c74f65d699fd9721e Mon Sep 17 00:00:00 2001 From: James Lu Date: Wed, 25 Sep 2024 09:09:39 +0800 Subject: [PATCH 28/29] feat(backup): upgrade CRs for multiple backup store Upgrade CRs for Volume, BackupVolume, Backup, BackupBackingImage, and BackingImageDataSource. Ref: 5411 Signed-off-by: James Lu --- types/types.go | 3 + upgrade/util/util.go | 60 ++++++++++++ upgrade/v17xto180/upgrade.go | 179 +++++++++++++++++++++++++++++++++++ 3 files changed, 242 insertions(+) diff --git a/types/types.go b/types/types.go index a13b845394..37ba442020 100644 --- a/types/types.go +++ b/types/types.go @@ -29,6 +29,9 @@ const ( LonghornKindEngine = "Engine" LonghornKindReplica = "Replica" LonghornKindBackup = "Backup" + LonghornKindBackupBackingImage = "BackupBackingImage" + LonghornKindBackupTarget = "BackupTarget" + LonghornKindBackupVolume = "BackupVolume" LonghornKindSnapshot = "Snapshot" LonghornKindEngineImage = "EngineImage" LonghornKindInstanceManager = "InstanceManager" diff --git a/upgrade/util/util.go b/upgrade/util/util.go index cad3038e55..3bbaaada1b 100644 --- a/upgrade/util/util.go +++ b/upgrade/util/util.go @@ -532,6 +532,46 @@ func ListAndUpdateEnginesInProvidedCache(namespace string, lhClient *lhclientset return engines, nil } +// ListAndUpdateBackupTargetsInProvidedCache list all backup targets and save them into the provided cached `resourceMap`. This method is not thread-safe. +func ListAndUpdateBackupTargetsInProvidedCache(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (map[string]*longhorn.BackupTarget, error) { + if v, ok := resourceMaps[types.LonghornKindBackupTarget]; ok { + return v.(map[string]*longhorn.BackupTarget), nil + } + + backupTargets := map[string]*longhorn.BackupTarget{} + backupTargetList, err := lhClient.LonghornV1beta2().BackupTargets(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + for i, backupTarget := range backupTargetList.Items { + backupTargets[backupTarget.Name] = &backupTargetList.Items[i] + } + + resourceMaps[types.LonghornKindBackupTarget] = backupTargets + + return backupTargets, nil +} + +// ListAndUpdateBackupVolumesInProvidedCache list all backup volumes and save them into the provided cached `resourceMap`. This method is not thread-safe. +func ListAndUpdateBackupVolumesInProvidedCache(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (map[string]*longhorn.BackupVolume, error) { + if v, ok := resourceMaps[types.LonghornKindBackupVolume]; ok { + return v.(map[string]*longhorn.BackupVolume), nil + } + + backupVolumes := map[string]*longhorn.BackupVolume{} + backupVolumeList, err := lhClient.LonghornV1beta2().BackupVolumes(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + for i, backupVolume := range backupVolumeList.Items { + backupVolumes[backupVolume.Name] = &backupVolumeList.Items[i] + } + + resourceMaps[types.LonghornKindBackupVolume] = backupVolumes + + return backupVolumes, nil +} + // ListAndUpdateBackupsInProvidedCache list all backups and save them into the provided cached `resourceMap`. This method is not thread-safe. func ListAndUpdateBackupsInProvidedCache(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (map[string]*longhorn.Backup, error) { if v, ok := resourceMaps[types.LonghornKindBackup]; ok { @@ -552,6 +592,26 @@ func ListAndUpdateBackupsInProvidedCache(namespace string, lhClient *lhclientset return backups, nil } +// ListAndUpdateBackupBackingImagesInProvidedCache list all backupBackingImages and save them into the provided cached `resourceMap`. This method is not thread-safe. +func ListAndUpdateBackupBackingImagesInProvidedCache(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (map[string]*longhorn.BackupBackingImage, error) { + if v, ok := resourceMaps[types.LonghornKindBackupBackingImage]; ok { + return v.(map[string]*longhorn.BackupBackingImage), nil + } + + bbis := map[string]*longhorn.BackupBackingImage{} + bbisList, err := lhClient.LonghornV1beta2().BackupBackingImages(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + for i, bids := range bbisList.Items { + bbis[bids.Name] = &bbisList.Items[i] + } + + resourceMaps[types.LonghornKindBackupBackingImage] = bbis + + return bbis, nil +} + // ListAndUpdateSnapshotsInProvidedCache list all snapshots and save them into the provided cached `resourceMap`. This method is not thread-safe. func ListAndUpdateSnapshotsInProvidedCache(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (map[string]*longhorn.Snapshot, error) { if v, ok := resourceMaps[types.LonghornKindSnapshot]; ok { diff --git a/upgrade/v17xto180/upgrade.go b/upgrade/v17xto180/upgrade.go index 2948aa8f5c..39a5fb3cbf 100644 --- a/upgrade/v17xto180/upgrade.go +++ b/upgrade/v17xto180/upgrade.go @@ -8,6 +8,7 @@ import ( "github.com/longhorn/longhorn-manager/types" + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" lhclientset "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned" upgradeutil "github.com/longhorn/longhorn-manager/upgrade/util" ) @@ -17,9 +18,187 @@ const ( ) func UpgradeResources(namespace string, lhClient *lhclientset.Clientset, kubeClient *clientset.Clientset, resourceMaps map[string]interface{}) error { + if err := upgradeVolumes(namespace, lhClient, resourceMaps); err != nil { + return err + } + + if err := upgradeBackupVolumes(namespace, lhClient, resourceMaps); err != nil { + return err + } + if err := upgradeBackups(namespace, lhClient, resourceMaps); err != nil { + return err + } + + if err := upgradeBackupBackingImages(namespace, lhClient, resourceMaps); err != nil { + return err + } + + if err := upgradeBackingImageDataSources(namespace, lhClient, resourceMaps); err != nil { + return err + } + return upgradeBackingImages(namespace, lhClient, resourceMaps) } +func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) { + defer func() { + err = errors.Wrapf(err, upgradeLogPrefix+"upgrade volume failed") + }() + + volumesMap, err := upgradeutil.ListAndUpdateVolumesInProvidedCache(namespace, lhClient, resourceMaps) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return errors.Wrapf(err, "failed to list all existing Longhorn volumes during the volume upgrade") + } + + for _, v := range volumesMap { + if v.Spec.BackupTargetName == "" { + v.Spec.BackupTargetName = types.DefaultBackupTargetName + } + } + + return nil +} + +func upgradeBackupVolumes(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) { + defer func() { + err = errors.Wrapf(err, upgradeLogPrefix+"upgrade backup volume failed") + }() + + backupVolumeMap, err := upgradeutil.ListAndUpdateBackupVolumesInProvidedCache(namespace, lhClient, resourceMaps) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return errors.Wrapf(err, "failed to list all existing Longhorn BackupVolumes during the Longhorn BackupVolume upgrade") + } + + for _, bv := range backupVolumeMap { + if bv.Spec.BackupTargetName == "" { + bv.Spec.BackupTargetName = types.DefaultBackupTargetName + } + + if bv.Spec.VolumeName == "" { + bv.Spec.VolumeName = bv.Name + } + + if bv.Labels == nil { + bv.Labels = make(map[string]string) + } + + _, exists := bv.Labels[types.LonghornLabelBackupVolume] + if !exists { + bv.Labels[types.LonghornLabelBackupVolume] = bv.Spec.VolumeName + } + + _, exists = bv.Labels[types.LonghornLabelBackupTarget] + if !exists { + bv.Labels[types.LonghornLabelBackupTarget] = types.DefaultBackupTargetName + } + } + + return nil +} + +func upgradeBackups(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) { + defer func() { + err = errors.Wrapf(err, upgradeLogPrefix+"upgrade backup failed") + }() + + backupMap, err := upgradeutil.ListAndUpdateBackupsInProvidedCache(namespace, lhClient, resourceMaps) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return errors.Wrapf(err, "failed to list all existing Longhorn Backups during the Longhorn Backup upgrade") + } + + for _, b := range backupMap { + if b.Spec.BackupTargetName == "" { + b.Spec.BackupTargetName = types.DefaultBackupTargetName + } + + if b.Labels == nil { + b.Labels = make(map[string]string) + } + + _, exists := b.Labels[types.LonghornLabelBackupTarget] + if !exists { + b.Labels[types.LonghornLabelBackupTarget] = b.Spec.BackupTargetName + } + + _, exists = b.Labels[types.LonghornLabelBackupVolume] + if !exists { + b.Labels[types.LonghornLabelBackupVolume] = b.Status.VolumeName + } + } + + return nil +} + +func upgradeBackupBackingImages(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) { + defer func() { + err = errors.Wrapf(err, upgradeLogPrefix+"upgrade backup backing images failed") + }() + + backupBackingImageMap, err := upgradeutil.ListAndUpdateBackupBackingImagesInProvidedCache(namespace, lhClient, resourceMaps) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return errors.Wrapf(err, "failed to list all existing Longhorn Backups during the Longhorn Backup Backing Images upgrade") + } + + for _, bbi := range backupBackingImageMap { + if bbi.Spec.BackingImage == "" { + bbi.Spec.BackingImage = bbi.Status.BackingImage + } + + if bbi.Spec.BackupTargetName == "" { + bbi.Spec.BackupTargetName = types.DefaultBackupTargetName + } + + if bbi.Labels == nil { + bbi.Labels = make(map[string]string) + } + + _, exists := bbi.Labels[types.LonghornLabelBackupTarget] + if !exists { + bbi.Labels[types.LonghornLabelBackupTarget] = bbi.Spec.BackupTargetName + } + + _, exists = bbi.Labels[types.LonghornLabelBackingImage] + if !exists { + bbi.Labels[types.LonghornLabelBackingImage] = bbi.Spec.BackingImage + } + } + + return nil +} + +func upgradeBackingImageDataSources(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) { + defer func() { + err = errors.Wrapf(err, upgradeLogPrefix+"upgrade backing image data source failed") + }() + + backingImageDataSourcesMap, err := upgradeutil.ListAndUpdateBackingImageDataSourcesInProvidedCache(namespace, lhClient, resourceMaps) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return errors.Wrapf(err, "failed to list all existing Longhorn Backups during the Longhorn Backup Backing Images upgrade") + } + + for _, bids := range backingImageDataSourcesMap { + if _, exists := bids.Spec.Parameters[longhorn.DataSourceTypeRestoreParameterBackupTargetName]; !exists { + bids.Spec.Parameters[longhorn.DataSourceTypeRestoreParameterBackupTargetName] = types.DefaultBackupTargetName + } + } + return nil +} + func upgradeBackingImages(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) { defer func() { err = errors.Wrapf(err, upgradeLogPrefix+"upgrade backing image failed") From 442d9f418af4301231d9b14443da54048d98411a Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 26 Sep 2024 17:22:49 +0800 Subject: [PATCH 29/29] feat(backup): add websocket for backup targets Ref: 5411 Signed-off-by: James Lu --- api/backup.go | 13 +++++++++++-- api/router.go | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/api/backup.go b/api/backup.go index 36a6c41971..24ff6541fb 100644 --- a/api/backup.go +++ b/api/backup.go @@ -38,14 +38,23 @@ func (s *Server) BackupTargetGet(w http.ResponseWriter, req *http.Request) error func (s *Server) BackupTargetList(w http.ResponseWriter, req *http.Request) error { apiContext := api.GetApiContext(req) - backupTargets, err := s.m.ListBackupTargetsSorted() + bts, err := s.backupTargetList(apiContext) if err != nil { return errors.Wrap(err, "failed to list backup targets") } - apiContext.Write(toBackupTargetCollection(backupTargets, apiContext)) + + apiContext.Write(bts) return nil } +func (s *Server) backupTargetList(apiContext *api.ApiContext) (*client.GenericCollection, error) { + bts, err := s.m.ListBackupTargetsSorted() + if err != nil { + return nil, errors.Wrap(err, "failed to list backup target") + } + return toBackupTargetCollection(bts, apiContext), nil +} + func (s *Server) BackupTargetCreate(rw http.ResponseWriter, req *http.Request) error { var input BackupTarget apiContext := api.GetApiContext(req) diff --git a/api/router.go b/api/router.go index e5c7953a04..75b453ceae 100644 --- a/api/router.go +++ b/api/router.go @@ -259,6 +259,10 @@ func NewRouter(s *Server) *mux.Router { r.Path("/v1/ws/backupvolumes").Handler(f(schemas, backupVolumeStream)) r.Path("/v1/ws/{period}/backupvolumes").Handler(f(schemas, backupVolumeStream)) + backupTargetStream := NewStreamHandlerFunc("backuptargets", s.wsc.NewWatcher("backupTarget"), s.backupTargetList) + r.Path("/v1/ws/backuptargets").Handler(f(schemas, backupTargetStream)) + r.Path("/v1/ws/{period}/backuptargets").Handler(f(schemas, backupTargetStream)) + // TODO: // We haven't found a way to allow passing the volume name as a parameter to filter // per-backup volume's backups change thru. WebSocket endpoint. Either by: