From 4d99c6c2e0da1a274424284b91ec3cccdea93b9f Mon Sep 17 00:00:00 2001 From: Jack Lin Date: Tue, 19 Mar 2024 17:31:46 +0800 Subject: [PATCH] feat(backup): support recurring and ondemand full backup ref: longhorn/longhorn 7070 Signed-off-by: Jack Lin --- api/model.go | 19 ++++++- api/recurringjob.go | 2 + api/snapshot.go | 10 +++- app/recurring_job.go | 53 +++++++++++++++++-- client/generated_backup_volume.go | 2 + controller/backup_volume_controller.go | 3 ++ datastore/longhorn.go | 35 ++++++++++++ engineapi/backup_monitor.go | 2 +- engineapi/backups.go | 3 +- engineapi/enginesim.go | 2 +- engineapi/proxy_backup.go | 4 +- engineapi/types.go | 3 +- go.mod | 1 + go.sum | 8 +-- k8s/crds.yaml | 32 ++++++----- k8s/generate_code.sh | 14 ++--- k8s/pkg/apis/longhorn/v1beta2/backup.go | 9 ++++ k8s/pkg/apis/longhorn/v1beta2/backupvolume.go | 3 ++ k8s/pkg/apis/longhorn/v1beta2/recurringjob.go | 3 ++ .../longhorn/v1beta2/zz_generated.deepcopy.go | 14 +++++ manager/engine.go | 3 +- types/types.go | 4 ++ util/util.go | 24 +++++++++ .../longhorn/backupstore/backupstore.go | 7 +++ .../longhorn/backupstore/deltablock.go | 17 +++++- .../longhorn/backupstore/inspect.go | 1 + .../github.com/longhorn/backupstore/list.go | 1 + .../longhorn/backupstore/types/types.go | 16 ++++++ vendor/modules.txt | 2 +- webhook/resources/backup/validator.go | 20 +++++++ webhook/resources/recurringjob/mutator.go | 6 +++ webhook/resources/recurringjob/validator.go | 2 + 32 files changed, 282 insertions(+), 43 deletions(-) diff --git a/api/model.go b/api/model.go index 05fc9f510c..b6f449a1e0 100644 --- a/api/model.go +++ b/api/model.go @@ -140,6 +140,7 @@ type BackupVolume struct { BackingImageName string `json:"backingImageName"` BackingImageChecksum string `json:"backingImageChecksum"` StorageClassName string `json:"storageClassName"` + BackupCount string `json:"backupCount"` } type Backup struct { @@ -155,12 +156,15 @@ type Backup struct { Created string `json:"created"` Size string `json:"size"` Labels map[string]string `json:"labels"` + Parameters map[string]string `json:"parameters"` Messages map[string]string `json:"messages"` VolumeName string `json:"volumeName"` VolumeSize string `json:"volumeSize"` VolumeCreated string `json:"volumeCreated"` VolumeBackingImageName string `json:"volumeBackingImageName"` CompressionMethod string `json:"compressionMethod"` + NewlyUploadedDataSize string `json:"newlyUploadDataSize"` + ReUploadedDataSize string `json:"reUploadedDataSize"` } type BackupBackingImage struct { @@ -277,8 +281,9 @@ type DetachInput struct { } type SnapshotInput struct { - Name string `json:"name"` - Labels map[string]string `json:"labels"` + Name string `json:"name"` + Labels map[string]string `json:"labels"` + Parameters map[string]string `json:"parameters"` } type SnapshotCRInput struct { @@ -806,6 +811,11 @@ func recurringJobSchema(job *client.Schema) { labels.Type = "map[string]" labels.Nullable = true job.ResourceFields["labels"] = labels + + parameters := job.ResourceFields["parameters"] + parameters.Type = "map[string]" + parameters.Nullable = true + job.ResourceFields["parameters"] = parameters } func kubernetesStatusSchema(status *client.Schema) { @@ -1742,6 +1752,7 @@ func toBackupVolumeResource(bv *longhorn.BackupVolume, apiContext *api.ApiContex BackingImageName: bv.Status.BackingImageName, BackingImageChecksum: bv.Status.BackingImageChecksum, StorageClassName: bv.Status.StorageClassName, + BackupCount: bv.Status.BackupCount, } b.Actions = map[string]string{ "backupList": apiContext.UrlBuilder.ActionLink(b.Resource, "backupList"), @@ -1834,12 +1845,15 @@ func toBackupResource(b *longhorn.Backup) *Backup { Created: b.Status.BackupCreatedAt, Size: b.Status.Size, Labels: b.Status.Labels, + Parameters: b.Spec.Parameters, Messages: b.Status.Messages, VolumeName: b.Status.VolumeName, VolumeSize: b.Status.VolumeSize, VolumeCreated: b.Status.VolumeCreated, VolumeBackingImageName: b.Status.VolumeBackingImageName, CompressionMethod: string(b.Status.CompressionMethod), + NewlyUploadedDataSize: b.Status.NewlyUploadedDataSize, + ReUploadedDataSize: b.Status.ReUploadedDataSize, } // Set the volume name from backup CR's label if it's empty. // This field is empty probably because the backup state is not Ready @@ -2182,6 +2196,7 @@ func toRecurringJobResource(recurringJob *longhorn.RecurringJob, apiContext *api Retain: recurringJob.Spec.Retain, Concurrency: recurringJob.Spec.Concurrency, Labels: recurringJob.Spec.Labels, + Parameters: recurringJob.Spec.Parameters, }, } } diff --git a/api/recurringjob.go b/api/recurringjob.go index bb8a1f39c1..2a95f54590 100644 --- a/api/recurringjob.go +++ b/api/recurringjob.go @@ -63,6 +63,7 @@ func (s *Server) RecurringJobCreate(rw http.ResponseWriter, req *http.Request) e Retain: input.Retain, Concurrency: input.Concurrency, Labels: input.Labels, + Parameters: input.Parameters, }) if err != nil { return errors.Wrapf(err, "failed to create recurring job %v", input.Name) @@ -90,6 +91,7 @@ func (s *Server) RecurringJobUpdate(rw http.ResponseWriter, req *http.Request) e Retain: input.Retain, Concurrency: input.Concurrency, Labels: input.Labels, + Parameters: input.Parameters, }) }) if err != nil { diff --git a/api/snapshot.go b/api/snapshot.go index 05c55b23d3..e8247701f0 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -186,6 +186,14 @@ func (s *Server) SnapshotBackup(w http.ResponseWriter, req *http.Request) (err e return err } + backupParameters := map[string]string{} + if input.Parameters == nil { + backupParameters = input.Parameters + } + if err := util.ValidateBackupParameters(backupParameters); err != nil { + return err + } + // Cannot directly compare the structs since KubernetesStatus contains a slice which cannot be compared. if !reflect.DeepEqual(vol.Status.KubernetesStatus, longhorn.KubernetesStatus{}) { kubeStatus, err := json.Marshal(vol.Status.KubernetesStatus) @@ -195,7 +203,7 @@ func (s *Server) SnapshotBackup(w http.ResponseWriter, req *http.Request) (err e labels[types.KubernetesStatusLabel] = string(kubeStatus) } - if err := s.m.BackupSnapshot(bsutil.GenerateName("backup"), volName, input.Name, labels); err != nil { + if err := s.m.BackupSnapshot(bsutil.GenerateName("backup"), volName, input.Name, labels, backupParameters); err != nil { return err } diff --git a/app/recurring_job.go b/app/recurring_job.go index 6070a5598b..31e38eb537 100644 --- a/app/recurring_job.go +++ b/app/recurring_job.go @@ -15,6 +15,7 @@ import ( "github.com/urfave/cli" "golang.org/x/sync/errgroup" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" @@ -28,6 +29,7 @@ import ( "github.com/longhorn/longhorn-manager/types" "github.com/longhorn/longhorn-manager/util" + btypes "github.com/longhorn/backupstore/types" longhornclient "github.com/longhorn/longhorn-manager/client" longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" lhclientset "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned" @@ -56,6 +58,7 @@ type Job struct { retain int task longhorn.RecurringJobType labels map[string]string + parameters map[string]string eventRecorder record.EventRecorder @@ -121,6 +124,15 @@ func recurringJob(c *cli.Context) (err error) { return errors.Wrap(err, "failed to get JSON encoding for labels") } + jobParameterMap := map[string]string{} + if recurringJob.Spec.Parameters != nil { + jobParameterMap = recurringJob.Spec.Parameters + } + parameterJSON, err := json.Marshal(jobParameterMap) + if err != nil { + return errors.Wrap(err, "failed to get JSON encoding for parameters") + } + allowDetachedSetting := types.SettingNameAllowRecurringJobWhileVolumeDetached allowDetached, err := getSettingAsBoolean(allowDetachedSetting, namespace, lhClient) if err != nil { @@ -153,7 +165,7 @@ func recurringJob(c *cli.Context) (err error) { for _, volumeName := range filteredVolumes { startJobVolumeName := volumeName ewg.Go(func() error { - return startVolumeJob(startJobVolumeName, logger, concurrentLimiter, managerURL, jobName, jobTask, jobRetain, jobConcurrent, jobGroups, jobLabelMap, labelJSON) + return startVolumeJob(startJobVolumeName, logger, concurrentLimiter, managerURL, jobName, jobTask, jobRetain, jobConcurrent, jobGroups, jobLabelMap, labelJSON, jobParameterMap, parameterJSON) }) } @@ -162,7 +174,7 @@ func recurringJob(c *cli.Context) (err error) { func startVolumeJob( volumeName string, logger *logrus.Logger, concurrentLimiter chan struct{}, managerURL string, - jobName string, jobTask longhorn.RecurringJobType, jobRetain int, jobConcurrent int, jobGroups []string, jobLabelMap map[string]string, labelJSON []byte) error { + jobName string, jobTask longhorn.RecurringJobType, jobRetain int, jobConcurrent int, jobGroups []string, jobLabelMap map[string]string, labelJSON []byte, jobParameterMap map[string]string, parameterJSON []byte) error { concurrentLimiter <- struct{}{} defer func() { @@ -177,6 +189,7 @@ func startVolumeJob( "concurrent": jobConcurrent, "groups": strings.Join(jobGroups, ","), "labels": string(labelJSON), + "parameters": string(parameterJSON), }) log.Info("Creating job") @@ -187,6 +200,7 @@ func startVolumeJob( volumeName, snapshotName, jobLabelMap, + jobParameterMap, jobRetain, jobTask) if err != nil { @@ -213,7 +227,7 @@ func sliceStringSafely(s string, begin, end int) string { return s[begin:end] } -func newJob(logger logrus.FieldLogger, managerURL, volumeName, snapshotName string, labels map[string]string, retain int, task longhorn.RecurringJobType) (*Job, error) { +func newJob(logger logrus.FieldLogger, managerURL, volumeName, snapshotName string, labels map[string]string, parameters map[string]string, retain int, task longhorn.RecurringJobType) (*Job, error) { namespace := os.Getenv(types.EnvPodNamespace) if namespace == "" { return nil, fmt.Errorf("failed detect pod namespace, environment variable %v is missing", types.EnvPodNamespace) @@ -263,6 +277,7 @@ func newJob(logger logrus.FieldLogger, managerURL, volumeName, snapshotName stri volumeName: volumeName, snapshotName: snapshotName, labels: labels, + parameters: parameters, retain: retain, task: task, api: apiClient, @@ -611,9 +626,37 @@ func (job *Job) doRecurringBackup() (err error) { return err } + backupParameters := map[string]string{} + if intervalStr, exists := job.parameters[types.RecurringJobBackupParameterFullBackupInterval]; exists { + interval, err := strconv.Atoi(intervalStr) + if err != nil { + return errors.Wrapf(err, "interval %v is not number", intervalStr) + } + + backupVolume, err := job.api.BackupVolume.ById(job.volumeName) + if err != nil { + if !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "failed to get backup volume %v", job.volumeName) + } + } + + backupCount := 0 + if backupVolume != nil && backupVolume.BackupCount != "" { + backupCount, err = strconv.Atoi(backupVolume.BackupCount) + if err != nil { + return errors.Wrapf(err, "backup count %v is not number", backupVolume.BackupCount) + } + } + + if backupCount%interval == 0 { + backupParameters[btypes.LonghornBackupOptionBackupMode] = btypes.LonghornBackupModeFull + } + } + if _, err := job.api.Volume.ActionSnapshotBackup(volume, &longhornclient.SnapshotInput{ - Labels: job.labels, - Name: job.snapshotName, + Labels: job.labels, + Name: job.snapshotName, + Parameters: job.parameters, }); err != nil { return err } diff --git a/client/generated_backup_volume.go b/client/generated_backup_volume.go index b1936e1988..5e89950989 100644 --- a/client/generated_backup_volume.go +++ b/client/generated_backup_volume.go @@ -28,6 +28,8 @@ type BackupVolume struct { Size string `json:"size,omitempty" yaml:"size,omitempty"` StorageClassName string `json:"storageClassName,omitempty" yaml:"storage_class_name,omitempty"` + + BackupCount string `json:"backupCount,omitempty" yaml:"backup_count,omitempty"` } type BackupVolumeCollection struct { diff --git a/controller/backup_volume_controller.go b/controller/backup_volume_controller.go index f018d7d477..5d7cc518a9 100644 --- a/controller/backup_volume_controller.go +++ b/controller/backup_volume_controller.go @@ -383,6 +383,8 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error return nil } + logrus.Infof("[DEBUG]: backupVolumeInfo: %v", backupVolumeInfo) + // Update the Backup CR spec.syncRequestAt to request the // backup_controller to reconcile the Backup CR if the last backup changed if backupVolume.Status.LastBackupName != backupVolumeInfo.LastBackupName { @@ -408,6 +410,7 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error backupVolume.Status.BackingImageChecksum = backupVolumeInfo.BackingImageChecksum backupVolume.Status.StorageClassName = backupVolumeInfo.StorageClassName backupVolume.Status.LastSyncedAt = syncTime + backupVolume.Status.BackupCount = backupVolumeInfo.BackupCount return nil } diff --git a/datastore/longhorn.go b/datastore/longhorn.go index a337588440..2cc1464286 100644 --- a/datastore/longhorn.go +++ b/datastore/longhorn.go @@ -4139,6 +4139,41 @@ func ValidateRecurringJob(job longhorn.RecurringJobSpec) error { return err } } + if job.Parameters != nil { + if err := ValidateRecurringJobParameters(job.Task, job.Labels); err != nil { + return err + } + } + return nil +} + +func ValidateRecurringJobParameters(task longhorn.RecurringJobType, parameters map[string]string) (err error) { + switch task { + case longhorn.RecurringJobTypeBackup, longhorn.RecurringJobTypeBackupForceCreate: + for key, value := range parameters { + if err := validateRecurringJobBackupParameter(key, value); err != nil { + return errors.Wrapf(err, "failed to validate recurring job backup task parameters") + } + } + // we don't support any parameters for other tasks currently + default: + return nil + } + + return nil +} + +func validateRecurringJobBackupParameter(key, value string) error { + switch key { + case types.RecurringJobBackupParameterFullBackupInterval: + _, err := strconv.Atoi(value) + if err != nil { + return errors.Wrapf(err, "%v:%v is not number", key, value) + } + default: + return fmt.Errorf("%v:%v is not a valid parameter", key, value) + } + return nil } diff --git a/engineapi/backup_monitor.go b/engineapi/backup_monitor.go index 17ab32afc9..74602fdc35 100644 --- a/engineapi/backup_monitor.go +++ b/engineapi/backup_monitor.go @@ -84,7 +84,7 @@ func NewBackupMonitor(logger logrus.FieldLogger, ds *datastore.DataStore, backup } _, replicaAddress, err := engineClientProxy.SnapshotBackup(engine, backup.Spec.SnapshotName, backup.Name, backupTargetClient.URL, volume.Spec.BackingImage, biChecksum, string(compressionMethod), concurrentLimit, storageClassName, - backup.Spec.Labels, backupTargetClient.Credential) + backup.Spec.Labels, backupTargetClient.Credential, backup.Spec.Parameters) if err != nil { if !strings.Contains(err.Error(), "DeadlineExceeded") { m.logger.WithError(err).Warn("Cannot take snapshot backup") diff --git a/engineapi/backups.go b/engineapi/backups.go index 9bece398af..649277b792 100644 --- a/engineapi/backups.go +++ b/engineapi/backups.go @@ -232,6 +232,7 @@ func (btc *BackupTargetClient) BackupVolumeDelete(destURL, volumeName string, cr // parseBackupVolumeConfig parses a backup volume config func parseBackupVolumeConfig(output string) (*BackupVolume, error) { backupVolume := new(BackupVolume) + logrus.Infof("[DEBUG] parseBackupVolumeConfig output: %v", output) if err := json.Unmarshal([]byte(output), backupVolume); err != nil { return nil, errors.Wrapf(err, "error parsing one backup volume config: \n%s", output) } @@ -318,7 +319,7 @@ func (btc *BackupTargetClient) BackupCleanUpAllMounts() (err error) { // TODO: Deprecated, replaced by gRPC proxy func (e *EngineBinary) SnapshotBackup(engine *longhorn.Engine, snapName, backupName, backupTarget, backingImageName, backingImageChecksum, compressionMethod string, concurrentLimit int, storageClassName string, - labels, credential map[string]string) (string, string, error) { + labels, credential, parameters map[string]string) (string, string, error) { if snapName == etypes.VolumeHeadName { return "", "", fmt.Errorf("invalid operation: cannot backup %v", etypes.VolumeHeadName) } diff --git a/engineapi/enginesim.go b/engineapi/enginesim.go index bca0757d58..2763913295 100644 --- a/engineapi/enginesim.go +++ b/engineapi/enginesim.go @@ -192,7 +192,7 @@ func (e *EngineSimulator) SnapshotPurgeStatus(*longhorn.Engine) (map[string]*lon func (e *EngineSimulator) SnapshotBackup(engine *longhorn.Engine, snapshotName, backupName, backupTarget, backingImageName, backingImageChecksum, compressionMethod string, concurrentLimit int, storageClassName string, - labels, credential map[string]string) (string, string, error) { + labels, credential, parameters map[string]string) (string, string, error) { return "", "", fmt.Errorf(ErrNotImplement) } diff --git a/engineapi/proxy_backup.go b/engineapi/proxy_backup.go index 39c05f7d0b..e01b2e46a7 100644 --- a/engineapi/proxy_backup.go +++ b/engineapi/proxy_backup.go @@ -14,7 +14,7 @@ import ( func (p *Proxy) SnapshotBackup(e *longhorn.Engine, snapshotName, backupName, backupTarget, backingImageName, backingImageChecksum, compressionMethod string, concurrentLimit int, storageClassName string, - labels, credential map[string]string) (string, string, error) { + labels, credential, parameters map[string]string) (string, string, error) { if snapshotName == etypes.VolumeHeadName { return "", "", fmt.Errorf("invalid operation: cannot backup %v", etypes.VolumeHeadName) } @@ -40,7 +40,7 @@ func (p *Proxy) SnapshotBackup(e *longhorn.Engine, snapshotName, backupName, bac backupID, replicaAddress, err := p.grpcClient.SnapshotBackup(string(e.Spec.DataEngine), e.Name, e.Spec.VolumeName, p.DirectToURL(e), backupName, snapshotName, backupTarget, backingImageName, - backingImageChecksum, compressionMethod, concurrentLimit, storageClassName, labels, credentialEnv, + backingImageChecksum, compressionMethod, concurrentLimit, storageClassName, labels, credentialEnv, parameters ) if err != nil { return "", "", err diff --git a/engineapi/types.go b/engineapi/types.go index 63d6be8a4f..9122acb031 100644 --- a/engineapi/types.go +++ b/engineapi/types.go @@ -98,7 +98,7 @@ type EngineClient interface { SnapshotRevert(engine *longhorn.Engine, name string) error SnapshotPurge(engine *longhorn.Engine) error SnapshotPurgeStatus(engine *longhorn.Engine) (map[string]*longhorn.PurgeStatus, error) - SnapshotBackup(engine *longhorn.Engine, snapshotName, backupName, backupTarget, backingImageName, backingImageChecksum, compressionMethod string, concurrentLimit int, storageClassName string, labels, credential map[string]string) (string, string, error) + SnapshotBackup(engine *longhorn.Engine, snapshotName, backupName, backupTarget, backingImageName, backingImageChecksum, compressionMethod string, concurrentLimit int, storageClassName string, labels, credential, parameters map[string]string) (string, string, error) SnapshotBackupStatus(engine *longhorn.Engine, backupName, replicaAddress, replicaName string) (*longhorn.EngineBackupStatus, error) SnapshotCloneStatus(engine *longhorn.Engine) (map[string]*longhorn.SnapshotCloneStatus, error) SnapshotClone(engine *longhorn.Engine, snapshotName, fromEngineAddress, fromVolumeName, fromEngineName string, fileSyncHTTPClientTimeout int64) error @@ -162,6 +162,7 @@ type BackupVolume struct { BackingImageName string `json:"backingImageName"` BackingImageChecksum string `json:"backingImageChecksum"` StorageClassName string `json:"storageClassName"` + BackupCount string `json:"backupCount"` } type Backup struct { diff --git a/go.mod b/go.mod index b03bdf895f..11f5499eae 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/longhorn/longhorn-manager go 1.21 replace ( + github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77 => github.com/ChanYiLin/backupstore v0.0.0-20240322024320-ab6fa5ee775f k8s.io/api => k8s.io/api v0.28.5 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.5 k8s.io/apimachinery => k8s.io/apimachinery v0.28.5 diff --git a/go.sum b/go.sum index e0e7ece68e..2bcaea8f78 100644 --- a/go.sum +++ b/go.sum @@ -605,6 +605,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ChanYiLin/backupstore v0.0.0-20240322024320-ab6fa5ee775f h1:H3+X1DZVKbxMIqeFQsnVGznAjOHJSEAvFSW8ZdNjfqM= +github.com/ChanYiLin/backupstore v0.0.0-20240322024320-ab6fa5ee775f/go.mod h1:4cbJWtlrD2cGTQxQLtdlPTYopiJiusXH7CpOBrn/s3k= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= @@ -1045,14 +1047,10 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhn github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/longhorn/backing-image-manager v1.6.0 h1:Jmlc8+W63l0VZoVhPwNLniAk+eBC4CNaadoqpqA51KE= github.com/longhorn/backing-image-manager v1.6.0/go.mod h1:IH0mgbK+Dr13xkY+LhDaufyd9YIpiKqYo1AeRLFYGrk= -github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77 h1:iJRq59kA22f9HIjFtY/lz5rKCorZJrrYXju70XoWdmE= -github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77/go.mod h1:4cbJWtlrD2cGTQxQLtdlPTYopiJiusXH7CpOBrn/s3k= github.com/longhorn/go-common-libs v0.0.0-20240307063052-6e77996eda29 h1:tyzIDCMjQGQzhqAtdJaeEMAaNUZJD/sHERXp+tYc+ms= github.com/longhorn/go-common-libs v0.0.0-20240307063052-6e77996eda29/go.mod h1:ePLGb2r/PJBUIVoVhLOt4bLOeu0S72ZB+fWDWwC8H28= github.com/longhorn/go-iscsi-helper v0.0.0-20240308033847-bc3aab599425 h1:koSD52H0VkzJAh3OIZCdgQ9mqoRXklkeuhqmuwQ1WzU= github.com/longhorn/go-iscsi-helper v0.0.0-20240308033847-bc3aab599425/go.mod h1:2aM6KBix3Khd56I4rihOBOOPOm0/YvYMjtr1KNclQsI= -github.com/longhorn/go-spdk-helper v0.0.0-20240301101140-6eb6aa5fc09d h1:vajqcFlGHmyQcqhBbGMRo33GNcrKMgNY8ca87rGuKnU= -github.com/longhorn/go-spdk-helper v0.0.0-20240301101140-6eb6aa5fc09d/go.mod h1:Zv0UpwuqZpijy5/vSW2PvR8zVPBX8CJPF3XeZAuc4Ek= github.com/longhorn/go-spdk-helper v0.0.0-20240308030201-9b252d6f7250 h1:bc9BtfvSuXvmztYiBCebvPWPxakbiCWBvydbeqat+Ek= github.com/longhorn/go-spdk-helper v0.0.0-20240308030201-9b252d6f7250/go.mod h1:re2QHb6FUU9G/CL3AnXDbzFjLNPwmweBvnTfeVpkZ1E= github.com/longhorn/longhorn-engine v1.6.0 h1:6CH2vvwCgFBIGW4TegcI79CL1Ego1nvLZIC3ioRjjdM= @@ -2231,8 +2229,6 @@ k8s.io/mount-utils v0.28.5/go.mod h1:ceMAZ+Nzlk8zOwN205YXXGJRGmf1o0/XIwsKnG44p0I k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= -k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY= k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= diff --git a/k8s/crds.yaml b/k8s/crds.yaml index 64851fe1bb..7227a63004 100644 --- a/k8s/crds.yaml +++ b/k8s/crds.yaml @@ -739,6 +739,11 @@ spec: type: string description: The labels of snapshot backup. type: object + parameters: + additionalProperties: + type: string + description: The parameters of snapshot backup. + type: object snapshotName: description: The snapshot name. type: string @@ -777,12 +782,18 @@ spec: description: The error messages when calling longhorn engine on listing or inspecting backups. nullable: true type: object + newlyUploadDataSize: + description: Size of newly uploaded data + type: string ownerID: description: The node ID on which the controller is responsible to reconcile this backup CR. type: string progress: description: The snapshot backup progress. type: integer + reUploadedDataSize: + description: Size of reuploaded data + type: string replicaAddress: description: The address of the replica that runs snapshot backup. type: string @@ -831,22 +842,11 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null labels: longhorn-manager: "" name: backuptargets.longhorn.io spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: longhorn-conversion-webhook - namespace: longhorn-system - path: /v1/webhook/conversion - port: 9443 - conversionReviewVersions: - - v1beta2 - - v1beta1 group: longhorn.io names: kind: BackupTarget @@ -1111,6 +1111,9 @@ spec: backingImageName: description: The backing image name. type: string + backupCount: + description: The number of backups that have been created. + type: string createdAt: description: The backup volume creation time. type: string @@ -2476,6 +2479,11 @@ spec: name: description: The recurring job name. type: string + parameters: + additionalProperties: + type: string + description: The parameters of the snapshot/backup. + type: object retain: description: The retain count of the snapshot/backup. type: integer diff --git a/k8s/generate_code.sh b/k8s/generate_code.sh index 7d8ef7970b..5374b12b2b 100755 --- a/k8s/generate_code.sh +++ b/k8s/generate_code.sh @@ -37,14 +37,14 @@ if [[ ! -d "${GOPATH}/src/k8s.io/code-generator" ]]; then fi # https://github.com/kubernetes-sigs/controller-tools/tree/v0.7.0/cmd/controller-gen -if ! command -v controller-gen > /dev/null; then +if ! command -v ${GOPATH}/bin/controller-gen > /dev/null; then echo "controller-gen is missing" echo "Prepare to install controller-gen" go install sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_TOOLS_VERSION} fi # https://github.com/kubernetes-sigs/kustomize/tree/kustomize/v3.10.0/kustomize -if ! command -v kustomize > /dev/null; then +if ! command -v ${GOPATH}/bin/kustomize > /dev/null; then echo "kustomize is missing" echo "Prepare to install kustomize" mkdir -p ${GOPATH}/src/github.com/kubernetes-sigs @@ -65,16 +65,16 @@ bash ${GOPATH}/src/k8s.io/code-generator/generate-groups.sh \ $@ echo Generating CRD -controller-gen crd paths=${APIS_DIR}/... output:crd:dir=${CRDS_DIR} +${GOPATH}/bin/controller-gen crd paths=${APIS_DIR}/... output:crd:dir=${CRDS_DIR} pushd ${CRDS_DIR} -kustomize create --autodetect 2>/dev/null || true -kustomize edit add label longhorn-manager: 2>/dev/null || true +${GOPATH}/bin/kustomize create --autodetect 2>/dev/null || true +${GOPATH}/bin/kustomize edit add label longhorn-manager: 2>/dev/null || true if [ -e ${GOPATH}/src/${LH_MANAGER_DIR}/k8s/patches/crd ]; then cp -a ${GOPATH}/src/${LH_MANAGER_DIR}/k8s/patches/crd patches - find patches -type f | xargs -i sh -c 'kustomize edit add patch --path {}' + find patches -type f | xargs -i sh -c '${GOPATH}/bin/kustomize edit add patch --path {}' fi popd echo "# Generated by the CRDs from ${APIS_DIR}" > ${GOPATH}/src/${LH_MANAGER_DIR}/k8s/crds.yaml -kustomize build ${CRDS_DIR} >> ${GOPATH}/src/${LH_MANAGER_DIR}/k8s/crds.yaml +${GOPATH}/bin/kustomize build ${CRDS_DIR} >> ${GOPATH}/src/${LH_MANAGER_DIR}/k8s/crds.yaml rm -r ${CRDS_DIR} diff --git a/k8s/pkg/apis/longhorn/v1beta2/backup.go b/k8s/pkg/apis/longhorn/v1beta2/backup.go index 45dd5dc59a..5ffdfd7a07 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/backup.go +++ b/k8s/pkg/apis/longhorn/v1beta2/backup.go @@ -35,6 +35,9 @@ type BackupSpec struct { // The labels of snapshot backup. // +optional Labels map[string]string `json:"labels"` + // The parameters of snapshot backup. + // +optional + Parameters map[string]string `json:"parameters"` } // BackupStatus defines the observed state of the Longhorn backup @@ -97,6 +100,12 @@ type BackupStatus struct { // Compression method // +optional CompressionMethod BackupCompressionMethod `json:"compressionMethod"` + // Size of newly uploaded data + // +optional + NewlyUploadedDataSize string `json:"newlyUploadDataSize"` + // Size of reuploaded data + // +optional + ReUploadedDataSize string `json:"reUploadedDataSize"` } // +genclient diff --git a/k8s/pkg/apis/longhorn/v1beta2/backupvolume.go b/k8s/pkg/apis/longhorn/v1beta2/backupvolume.go index 47aeeb3a26..99f7492c94 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/backupvolume.go +++ b/k8s/pkg/apis/longhorn/v1beta2/backupvolume.go @@ -55,6 +55,9 @@ type BackupVolumeStatus struct { // +optional // +nullable LastSyncedAt metav1.Time `json:"lastSyncedAt"` + // The number of backups that have been created. + // +optional + BackupCount string `json:"backupCount"` } // +genclient diff --git a/k8s/pkg/apis/longhorn/v1beta2/recurringjob.go b/k8s/pkg/apis/longhorn/v1beta2/recurringjob.go index 4fd1f07ea3..35985018b2 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/recurringjob.go +++ b/k8s/pkg/apis/longhorn/v1beta2/recurringjob.go @@ -53,6 +53,9 @@ type RecurringJobSpec struct { // The label of the snapshot/backup. // +optional Labels map[string]string `json:"labels,omitempty"` + // The parameters of the snapshot/backup. + // +optional + Parameters map[string]string `json:"parameters,omitempty"` } // RecurringJobStatus defines the observed state of the Longhorn recurring job diff --git a/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go b/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go index 48cd18dada..5690244b5d 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go +++ b/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go @@ -632,6 +632,13 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { (*out)[key] = val } } + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } @@ -1894,6 +1901,13 @@ func (in *RecurringJobSpec) DeepCopyInto(out *RecurringJobSpec) { (*out)[key] = val } } + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } diff --git a/manager/engine.go b/manager/engine.go index 7f6e023a1a..eac74a8911 100644 --- a/manager/engine.go +++ b/manager/engine.go @@ -244,7 +244,7 @@ func (m *VolumeManager) PurgeSnapshot(volumeName string) error { return nil } -func (m *VolumeManager) BackupSnapshot(backupName, volumeName, snapshotName string, labels map[string]string) error { +func (m *VolumeManager) BackupSnapshot(backupName, volumeName, snapshotName string, labels map[string]string, parameters map[string]string) error { if volumeName == "" || snapshotName == "" { return fmt.Errorf("volume and snapshot name required") } @@ -260,6 +260,7 @@ func (m *VolumeManager) BackupSnapshot(backupName, volumeName, snapshotName stri Spec: longhorn.BackupSpec{ SnapshotName: snapshotName, Labels: labels, + Parameters: parameters, }, } _, err := m.ds.CreateBackup(backupCR, volumeName) diff --git a/types/types.go b/types/types.go index e2d62b2768..cb232c1e2d 100644 --- a/types/types.go +++ b/types/types.go @@ -207,6 +207,10 @@ const ( StorageNetworkInterface = "lhnet1" ) +const ( + RecurringJobBackupParameterFullBackupInterval = "full-backup-interval" +) + const ( KubernetesMinVersion = "v1.18.0" ) diff --git a/util/util.go b/util/util.go index e6612e242c..0ff15f41c7 100644 --- a/util/util.go +++ b/util/util.go @@ -47,6 +47,7 @@ import ( lhns "github.com/longhorn/go-common-libs/ns" lhtypes "github.com/longhorn/go-common-libs/types" + btypes "github.com/longhorn/backupstore/types" longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ) @@ -502,6 +503,29 @@ func ValidateSnapshotLabels(labels map[string]string) (map[string]string, error) return validLabels, nil } +func ValidateBackupParameters(parameters map[string]string) error { + for key, value := range parameters { + if err := validateBackupParameter(key, value); err != nil { + return err + } + } + return nil +} + +func validateBackupParameter(key, value string) error { + switch key { + case btypes.LonghornBackupOptionBackupMode: + if value != btypes.LonghornBackupModeFull && + value != btypes.LonghornBackupModeIncremental { + return fmt.Errorf("%v:%v is not a valid option", key, value) + } + default: + return fmt.Errorf("%v:%v is not a valid option", key, value) + } + + return nil +} + func ValidateTags(inputTags []string) ([]string, error) { foundTags := make(map[string]struct{}) var tags []string diff --git a/vendor/github.com/longhorn/backupstore/backupstore.go b/vendor/github.com/longhorn/backupstore/backupstore.go index 9e79e8c8e8..fff2d69cb6 100644 --- a/vendor/github.com/longhorn/backupstore/backupstore.go +++ b/vendor/github.com/longhorn/backupstore/backupstore.go @@ -23,6 +23,10 @@ type Volume struct { CompressionMethod string `json:",string"` StorageClassName string `json:",string"` DataEngine string `json:",string"` + + // It is not the count of the current backup on the backupstore, + // but how many backups of the volume have been created before. + BackupCount int64 `json:",string"` } type Snapshot struct { @@ -74,6 +78,9 @@ func addVolume(driver BackupStoreDriver, volume *Volume) error { return fmt.Errorf("invalid volume name %v", volume.Name) } + // first time create backup volume means first time create backup on this backupstore + volume.BackupCount = 0 + if err := saveVolume(driver, volume); err != nil { log.WithError(err).Errorf("Failed to add volume %v", volume.Name) return err diff --git a/vendor/github.com/longhorn/backupstore/deltablock.go b/vendor/github.com/longhorn/backupstore/deltablock.go index 73851e53b5..00a22cdc13 100644 --- a/vendor/github.com/longhorn/backupstore/deltablock.go +++ b/vendor/github.com/longhorn/backupstore/deltablock.go @@ -175,7 +175,7 @@ func CreateDeltaBlockBackup(backupName string, config *DeltaBackupConfig) (isInc } backupRequest := &backupRequest{} - if volume.LastBackupName != "" { + if volume.LastBackupName != "" && !isFullBackup(config) { lastBackupName := volume.LastBackupName var backup, err = loadBackup(bsDriver, lastBackupName, volume.Name) if err != nil { @@ -208,6 +208,9 @@ func CreateDeltaBlockBackup(backupName string, config *DeltaBackupConfig) (isInc } } + logrus.Infof("[DEBUG] isFullBackup(config): %v", isFullBackup(config)) + logrus.Infof("[DEBUG] backupRequest.lastBackup: %v", backupRequest.lastBackup) + log.WithFields(logrus.Fields{ LogFieldReason: LogReasonStart, LogFieldObject: LogObjectSnapshot, @@ -368,7 +371,7 @@ func backupBlock(bsDriver BackupStoreDriver, config *DeltaBackupConfig, }() blkFile := getBlockFilePath(volume.Name, checksum) - if bsDriver.FileExists(blkFile) { + if bsDriver.FileExists(blkFile) && !isFullBackup(config) { log.Debugf("Found existing block matching at %v", blkFile) return nil } @@ -554,6 +557,7 @@ func performBackup(bsDriver BackupStoreDriver, config *DeltaBackupConfig, delta volume.CompressionMethod = config.Volume.CompressionMethod volume.StorageClassName = config.Volume.StorageClassName volume.DataEngine = config.Volume.DataEngine + volume.BackupCount = volume.BackupCount + 1 if err := saveVolume(bsDriver, volume); err != nil { return progress.progress, "", err @@ -1303,3 +1307,12 @@ func getBlockNamesForVolume(driver BackupStoreDriver, volumeName string) ([]stri return util.ExtractNames(names, "", BLK_SUFFIX), nil } + +func isFullBackup(config *DeltaBackupConfig) bool { + if config.Labels != nil { + if backupMode, exist := config.Labels[types.GetLonghornLabelKey(types.LonghornBackupOptionBackupMode)]; exist { + return backupMode == types.LonghornBackupModeFull + } + } + return false +} diff --git a/vendor/github.com/longhorn/backupstore/inspect.go b/vendor/github.com/longhorn/backupstore/inspect.go index 590709aa54..2b221cfdd1 100644 --- a/vendor/github.com/longhorn/backupstore/inspect.go +++ b/vendor/github.com/longhorn/backupstore/inspect.go @@ -77,6 +77,7 @@ func fillVolumeInfo(volume *Volume) *VolumeInfo { BackingImageChecksum: volume.BackingImageChecksum, StorageClassname: volume.StorageClassName, DataEngine: volume.DataEngine, + BackupCount: volume.BackupCount, } } diff --git a/vendor/github.com/longhorn/backupstore/list.go b/vendor/github.com/longhorn/backupstore/list.go index c61967fc18..1fe9e55d38 100644 --- a/vendor/github.com/longhorn/backupstore/list.go +++ b/vendor/github.com/longhorn/backupstore/list.go @@ -29,6 +29,7 @@ type VolumeInfo struct { BackingImageChecksum string StorageClassname string DataEngine string + BackupCount int64 `json:",string"` } type BackupInfo struct { diff --git a/vendor/github.com/longhorn/backupstore/types/types.go b/vendor/github.com/longhorn/backupstore/types/types.go index bc626cc5f4..b71880a020 100644 --- a/vendor/github.com/longhorn/backupstore/types/types.go +++ b/vendor/github.com/longhorn/backupstore/types/types.go @@ -1,5 +1,7 @@ package types +import "fmt" + type ProgressState string const ( @@ -29,6 +31,16 @@ const ( NOProxy = "NO_PROXY" VirtualHostedStyle = "VIRTUAL_HOSTED_STYLE" + + LonghornLabelKeyPrefix = "longhorn.io" +) + +const ( + LonghornBackupOptionBackupMode = "backup-mode" + LonghornBackupModeFull = "full" + LonghornBackupModeIncremental = "incremental" + + LonghornBackupOptionFullBackupInterval = "full-backup-interval" ) type Mapping struct { @@ -65,3 +77,7 @@ const ( const ( ErrorMsgRestoreCancelled = "backup restoration is cancelled" ) + +func GetLonghornLabelKey(name string) string { + return fmt.Sprintf("%s/%s", LonghornLabelKeyPrefix, name) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 4a33c35218..e8fc1b6492 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -232,7 +232,7 @@ github.com/longhorn/backing-image-manager/pkg/meta github.com/longhorn/backing-image-manager/pkg/rpc github.com/longhorn/backing-image-manager/pkg/types github.com/longhorn/backing-image-manager/pkg/util -# github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77 +# github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77 => github.com/ChanYiLin/backupstore v0.0.0-20240322024320-ab6fa5ee775f ## explicit; go 1.21 github.com/longhorn/backupstore github.com/longhorn/backupstore/backupbackingimage diff --git a/webhook/resources/backup/validator.go b/webhook/resources/backup/validator.go index 03697dee38..7f0cd6ae58 100644 --- a/webhook/resources/backup/validator.go +++ b/webhook/resources/backup/validator.go @@ -1,10 +1,15 @@ package backup import ( + "fmt" + admissionregv1 "k8s.io/api/admissionregistration/v1" + "k8s.io/apimachinery/pkg/runtime" "github.com/longhorn/longhorn-manager/datastore" + "github.com/longhorn/longhorn-manager/util" "github.com/longhorn/longhorn-manager/webhook/admission" + werror "github.com/longhorn/longhorn-manager/webhook/error" longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ) @@ -28,3 +33,18 @@ func (b *backupValidator) Resource() admission.Resource { OperationTypes: []admissionregv1.OperationType{}, } } + +func (b *backupValidator) Create(request *admission.Request, newObj runtime.Object) error { + backup, ok := newObj.(*longhorn.Backup) + if !ok { + return werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.Backup", newObj), "") + } + + if backup.Spec.Parameters != nil { + if err := util.ValidateBackupParameters(backup.Spec.Parameters); err != nil { + return werror.NewInvalidError(err.Error(), "") + } + } + + return nil +} diff --git a/webhook/resources/recurringjob/mutator.go b/webhook/resources/recurringjob/mutator.go index 8d6538a2e2..e8b1ea2da7 100644 --- a/webhook/resources/recurringjob/mutator.go +++ b/webhook/resources/recurringjob/mutator.go @@ -58,6 +58,9 @@ func (r *recurringJobMutator) Create(request *admission.Request, newObj runtime. if recurringjob.Spec.Labels == nil { patchOps = append(patchOps, `{"op": "replace", "path": "/spec/labels", "value": {}}`) } + if recurringjob.Spec.Parameters == nil { + patchOps = append(patchOps, `{"op": "replace", "path": "/spec/parameters", "value": {}}`) + } log := logrus.WithFields(logrus.Fields{ "recurringJob": recurringjob.Name, @@ -100,6 +103,9 @@ func (r *recurringJobMutator) Update(request *admission.Request, oldObj runtime. if newRecurringjob.Spec.Labels == nil { patchOps = append(patchOps, `{"op": "replace", "path": "/spec/labels", "value": {}}`) } + if newRecurringjob.Spec.Parameters == nil { + patchOps = append(patchOps, `{"op": "replace", "path": "/spec/parameters", "value": {}}`) + } log := logrus.WithFields(logrus.Fields{ "recurringJob": newRecurringjob.Name, diff --git a/webhook/resources/recurringjob/validator.go b/webhook/resources/recurringjob/validator.go index c322e01aa4..97d02d6e6e 100644 --- a/webhook/resources/recurringjob/validator.go +++ b/webhook/resources/recurringjob/validator.go @@ -71,6 +71,7 @@ func (r *recurringJobValidator) Create(request *admission.Request, newObj runtim Retain: recurringJob.Spec.Retain, Concurrency: recurringJob.Spec.Concurrency, Labels: recurringJob.Spec.Labels, + Parameters: recurringJob.Spec.Parameters, }, } if err := r.ds.ValidateRecurringJobs(jobs); err != nil { @@ -105,6 +106,7 @@ func (r *recurringJobValidator) Update(request *admission.Request, oldObj runtim Retain: newRecurringJob.Spec.Retain, Concurrency: newRecurringJob.Spec.Concurrency, Labels: newRecurringJob.Spec.Labels, + Parameters: newRecurringJob.Spec.Parameters, }, } if err := r.ds.ValidateRecurringJobs(jobs); err != nil {