From 70f42e1c51e5ac953d4038be065a1d244c47718e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 29 Jun 2024 02:06:52 +0000 Subject: [PATCH] fix(deps): update github.com/longhorn/backupstore digest to e98e31e --- go.mod | 2 +- go.sum | 4 +- .../longhorn/backupstore/backupstore.go | 21 +++--- .../longhorn/backupstore/deltablock.go | 67 +++++++++++++++++-- .../longhorn/backupstore/inspect.go | 21 +++--- .../github.com/longhorn/backupstore/list.go | 21 +++--- .../longhorn/go-common-libs/backup/types.go | 10 +++ vendor/modules.txt | 3 +- 8 files changed, 112 insertions(+), 37 deletions(-) create mode 100644 vendor/github.com/longhorn/go-common-libs/backup/types.go diff --git a/go.mod b/go.mod index 78f191bd..f79e9622 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/0xPolygon/polygon-edge v1.3.3 github.com/RoaringBitmap/roaring v1.9.4 github.com/google/uuid v1.6.0 - github.com/longhorn/backupstore v0.0.0-20240616091913-84428641a11d + github.com/longhorn/backupstore v0.0.0-20240624084713-e98e31ebcebb github.com/longhorn/go-common-libs v0.0.0-20240627075631-d78642cff5e1 github.com/longhorn/go-spdk-helper v0.0.0-20240627071357-3765b24418d9 github.com/longhorn/types v0.0.0-20240624083620-f11ba48bf396 diff --git a/go.sum b/go.sum index 4a653e6d..2efc7880 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/longhorn/backupstore v0.0.0-20240616091913-84428641a11d h1:N5xR9QHU1vbhhfUij+hEOWd5OaQOTD/Jml0jC4vlVqI= -github.com/longhorn/backupstore v0.0.0-20240616091913-84428641a11d/go.mod h1:LxuU2n2baZOWFPE9modv3mYOjpFfD9AT7KPoWehCpk4= +github.com/longhorn/backupstore v0.0.0-20240624084713-e98e31ebcebb h1:equCPROzsJiPG94T4GkF78qROAIIEk107OAz3gUUJAI= +github.com/longhorn/backupstore v0.0.0-20240624084713-e98e31ebcebb/go.mod h1:8+cn7oATMLHE8x8BUZ+U/HSn1odDHVbRgnsLSB43i/8= github.com/longhorn/go-common-libs v0.0.0-20240627075631-d78642cff5e1 h1:VGSNK9AEL6r9UocxZ0LoFPv1mn/jcstEc3LDS3GedZk= github.com/longhorn/go-common-libs v0.0.0-20240627075631-d78642cff5e1/go.mod h1:wpLEAlsDCnqBA7QfZg0gxYeR8MmLbWHbdidWYwnRbyM= github.com/longhorn/go-spdk-helper v0.0.0-20240627071357-3765b24418d9 h1:IS7qr8vvZmsq/NwEKgVxh6Eviw6nuH1zhPV82mUEwkQ= diff --git a/vendor/github.com/longhorn/backupstore/backupstore.go b/vendor/github.com/longhorn/backupstore/backupstore.go index 9e79e8c8..9ff25c09 100644 --- a/vendor/github.com/longhorn/backupstore/backupstore.go +++ b/vendor/github.com/longhorn/backupstore/backupstore.go @@ -37,15 +37,18 @@ type ProcessingBlocks struct { type Backup struct { sync.Mutex - Name string - VolumeName string - SnapshotName string - SnapshotCreatedAt string - CreatedTime string - Size int64 `json:",string"` - Labels map[string]string - IsIncremental bool - CompressionMethod string + Name string + VolumeName string + SnapshotName string + SnapshotCreatedAt string + CreatedTime string + Size int64 `json:",string"` + Labels map[string]string + Parameters map[string]string + IsIncremental bool + CompressionMethod string + NewlyUploadedDataSize int64 `json:",string"` + ReUploadedDataSize int64 `json:",string"` ProcessingBlocks *ProcessingBlocks diff --git a/vendor/github.com/longhorn/backupstore/deltablock.go b/vendor/github.com/longhorn/backupstore/deltablock.go index 8f972a8f..16c5ad42 100644 --- a/vendor/github.com/longhorn/backupstore/deltablock.go +++ b/vendor/github.com/longhorn/backupstore/deltablock.go @@ -16,6 +16,7 @@ import ( . "github.com/longhorn/backupstore/logging" "github.com/longhorn/backupstore/types" "github.com/longhorn/backupstore/util" + lhbackup "github.com/longhorn/go-common-libs/backup" ) type DeltaBackupConfig struct { @@ -26,6 +27,7 @@ type DeltaBackupConfig struct { DeltaOps DeltaBlockBackupOperations Labels map[string]string ConcurrentLimit int32 + Parameters map[string]string } type DeltaRestoreConfig struct { @@ -181,7 +183,7 @@ func CreateDeltaBlockBackup(backupName string, config *DeltaBackupConfig) (isInc } backupRequest := &backupRequest{} - if volume.LastBackupName != "" { + if volume.LastBackupName != "" && !isFullBackup(config) { lastBackupName := volume.LastBackupName var backup, err = loadBackup(bsDriver, lastBackupName, volume.Name) if err != nil { @@ -401,19 +403,60 @@ func backupBlock(bsDriver BackupStoreDriver, config *DeltaBackupConfig, }() blkFile := getBlockFilePath(volume.Name, checksum) + reUpload := false if bsDriver.FileExists(blkFile) { - log.Debugf("Found existing block matching at %v", blkFile) - return nil + if !isFullBackup(config) { + log.Debugf("Found existing block matching at %v", blkFile) + return nil + } + log.Debugf("Reupload existing block matching at %v", blkFile) + reUpload = true } - log.Tracef("Creating new block file at %v", blkFile) - newBlock = true + log.Tracef("Uploading block file at %v", blkFile) + newBlock = !reUpload rs, err := util.CompressData(deltaBackup.CompressionMethod, block) if err != nil { return err } - return bsDriver.Write(blkFile, rs) + dataSize, err := getTransferDataSize(rs) + if err != nil { + return errors.Wrapf(err, "failed to get transfer data size during saving blocks") + } + + if err := bsDriver.Write(blkFile, rs); err != nil { + return errors.Wrapf(err, "failed to write data during saving blocks") + } + + updateUploadDataSize(reUpload, deltaBackup, dataSize) + + return nil +} + +func getTransferDataSize(rs io.ReadSeeker) (int64, error) { + size, err := rs.Seek(0, io.SeekEnd) + if err != nil { + return 0, err + } + + // reset to start + if _, err = rs.Seek(0, io.SeekStart); err != nil { + return 0, err + } + + return size, nil +} + +func updateUploadDataSize(reUpload bool, deltaBackup *Backup, dataSize int64) { + deltaBackup.Lock() + defer deltaBackup.Unlock() + + if reUpload { + deltaBackup.ReUploadedDataSize += dataSize + } else { + deltaBackup.NewlyUploadedDataSize += dataSize + } } func backupMapping(bsDriver BackupStoreDriver, config *DeltaBackupConfig, @@ -565,7 +608,10 @@ func performBackup(bsDriver BackupStoreDriver, config *DeltaBackupConfig, delta backup.CreatedTime = util.Now() backup.Size = int64(len(backup.Blocks)) * DEFAULT_BLOCK_SIZE backup.Labels = config.Labels + backup.Parameters = config.Parameters backup.IsIncremental = lastBackup != nil + backup.NewlyUploadedDataSize = deltaBackup.NewlyUploadedDataSize + backup.ReUploadedDataSize = deltaBackup.ReUploadedDataSize if err := saveBackup(bsDriver, backup); err != nil { return progress.progress, "", err @@ -1354,3 +1400,12 @@ func getBlockNamesForVolume(driver BackupStoreDriver, volumeName string) ([]stri return util.ExtractNames(names, "", BLK_SUFFIX), nil } + +func isFullBackup(config *DeltaBackupConfig) bool { + if config.Parameters != nil { + if backupMode, exist := config.Parameters[lhbackup.LonghornBackupParameterBackupMode]; exist { + return lhbackup.LonghornBackupMode(backupMode) == lhbackup.LonghornBackupModeFull + } + } + return false +} diff --git a/vendor/github.com/longhorn/backupstore/inspect.go b/vendor/github.com/longhorn/backupstore/inspect.go index 590709aa..55ae1f80 100644 --- a/vendor/github.com/longhorn/backupstore/inspect.go +++ b/vendor/github.com/longhorn/backupstore/inspect.go @@ -82,15 +82,18 @@ func fillVolumeInfo(volume *Volume) *VolumeInfo { func fillBackupInfo(backup *Backup, destURL string) *BackupInfo { return &BackupInfo{ - Name: backup.Name, - URL: EncodeBackupURL(backup.Name, backup.VolumeName, destURL), - SnapshotName: backup.SnapshotName, - SnapshotCreated: backup.SnapshotCreatedAt, - Created: backup.CreatedTime, - Size: backup.Size, - Labels: backup.Labels, - IsIncremental: backup.IsIncremental, - CompressionMethod: backup.CompressionMethod, + Name: backup.Name, + URL: EncodeBackupURL(backup.Name, backup.VolumeName, destURL), + SnapshotName: backup.SnapshotName, + SnapshotCreated: backup.SnapshotCreatedAt, + Created: backup.CreatedTime, + Size: backup.Size, + Labels: backup.Labels, + Parameters: backup.Parameters, + IsIncremental: backup.IsIncremental, + CompressionMethod: backup.CompressionMethod, + NewlyUploadedDataSize: backup.NewlyUploadedDataSize, + ReUploadedDataSize: backup.ReUploadedDataSize, } } diff --git a/vendor/github.com/longhorn/backupstore/list.go b/vendor/github.com/longhorn/backupstore/list.go index c61967fc..d56d9cfd 100644 --- a/vendor/github.com/longhorn/backupstore/list.go +++ b/vendor/github.com/longhorn/backupstore/list.go @@ -32,15 +32,18 @@ type VolumeInfo struct { } type BackupInfo struct { - Name string - URL string - SnapshotName string - SnapshotCreated string - Created string - Size int64 `json:",string"` - Labels map[string]string - IsIncremental bool - CompressionMethod string `json:",omitempty"` + Name string + URL string + SnapshotName string + SnapshotCreated string + Created string + Size int64 `json:",string"` + Labels map[string]string + Parameters map[string]string + IsIncremental bool + CompressionMethod string `json:",omitempty"` + NewlyUploadedDataSize int64 `json:",string"` + ReUploadedDataSize int64 `json:",string"` VolumeName string `json:",omitempty"` VolumeSize int64 `json:",string,omitempty"` diff --git a/vendor/github.com/longhorn/go-common-libs/backup/types.go b/vendor/github.com/longhorn/go-common-libs/backup/types.go new file mode 100644 index 00000000..3b77f467 --- /dev/null +++ b/vendor/github.com/longhorn/go-common-libs/backup/types.go @@ -0,0 +1,10 @@ +package backup + +type LonghornBackupMode string + +const ( + LonghornBackupParameterBackupMode = "backup-mode" + + LonghornBackupModeFull = LonghornBackupMode("full") + LonghornBackupModeIncremental = LonghornBackupMode("incremental") +) diff --git a/vendor/modules.txt b/vendor/modules.txt index fd7814e0..28de99ee 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -39,7 +39,7 @@ github.com/kr/pretty # github.com/kr/text v0.2.0 ## explicit github.com/kr/text -# github.com/longhorn/backupstore v0.0.0-20240616091913-84428641a11d +# github.com/longhorn/backupstore v0.0.0-20240624084713-e98e31ebcebb ## explicit; go 1.21 github.com/longhorn/backupstore github.com/longhorn/backupstore/logging @@ -47,6 +47,7 @@ github.com/longhorn/backupstore/types github.com/longhorn/backupstore/util # github.com/longhorn/go-common-libs v0.0.0-20240627075631-d78642cff5e1 ## explicit; go 1.22.0 +github.com/longhorn/go-common-libs/backup github.com/longhorn/go-common-libs/exec github.com/longhorn/go-common-libs/io github.com/longhorn/go-common-libs/net