From 746f11498bf8546188544fe4c0d06a30d1eb97cb Mon Sep 17 00:00:00 2001 From: Chin-Ya Huang Date: Wed, 31 Jul 2024 11:50:44 +0800 Subject: [PATCH] chore: update vendor longhorn/longhorn-6613 Signed-off-by: Chin-Ya Huang --- go.mod | 4 ++ go.sum | 8 +-- .../longhorn/backupstore/deltablock.go | 41 ++++++----- .../longhorn-spdk-engine/pkg/spdk/engine.go | 69 ++++++++++++++++++- .../longhorn-spdk-engine/pkg/spdk/replica.go | 60 +++++++++++++--- .../longhorn-spdk-engine/pkg/spdk/restore.go | 4 +- vendor/modules.txt | 6 +- 7 files changed, 155 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index 23ab0b3eb..0f8c4a12c 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,10 @@ module github.com/longhorn/longhorn-instance-manager go 1.22.2 +replace github.com/longhorn/longhorn-spdk-engine => github.com/c3y1huang/longhorn-spdk-engine v0.0.0-20240820075856-da4b96680f2e + +replace github.com/longhorn/backupstore => github.com/c3y1huang/backupstore v0.0.0-20240820074828-9c490fd434d0 + require ( github.com/RoaringBitmap/roaring v1.9.4 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 18c1587ec..77490574a 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/c3y1huang/backupstore v0.0.0-20240820074828-9c490fd434d0 h1:FG4Y2qQKaKUgcnW51Xbc1LO7nZGh56PqqgS/X9wkWWY= +github.com/c3y1huang/backupstore v0.0.0-20240820074828-9c490fd434d0/go.mod h1:U02ktOpAMSejR4vUztX2zt/78btYsUICDH5AipTddu4= +github.com/c3y1huang/longhorn-spdk-engine v0.0.0-20240820075856-da4b96680f2e h1:/3O1SiKlw0loHe+bXPRsfPeAL6T5B79ZSFOAeQkTYZk= +github.com/c3y1huang/longhorn-spdk-engine v0.0.0-20240820075856-da4b96680f2e/go.mod h1:Duhv9B9QT/pSIPgCTifsRSDmzNHewFiYdHlNlIZlyK8= github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 h1:SjZ2GvvOononHOpK84APFuMvxqsk3tEIaKH/z4Rpu3g= github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8/go.mod h1:uEyr4WpAH4hio6LFriaPkL938XnrvLpNPmQHBdrmbIE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -96,8 +100,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/longhorn/backupstore v0.0.0-20240811043357-5c2b7879457f h1:VBXKJ+BJoqhz+PGmHxDl6h+aWX9X9N3KQm1cxz4fv/w= -github.com/longhorn/backupstore v0.0.0-20240811043357-5c2b7879457f/go.mod h1:U02ktOpAMSejR4vUztX2zt/78btYsUICDH5AipTddu4= github.com/longhorn/go-common-libs v0.0.0-20240811024046-b6ddc3efb72e h1:cmns0zh6HMocqH7tnjFrgI8mf/h1i0bbnLznWfjQ63k= github.com/longhorn/go-common-libs v0.0.0-20240811024046-b6ddc3efb72e/go.mod h1:Yrq49KPpchy++7Z15GrMKPkiwr5IduPboLTL99a1TGA= github.com/longhorn/go-iscsi-helper v0.0.0-20240727052644-d2442a50e906 h1:1FoEoyIQRtKAKv3O8MJ1nczly3+WMzbFbjT3pmakpWw= @@ -106,8 +108,6 @@ github.com/longhorn/go-spdk-helper v0.0.0-20240811121608-9383fa59dd7c h1:ztjrkxi github.com/longhorn/go-spdk-helper v0.0.0-20240811121608-9383fa59dd7c/go.mod h1:BrtXoVnIZ97+uZ+cMcaICc5KnuJkh8j3+G1NjKxh+8Q= github.com/longhorn/longhorn-engine v1.7.0-rc3 h1:YTt++OeSrEOlifz++8VAOH/aJ4lGShD2TaJP1ZaQ3Uw= github.com/longhorn/longhorn-engine v1.7.0-rc3/go.mod h1:2Hq/3QzW4fF2yUg+kauiAT3ps5WCKLMkrwXW2Wyfj9o= -github.com/longhorn/longhorn-spdk-engine v0.0.0-20240815033436-41791a2d0239 h1:1bHRQ6MYz4z3ul614ZGFjcT9EdeIUtUYKBi/Qjg5dHc= -github.com/longhorn/longhorn-spdk-engine v0.0.0-20240815033436-41791a2d0239/go.mod h1:4X7OQr01G10WSxq0ETEFxS/gQvgcE3YUEIU3MZujTcc= github.com/longhorn/nsfilelock v0.0.0-20200723175406-fa7c83ad0003 h1:Jw9uANsGcHTxp6HcC++/vN17LfeuDmozHI2j6DoZf5E= github.com/longhorn/nsfilelock v0.0.0-20200723175406-fa7c83ad0003/go.mod h1:0CLeXlf59Lg6C0kjLSDf47ft73Dh37CwymYRKWwAn04= github.com/longhorn/sparse-tools v0.0.0-20240703010727-92451e38077a h1:+o63c0oh7ZNKeQdc0Hawfzz5vRa4LiDvLOtJYjegtnk= diff --git a/vendor/github.com/longhorn/backupstore/deltablock.go b/vendor/github.com/longhorn/backupstore/deltablock.go index 16c5ad42c..f08392c98 100644 --- a/vendor/github.com/longhorn/backupstore/deltablock.go +++ b/vendor/github.com/longhorn/backupstore/deltablock.go @@ -110,7 +110,7 @@ type DeltaBlockBackupOperations interface { } type DeltaRestoreOperations interface { - OpenVolumeDev(volDevName string) (*os.File, string, error) + OpenVolumeDev(volDevName string, dmDeviceAndEndpointCleanupRequired bool) (*os.File, string, error) CloseVolumeDev(volDev *os.File) error UpdateRestoreStatus(snapshot string, restoreProgress int, err error) Stop() @@ -735,7 +735,7 @@ func RestoreDeltaBlockBackup(ctx context.Context, config *DeltaRestoreConfig) er return fmt.Errorf("invalid volume size %v", vol.Size) } - volDev, volDevPath, err := deltaOps.OpenVolumeDev(volDevName) + volDev, volDevPath, err := deltaOps.OpenVolumeDev(volDevName, false) if err != nil { return errors.Wrapf(err, "failed to open volume device %v", volDevName) } @@ -843,8 +843,9 @@ func RestoreDeltaBlockBackupIncrementally(ctx context.Context, config *DeltaRest lastBackupName := config.LastBackupName deltaOps := config.DeltaOps if deltaOps == nil { - return fmt.Errorf("missing DeltaBlockBackupOperations") + return fmt.Errorf("missing DeltaRestoreOperations") } + bsDriver, err := GetBackupStoreDriver(backupURL) if err != nil { return err @@ -894,14 +895,14 @@ func RestoreDeltaBlockBackupIncrementally(ctx context.Context, config *DeltaRest } } - volDev, err := os.Create(volDevName) + volDev, volDevPath, err := deltaOps.OpenVolumeDev(volDevName, true) if err != nil { - return err + return errors.Wrapf(err, "failed to open volume device %v", volDevName) } defer func() { // make sure to close the device if err != nil { - _ = volDev.Close() + _ = deltaOps.CloseVolumeDev(volDev) } }() @@ -933,10 +934,16 @@ func RestoreDeltaBlockBackupIncrementally(ctx context.Context, config *DeltaRest return err } go func() { - defer volDev.Close() + var err error + finalProgress := 0 + defer func() { + _ = deltaOps.CloseVolumeDev(volDev) + + deltaOps.UpdateRestoreStatus(volDevName, finalProgress, err) + if unlockErr := lock.Unlock(); unlockErr != nil { - logrus.WithError(err).Warn("Failed to unlock") + logrus.WithError(unlockErr).Warn("Failed to unlock") } }() @@ -945,20 +952,20 @@ func RestoreDeltaBlockBackupIncrementally(ctx context.Context, config *DeltaRest // closed. // https://github.com/longhorn/longhorn/issues/2503 // We want to truncate regular files, but not device - if stat.Mode()&os.ModeType == 0 { - log.Debugf("Truncate %v to size %v", volDevName, vol.Size) - if err := volDev.Truncate(vol.Size); err != nil { - deltaOps.UpdateRestoreStatus(volDevName, 0, err) + if stat.Mode().IsRegular() { + log.Infof("Truncate %v to size %v", volDevName, vol.Size) + err = volDev.Truncate(vol.Size) + if err != nil { return } } - if err := performIncrementalRestore(ctx, bsDriver, config, srcVolumeName, volDevName, lastBackup, backup); err != nil { - deltaOps.UpdateRestoreStatus(volDevName, 0, err) + err = performIncrementalRestore(ctx, bsDriver, config, srcVolumeName, volDevPath, lastBackup, backup) + if err != nil { return } - deltaOps.UpdateRestoreStatus(volDevName, PROGRESS_PERCENTAGE_BACKUP_TOTAL, nil) + finalProgress = PROGRESS_PERCENTAGE_BACKUP_TOTAL }() return nil } @@ -1107,7 +1114,7 @@ func restoreBlocks(ctx context.Context, bsDriver BackupStoreDriver, deltaOps Del } func performIncrementalRestore(ctx context.Context, bsDriver BackupStoreDriver, config *DeltaRestoreConfig, - srcVolumeName, volDevName string, lastBackup *Backup, backup *Backup) error { + srcVolumeName, volDevPath string, lastBackup *Backup, backup *Backup) error { var err error concurrentLimit := config.ConcurrentLimit @@ -1119,7 +1126,7 @@ func performIncrementalRestore(ctx context.Context, bsDriver BackupStoreDriver, errorChans := []<-chan error{errChan} for i := 0; i < int(concurrentLimit); i++ { - errorChans = append(errorChans, restoreBlocks(ctx, bsDriver, config.DeltaOps, config.Filename, srcVolumeName, blockChan, progress)) + errorChans = append(errorChans, restoreBlocks(ctx, bsDriver, config.DeltaOps, volDevPath, srcVolumeName, blockChan, progress)) } mergedErrChan := mergeErrorChannels(ctx, errorChans...) diff --git a/vendor/github.com/longhorn/longhorn-spdk-engine/pkg/spdk/engine.go b/vendor/github.com/longhorn/longhorn-spdk-engine/pkg/spdk/engine.go index 41dec0f71..2834454dd 100644 --- a/vendor/github.com/longhorn/longhorn-spdk-engine/pkg/spdk/engine.go +++ b/vendor/github.com/longhorn/longhorn-spdk-engine/pkg/spdk/engine.go @@ -1753,9 +1753,20 @@ func (e *Engine) BackupRestore(spdkClient *spdkclient.Client, backupUrl, engineN e.log.Infof("Generating a snapshot name %s for the full restore", snapshotName) } } else { - return nil, errors.Errorf("incremental restore is not supported yet") + if snapshotName == "" { + snapshotName = util.UUID() + e.log.Infof("Generating a snapshot name %s for the incremental restore", snapshotName) + } } + defer func() { + go func() { + if err := e.completeBackupRestore(spdkClient); err != nil { + logrus.WithError(err).Warn("Failed to complete backup restore") + } + }() + }() + resp := &spdkrpc.EngineBackupRestoreResponse{ Errors: map[string]string{}, } @@ -1793,6 +1804,62 @@ func (e *Engine) BackupRestore(spdkClient *spdkclient.Client, backupUrl, engineN return resp, nil } +func (e *Engine) completeBackupRestore(spdkClient *spdkclient.Client) error { + if err := e.waitForRestoreComplete(); err != nil { + return errors.Wrapf(err, "failed to wait for restore complete") + } + + return e.BackupRestoreFinish(spdkClient) +} + +func (e *Engine) waitForRestoreComplete() error { + periodicChecker := time.NewTicker(time.Duration(restorePeriodicRefreshInterval.Seconds()) * time.Second) + defer periodicChecker.Stop() + + var err error + for range periodicChecker.C { + isReplicaRestoreCompleted := true + for replicaName, replicaAddress := range e.ReplicaAddressMap { + isReplicaRestoreCompleted, err = e.isReplicaRestoreCompleted(replicaName, replicaAddress) + if err != nil { + return errors.Wrapf(err, "failed to check replica %s restore status", replicaName) + } + + if !isReplicaRestoreCompleted { + break + } + } + + if isReplicaRestoreCompleted { + e.log.Info("Backup restoration completed successfully") + return nil + } + } + + return errors.Errorf("failed to wait for engine %s restore complete", e.Name) +} + +func (e *Engine) isReplicaRestoreCompleted(replicaName, replicaAddress string) (bool, error) { + log := e.log.WithFields(logrus.Fields{ + "replica": replicaName, + "address": replicaAddress, + }) + log.Trace("Checking replica restore status") + + replicaServiceCli, err := GetServiceClient(replicaAddress) + if err != nil { + return false, errors.Wrapf(err, "failed to get replica %v service client %s", replicaName, replicaAddress) + } + defer replicaServiceCli.Close() + + status, err := replicaServiceCli.ReplicaRestoreStatus(replicaName) + if err != nil { + return false, errors.Wrapf(err, "failed to check replica %s restore status", replicaName) + } + + return !status.IsRestoring, nil +} + func (e *Engine) BackupRestoreFinish(spdkClient *spdkclient.Client) error { e.Lock() defer e.Unlock() diff --git a/vendor/github.com/longhorn/longhorn-spdk-engine/pkg/spdk/replica.go b/vendor/github.com/longhorn/longhorn-spdk-engine/pkg/spdk/replica.go index 9ca9ef4de..5c7a9e99e 100644 --- a/vendor/github.com/longhorn/longhorn-spdk-engine/pkg/spdk/replica.go +++ b/vendor/github.com/longhorn/longhorn-spdk-engine/pkg/spdk/replica.go @@ -1925,14 +1925,18 @@ func (r *Replica) BackupRestore(spdkClient *spdkclient.Client, backupUrl, snapsh return grpcstatus.Errorf(grpccodes.Internal, err.Error()) } } else { + r.log.Info("Resetting the restore for backup %v", backupUrl) + var lvolName string var snapshotNameToBeRestored string validLastRestoredBackup := r.canDoIncrementalRestore(restore, backupUrl, backupName) if validLastRestoredBackup { + r.log.Infof("Starting an incremental restore for backup %v", backupUrl) lvolName = GetReplicaSnapshotLvolName(r.Name, restore.LastRestored) snapshotNameToBeRestored = restore.LastRestored } else { + r.log.Infof("Starting a full restore for backup %v", backupUrl) lvolName = GetReplicaSnapshotLvolName(r.Name, snapshotName) snapshotNameToBeRestored = snapshotName } @@ -1948,26 +1952,53 @@ func (r *Replica) BackupRestore(spdkClient *spdkclient.Client, backupUrl, snapsh } }() - if newRestore.LastRestored == "" { + isFullRestore := newRestore.LastRestored == "" + + defer func() { + go func() { + if err := r.completeBackupRestore(spdkClient, isFullRestore); err != nil { + logrus.WithError(err).Warn("Failed to complete backup restore") + } + }() + }() + + if isFullRestore { r.log.Infof("Starting a new full restore for backup %v", backupUrl) if err := r.backupRestore(backupUrl, newRestore.LvolName, concurrentLimit); err != nil { return errors.Wrapf(err, "failed to start full backup restore") } r.log.Infof("Successfully initiated full restore for %v to %v", backupUrl, newRestore.LvolName) } else { - return fmt.Errorf("incremental restore is not supported yet") - } - - go func() { - if err := r.completeBackupRestore(spdkClient); err != nil { - logrus.WithError(err).Warn("Failed to complete backup restore") + r.log.Infof("Starting an incremental restore for backup %v", backupUrl) + if err := r.backupRestoreIncrementally(backupUrl, newRestore.LastRestored, newRestore.LvolName, concurrentLimit); err != nil { + return errors.Wrapf(err, "failed to start incremental backup restore") } - }() + r.log.Infof("Successfully initiated incremental restore for %v to %v", backupUrl, newRestore.LvolName) + } return nil } +func (r *Replica) backupRestoreIncrementally(backupURL, lastRestored, snapshotLvolName string, concurrentLimit int32) error { + backupURL = butil.UnescapeURL(backupURL) + + logrus.WithFields(logrus.Fields{ + "backupURL": backupURL, + "lastRestored": lastRestored, + "snapshotLvolName": snapshotLvolName, + "concurrentLimit": concurrentLimit, + }).Info("Start restoring backup incrementally") + + return backupstore.RestoreDeltaBlockBackupIncrementally(r.ctx, &backupstore.DeltaRestoreConfig{ + BackupURL: backupURL, + DeltaOps: r.restore, + LastBackupName: lastRestored, + Filename: snapshotLvolName, + ConcurrentLimit: int32(concurrentLimit), + }) +} + func (r *Replica) backupRestore(backupURL, snapshotLvolName string, concurrentLimit int32) error { backupURL = butil.UnescapeURL(backupURL) @@ -1997,7 +2028,7 @@ func (r *Replica) canDoIncrementalRestore(restore *Restore, backupURL, requested return true } -func (r *Replica) completeBackupRestore(spdkClient *spdkclient.Client) (err error) { +func (r *Replica) completeBackupRestore(spdkClient *spdkclient.Client, isFullRestore bool) (err error) { defer func() { if extraErr := r.finishRestore(err); extraErr != nil { r.log.WithError(extraErr).Error("Failed to finish backup restore") @@ -2012,9 +2043,11 @@ func (r *Replica) completeBackupRestore(spdkClient *spdkclient.Client) (err erro restore := r.restore.DeepCopy() r.RUnlock() - // TODO: Support postIncrementalRestoreOperations + if isFullRestore { + return r.postFullRestoreOperations(spdkClient, restore) + } - return r.postFullRestoreOperations(spdkClient, restore) + return r.postIncrementalRestoreOperations(restore) } func (r *Replica) waitForRestoreComplete() error { @@ -2045,6 +2078,11 @@ func (r *Replica) waitForRestoreComplete() error { return nil } +func (r *Replica) postIncrementalRestoreOperations(restore *Restore) error { + r.log.Infof("Done running incremental restore %v to lvol %v", restore.BackupURL, restore.LvolName) + return nil +} + func (r *Replica) postFullRestoreOperations(spdkClient *spdkclient.Client, restore *Restore) error { if r.restore.State == btypes.ProgressStateCanceled { r.log.Info("Doing nothing for canceled backup restoration") diff --git a/vendor/github.com/longhorn/longhorn-spdk-engine/pkg/spdk/restore.go b/vendor/github.com/longhorn/longhorn-spdk-engine/pkg/spdk/restore.go index 74c8b08ed..bf9020856 100644 --- a/vendor/github.com/longhorn/longhorn-spdk-engine/pkg/spdk/restore.go +++ b/vendor/github.com/longhorn/longhorn-spdk-engine/pkg/spdk/restore.go @@ -113,7 +113,7 @@ func (r *Restore) DeepCopy() *Restore { } } -func (r *Restore) OpenVolumeDev(volDevName string) (*os.File, string, error) { +func (r *Restore) OpenVolumeDev(volDevName string, dmDeviceAndEndpointCleanupRequired bool) (*os.File, string, error) { lvolName := r.replica.Name r.log.Info("Unexposing lvol bdev before restoration") @@ -141,7 +141,7 @@ func (r *Restore) OpenVolumeDev(volDevName string) (*os.File, string, error) { if err != nil { return nil, "", errors.Wrapf(err, "failed to create NVMe initiator for lvol bdev %v", lvolName) } - if _, err := initiator.Start(r.ip, strconv.Itoa(int(r.port)), false); err != nil { + if _, err := initiator.Start(r.ip, strconv.Itoa(int(r.port)), dmDeviceAndEndpointCleanupRequired); err != nil { return nil, "", errors.Wrapf(err, "failed to start NVMe initiator for lvol bdev %v", lvolName) } r.initiator = initiator diff --git a/vendor/modules.txt b/vendor/modules.txt index 5bef64abb..dceed4f18 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -180,7 +180,7 @@ github.com/kr/pretty # github.com/kr/text v0.2.0 ## explicit github.com/kr/text -# github.com/longhorn/backupstore v0.0.0-20240811043357-5c2b7879457f +# github.com/longhorn/backupstore v0.0.0-20240811043357-5c2b7879457f => github.com/c3y1huang/backupstore v0.0.0-20240820074828-9c490fd434d0 ## explicit; go 1.22.0 github.com/longhorn/backupstore github.com/longhorn/backupstore/azblob @@ -236,7 +236,7 @@ github.com/longhorn/longhorn-engine/pkg/sync github.com/longhorn/longhorn-engine/pkg/types github.com/longhorn/longhorn-engine/pkg/util github.com/longhorn/longhorn-engine/pkg/util/disk -# github.com/longhorn/longhorn-spdk-engine v0.0.0-20240815033436-41791a2d0239 +# github.com/longhorn/longhorn-spdk-engine v0.0.0-20240815033436-41791a2d0239 => github.com/c3y1huang/longhorn-spdk-engine v0.0.0-20240820075856-da4b96680f2e ## explicit; go 1.22.0 github.com/longhorn/longhorn-spdk-engine/pkg/api github.com/longhorn/longhorn-spdk-engine/pkg/client @@ -775,3 +775,5 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml +# github.com/longhorn/longhorn-spdk-engine => github.com/c3y1huang/longhorn-spdk-engine v0.0.0-20240820075856-da4b96680f2e +# github.com/longhorn/backupstore => github.com/c3y1huang/backupstore v0.0.0-20240820074828-9c490fd434d0