From c566098c81bcc96fac0a88b48c6645b3e9241d9a Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 16 Mar 2023 22:29:02 +0800 Subject: [PATCH] vendor: update `backupstore` by v1.3.x branch from `20220913112826-5f5c95274f2a` to `20230306022849-8d5e216c3b33` Ref: 5265 Signed-off-by: James Lu --- go.mod | 2 +- go.sum | 3 +- .../longhorn/backupstore/Dockerfile.dapper | 2 +- .../longhorn/backupstore/backupstore.go | 26 +- .../github.com/longhorn/backupstore/config.go | 63 ++- .../longhorn/backupstore/deltablock.go | 36 +- .../github.com/longhorn/backupstore/driver.go | 8 +- .../longhorn/backupstore/fsops/fsops.go | 15 +- vendor/github.com/longhorn/backupstore/go.mod | 1 + .../longhorn/backupstore/http/client.go | 6 + .../github.com/longhorn/backupstore/list.go | 4 +- .../github.com/longhorn/backupstore/lock.go | 8 +- .../longhorn/backupstore/logging/logging.go | 1 + .../longhorn/backupstore/nfs/nfs.go | 19 +- .../github.com/longhorn/backupstore/s3/s3.go | 26 +- .../longhorn/backupstore/s3/s3_service.go | 7 +- .../longhorn/backupstore/singlefile.go | 10 +- .../longhorn/backupstore/util/util.go | 21 +- .../longhorn/backupstore/vfs/vfs.go | 2 +- .../golang.org/x/net/http/httpproxy/proxy.go | 370 ++++++++++++++++++ vendor/modules.txt | 3 +- 21 files changed, 533 insertions(+), 100 deletions(-) create mode 100644 vendor/golang.org/x/net/http/httpproxy/proxy.go diff --git a/go.mod b/go.mod index 6c2c040b9..bea850b41 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/RoaringBitmap/roaring v0.4.18 github.com/golang/protobuf v1.3.3-0.20190920234318-1680a479a2cf github.com/google/uuid v1.3.0 - github.com/longhorn/backupstore v0.0.0-20220913112826-5f5c95274f2a + github.com/longhorn/backupstore v0.0.0-20230306022849-8d5e216c3b33 github.com/longhorn/longhorn-engine v1.3.3-0.20230216042703-718990dc8a35 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 diff --git a/go.sum b/go.sum index 055becc6f..597880990 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,9 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/longhorn/backupstore v0.0.0-20220913112826-5f5c95274f2a h1:f+mLqp3A5M7plw1pBgf8K1nvJxSU7mrGtU7bii+W5Bk= github.com/longhorn/backupstore v0.0.0-20220913112826-5f5c95274f2a/go.mod h1:hvIVsrpjPey7KupirAh0WoPMg0ArWnE6fA5bI30X7AI= +github.com/longhorn/backupstore v0.0.0-20230306022849-8d5e216c3b33 h1:zfRoewEi64drdl4PzGcdw88jobkiAbQpxBT6IbT+zmY= +github.com/longhorn/backupstore v0.0.0-20230306022849-8d5e216c3b33/go.mod h1:NFJYOfVUk22TZPSVbKHFgL0JgbJP0suXdixr1042sM8= github.com/longhorn/go-iscsi-helper v0.0.0-20230215045129-588aa7586e4c h1:M4UR1cVrVHHKqwc4aL3MUlp1d/Z2i7T7VS1y0SSUwa8= github.com/longhorn/go-iscsi-helper v0.0.0-20230215045129-588aa7586e4c/go.mod h1:9z/y9glKmWEdV50tjlUPxFwi1goQfIrrsoZbnMyIZbY= github.com/longhorn/longhorn-engine v1.3.3-0.20230216042703-718990dc8a35 h1:3RgCpW+8XGrUq3U5pSH1cDszVu6gIWhGs2akEEdtAZc= diff --git a/vendor/github.com/longhorn/backupstore/Dockerfile.dapper b/vendor/github.com/longhorn/backupstore/Dockerfile.dapper index b965fb96a..c6c3f58b6 100644 --- a/vendor/github.com/longhorn/backupstore/Dockerfile.dapper +++ b/vendor/github.com/longhorn/backupstore/Dockerfile.dapper @@ -19,7 +19,7 @@ ENV GOLANG_ARCH_amd64=amd64 GOLANG_ARCH_arm=armv6l GOLANG_ARCH=GOLANG_ARCH_${ARC GOPATH=/go PATH=/go/bin:/usr/local/go/bin:${PATH} SHELL=/bin/bash RUN wget -O - https://storage.googleapis.com/golang/go1.13.3.linux-${!GOLANG_ARCH}.tar.gz | tar -xzf - -C /usr/local && \ - go get github.com/rancher/trash && go get -u golang.org/x/lint/golint + go get github.com/rancher/trash && GO111MODULE=on go get golang.org/x/lint/golint@v0.0.0-20210508222113-6edffad5e616 ENV DAPPER_SOURCE /go/src/github.com/longhorn/backupstore ENV DAPPER_OUTPUT ./bin diff --git a/vendor/github.com/longhorn/backupstore/backupstore.go b/vendor/github.com/longhorn/backupstore/backupstore.go index 3321066fb..f0008fae2 100644 --- a/vendor/github.com/longhorn/backupstore/backupstore.go +++ b/vendor/github.com/longhorn/backupstore/backupstore.go @@ -5,6 +5,7 @@ import ( "net/url" "github.com/longhorn/backupstore/util" + "github.com/pkg/errors" ) type Volume struct { @@ -56,21 +57,20 @@ func addVolume(volume *Volume, driver BackupStoreDriver) error { } if !util.ValidateName(volume.Name) { - return fmt.Errorf("Invalid volume name %v", volume.Name) + return fmt.Errorf("invalid volume name %v", volume.Name) } if err := saveVolume(volume, driver); err != nil { - log.Error("Fail add volume ", volume.Name) + log.WithError(err).Errorf("Failed to add volume %v", volume.Name) return err } - log.Debug("Added backupstore volume ", volume.Name) - + log.Infof("Added backupstore volume %v", volume.Name) return nil } func removeVolume(volumeName string, driver BackupStoreDriver) error { if !util.ValidateName(volumeName) { - return fmt.Errorf("Invalid volume name %v", volumeName) + return fmt.Errorf("invalid volume name %v", volumeName) } volumeDir := getVolumePath(volumeName) @@ -78,20 +78,20 @@ func removeVolume(volumeName string, driver BackupStoreDriver) error { volumeBackupsDirectory := getBackupPath(volumeName) volumeLocksDirectory := getLockPath(volumeName) if err := driver.Remove(volumeBackupsDirectory); err != nil { - return fmt.Errorf("failed to remove all the backups for volume %v: %v", volumeName, err) + return errors.Wrapf(err, "failed to remove all the backups for volume %v", volumeName) } if err := driver.Remove(volumeBlocksDirectory); err != nil { - return fmt.Errorf("failed to remove all the blocks for volume %v: %v", volumeName, err) + return errors.Wrapf(err, "failed to remove all the blocks for volume %v", volumeName) } if err := driver.Remove(volumeLocksDirectory); err != nil { - return fmt.Errorf("failed to remove all the locks for volume %v: %v", volumeName, err) + return errors.Wrapf(err, "failed to remove all the locks for volume %v", volumeName) } if err := driver.Remove(volumeDir); err != nil { - return fmt.Errorf("failed to remove backup volume %v directory in backupstore: %v", volumeName, err) + return errors.Wrapf(err, "failed to remove backup volume %v directory in backupstore", volumeName) } - log.Debug("Removed volume directory in backupstore: ", volumeDir) - log.Debug("Removed backupstore volume ", volumeName) + log.Infof("Removed volume directory in backupstore %v", volumeDir) + log.Infof("Removed backupstore volume %v", volumeName) return nil } @@ -114,10 +114,10 @@ func DecodeBackupURL(backupURL string) (string, string, string, error) { volumeName := v.Get("volume") backupName := v.Get("backup") if !util.ValidateName(volumeName) { - return "", "", "", fmt.Errorf("Invalid volume name parsed, got %v", volumeName) + return "", "", "", fmt.Errorf("invalid volume name parsed, got %v", volumeName) } if backupName != "" && !util.ValidateName(backupName) { - return "", "", "", fmt.Errorf("Invalid backup name parsed, got %v", backupName) + return "", "", "", fmt.Errorf("invalid backup name parsed, got %v", backupName) } u.RawQuery = "" destURL := u.String() diff --git a/vendor/github.com/longhorn/backupstore/config.go b/vendor/github.com/longhorn/backupstore/config.go index 9e38d3af7..8b5e3b302 100644 --- a/vendor/github.com/longhorn/backupstore/config.go +++ b/vendor/github.com/longhorn/backupstore/config.go @@ -33,7 +33,7 @@ func getBackupConfigName(id string) string { return BACKUP_CONFIG_PREFIX + id + CFG_SUFFIX } -func loadConfigInBackupStore(filePath string, driver BackupStoreDriver, v interface{}) error { +func LoadConfigInBackupStore(filePath string, driver BackupStoreDriver, v interface{}) error { if !driver.FileExists(filePath) { return fmt.Errorf("cannot find %v in backupstore", filePath) } @@ -61,7 +61,7 @@ func loadConfigInBackupStore(filePath string, driver BackupStoreDriver, v interf return nil } -func saveConfigInBackupStore(filePath string, driver BackupStoreDriver, v interface{}) error { +func SaveConfigInBackupStore(filePath string, driver BackupStoreDriver, v interface{}) error { j, err := json.Marshal(v) if err != nil { return err @@ -84,6 +84,49 @@ func saveConfigInBackupStore(filePath string, driver BackupStoreDriver, v interf return nil } +func SaveLocalFileToBackupStore(localFilePath, backupStoreFilePath string, driver BackupStoreDriver) error { + log := log.WithFields(logrus.Fields{ + LogFieldReason: LogReasonStart, + LogFieldObject: LogObjectConfig, + LogFieldKind: driver.Kind(), + LogFieldFilepath: localFilePath, + LogFieldDestURL: backupStoreFilePath, + }) + log.Debug() + + if driver.FileExists(backupStoreFilePath) { + return fmt.Errorf("%v already exists", backupStoreFilePath) + } + + if err := driver.Upload(localFilePath, backupStoreFilePath); err != nil { + return err + } + + log.WithField(LogFieldReason, LogReasonComplete).Debug() + return nil +} + +func SaveBackupStoreToLocalFile(backupStoreFileURL, localFilePath string, driver BackupStoreDriver) error { + log := log.WithFields(logrus.Fields{ + LogFieldReason: LogReasonStart, + LogFieldObject: LogObjectConfig, + LogFieldKind: driver.Kind(), + LogFieldFilepath: localFilePath, + LogFieldSourceURL: backupStoreFileURL, + }) + log.Debug() + + if err := driver.Download(backupStoreFileURL, localFilePath); err != nil { + return err + } + + log = log.WithFields(logrus.Fields{ + LogFieldReason: LogReasonComplete, + }) + log.Debug() + return nil +} + func volumeExists(volumeName string, driver BackupStoreDriver) bool { volumeFile := getVolumeFilePath(volumeName) return driver.FileExists(volumeFile) @@ -108,7 +151,7 @@ func getVolumeNames(jobQueues *jobq.WorkerDispatcher, jobQueueTimeout time.Durat volumePathBase := filepath.Join(backupstoreBase, VOLUME_DIRECTORY) lv1Dirs, err := driver.List(volumePathBase) if err != nil { - log.Warnf("failed to list first level dirs for path: %v reason: %v", volumePathBase, err) + log.WithError(err).Warnf("Failed to list first level dirs for path %v", volumePathBase) return names, err } @@ -122,7 +165,7 @@ func getVolumeNames(jobQueues *jobq.WorkerDispatcher, jobQueueTimeout time.Durat lv1Tracker := jobQueues.QueueTimedFunc(context.Background(), func(ctx context.Context) (interface{}, error) { lv2Dirs, err := driver.List(path) if err != nil { - log.Warnf("failed to list second level dirs for path: %v reason: %v", path, err) + log.WithError(err).Warnf("Failed to list second level dirs for path %v", path) return nil, err } @@ -148,7 +191,7 @@ func getVolumeNames(jobQueues *jobq.WorkerDispatcher, jobQueueTimeout time.Durat lv2Tracker := jobQueues.QueueTimedFunc(context.Background(), func(ctx context.Context) (interface{}, error) { volumeNames, err := driver.List(path) if err != nil { - log.Warnf("failed to list volume names for path: %v reason: %v", path, err) + log.WithError(err).Warnf("Failed to list volume names for path %v", path) return nil, err } return volumeNames, nil @@ -175,7 +218,7 @@ func getVolumeNames(jobQueues *jobq.WorkerDispatcher, jobQueueTimeout time.Durat func loadVolume(volumeName string, driver BackupStoreDriver) (*Volume, error) { v := &Volume{} file := getVolumeFilePath(volumeName) - if err := loadConfigInBackupStore(file, driver, v); err != nil { + if err := LoadConfigInBackupStore(file, driver, v); err != nil { return nil, err } return v, nil @@ -183,7 +226,7 @@ func loadVolume(volumeName string, driver BackupStoreDriver) (*Volume, error) { func saveVolume(v *Volume, driver BackupStoreDriver) error { file := getVolumeFilePath(v.Name) - if err := saveConfigInBackupStore(file, driver, v); err != nil { + if err := SaveConfigInBackupStore(file, driver, v); err != nil { return err } return nil @@ -219,7 +262,7 @@ func backupExists(backupName, volumeName string, bsDriver BackupStoreDriver) boo func loadBackup(backupName, volumeName string, bsDriver BackupStoreDriver) (*Backup, error) { backup := &Backup{} - if err := loadConfigInBackupStore(getBackupConfigPath(backupName, volumeName), bsDriver, backup); err != nil { + if err := LoadConfigInBackupStore(getBackupConfigPath(backupName, volumeName), bsDriver, backup); err != nil { return nil, err } return backup, nil @@ -230,7 +273,7 @@ func saveBackup(backup *Backup, bsDriver BackupStoreDriver) error { return fmt.Errorf("missing volume specifier for backup: %v", backup.Name) } filePath := getBackupConfigPath(backup.Name, backup.VolumeName) - if err := saveConfigInBackupStore(filePath, bsDriver, backup); err != nil { + if err := SaveConfigInBackupStore(filePath, bsDriver, backup); err != nil { return err } return nil @@ -241,6 +284,6 @@ func removeBackup(backup *Backup, bsDriver BackupStoreDriver) error { if err := bsDriver.Remove(filePath); err != nil { return err } - log.Debugf("Removed %v on backupstore", filePath) + log.Infof("Removed %v on backupstore", filePath) return nil } diff --git a/vendor/github.com/longhorn/backupstore/deltablock.go b/vendor/github.com/longhorn/backupstore/deltablock.go index 28bd1b723..81f8fb0e4 100644 --- a/vendor/github.com/longhorn/backupstore/deltablock.go +++ b/vendor/github.com/longhorn/backupstore/deltablock.go @@ -7,6 +7,7 @@ import ( "path/filepath" "time" + "github.com/pkg/errors" "github.com/sirupsen/logrus" . "github.com/longhorn/backupstore/logging" @@ -101,7 +102,7 @@ const ( func CreateDeltaBlockBackup(config *DeltaBackupConfig) (string, bool, error) { if config == nil { - return "", false, fmt.Errorf("Invalid empty config for backup") + return "", false, fmt.Errorf("invalid empty config for backup") } volume := config.Volume @@ -109,7 +110,7 @@ func CreateDeltaBlockBackup(config *DeltaBackupConfig) (string, bool, error) { destURL := config.DestURL deltaOps := config.DeltaOps if deltaOps == nil { - return "", false, fmt.Errorf("Missing DeltaBlockBackupOperations") + return "", false, fmt.Errorf("missing DeltaBlockBackupOperations") } bsDriver, err := GetBackupStoreDriver(destURL) @@ -182,6 +183,7 @@ func CreateDeltaBlockBackup(config *DeltaBackupConfig) (string, bool, error) { LogFieldSnapshot: snapshot.Name, LogFieldLastSnapshot: backupRequest.getLastSnapshotName(), }).Debug("Generating snapshot changed blocks config") + delta, err := deltaOps.CompareSnapshot(snapshot.Name, backupRequest.getLastSnapshotName(), volume.Name) if err != nil { deltaOps.CloseSnapshot(snapshot.Name, volume.Name) @@ -426,13 +428,13 @@ func RestoreDeltaBlockBackup(config *DeltaRestoreConfig) error { } if vol.Size == 0 || vol.Size%DEFAULT_BLOCK_SIZE != 0 { - return fmt.Errorf("Read invalid volume size %v", vol.Size) + return fmt.Errorf("read invalid volume size %v", vol.Size) } if _, err := os.Stat(volDevName); err == nil { logrus.Warnf("File %s for the restore exists, will remove and re-create it", volDevName) if err := os.Remove(volDevName); err != nil { - return fmt.Errorf("failed to clean up the existing file %v before restore: %v", volDevName, err) + return errors.Wrapf(err, "failed to clean up the existing file %v before restore", volDevName) } } @@ -579,7 +581,7 @@ func RestoreDeltaBlockBackupIncrementally(config *DeltaRestoreConfig) error { if _, err := os.Stat(volDevName); err == nil { logrus.Warnf("File %s for the incremental restore exists, will remove and re-create it", volDevName) if err := os.Remove(volDevName); err != nil { - return fmt.Errorf("failed to clean up the existing file %v before incremental restore: %v", volDevName, err) + return errors.Wrapf(err, "failed to clean up the existing file %v before incremental restore", volDevName) } } @@ -721,18 +723,14 @@ func DeleteBackupVolume(volumeName string, destURL string) error { return err } defer lock.Unlock() - if err := removeVolume(volumeName, bsDriver); err != nil { - return err - } - return nil + return removeVolume(volumeName, bsDriver) } func checkBlockReferenceCount(blockInfos map[string]*BlockInfo, backup *Backup, volumeName string, driver BackupStoreDriver) { for _, block := range backup.Blocks { info, known := blockInfos[block.BlockChecksum] if !known { - log.Errorf("backup %v refers to unknown block %v", - backup.Name, block.BlockChecksum) + log.Errorf("Backup %v refers to unknown block %v", backup.Name, block.BlockChecksum) info = &BlockInfo{checksum: block.BlockChecksum} blockInfos[block.BlockChecksum] = info } @@ -750,12 +748,12 @@ func getLatestBackup(backup *Backup, lastBackup *Backup) error { backupTime, err := time.Parse(time.RFC3339, backup.SnapshotCreatedAt) if err != nil { - return fmt.Errorf("Cannot parse backup %v time %v due to %v", backup.Name, backup.SnapshotCreatedAt, err) + return errors.Wrapf(err, "cannot parse backup %v time %v", backup.Name, backup.SnapshotCreatedAt) } lastBackupTime, err := time.Parse(time.RFC3339, lastBackup.SnapshotCreatedAt) if err != nil { - return fmt.Errorf("Cannot parse last backup %v time %v due to %v", lastBackup.Name, lastBackup.SnapshotCreatedAt, err) + return errors.Wrapf(err, "cannot parse last backup %v time %v", lastBackup.Name, lastBackup.SnapshotCreatedAt) } if backupTime.After(lastBackupTime) { @@ -792,7 +790,7 @@ func DeleteDeltaBlockBackup(backupURL string) error { // If we fail to load the backup we still want to proceed with the deletion of the backup file backupToBeDeleted, err := loadBackup(backupName, volumeName, bsDriver) if err != nil { - log.WithError(err).Warn("failed to load to be deleted backup") + log.WithError(err).Warn("Failed to load to be deleted backup") backupToBeDeleted = &Backup{ Name: backupName, VolumeName: volumeName, @@ -807,7 +805,7 @@ func DeleteDeltaBlockBackup(backupURL string) error { v, err := loadVolume(volumeName, bsDriver) if err != nil { - return fmt.Errorf("Cannot find volume in backupstore due to: %v", err) + return errors.Wrap(err, "cannot find volume in backupstore") } updateLastBackup := false if backupToBeDeleted.Name == v.LastBackupName { @@ -820,7 +818,7 @@ func DeleteDeltaBlockBackup(backupURL string) error { deleteBlocks := true backupNames, err := getBackupNamesForVolume(volumeName, bsDriver) if err != nil { - log.WithError(err).Warn("failed to load backup names, skip block deletion") + log.WithError(err).Warn("Failed to load backup names, skip block deletion") deleteBlocks = false } @@ -842,7 +840,7 @@ func DeleteDeltaBlockBackup(backupURL string) error { log := log.WithField("backup", name) backup, err := loadBackup(name, volumeName, bsDriver) if err != nil { - log.WithError(err).Warn("failed to load backup, skip block deletion") + log.WithError(err).Warn("Failed to load backup, skip block deletion") deleteBlocks = false break } @@ -861,7 +859,7 @@ func DeleteDeltaBlockBackup(backupURL string) error { if updateLastBackup { err := getLatestBackup(backup, lastBackup) if err != nil { - log.WithError(err).Warn("failed to find last backup, skip block deletion") + log.WithError(err).Warn("Failed to find last backup, skip block deletion") deleteBlocks = false break } @@ -904,7 +902,7 @@ func cleanupBlocks(blockMap map[string]*BlockInfo, volume string, driver BackupS deletionFailures = append(deletionFailures, blk.checksum) continue } - log.Debugf("deleted block %v for volume %v", blk.checksum, volume) + log.Debugf("Deleted block %v for volume %v", blk.checksum, volume) deletedBlockCount++ } else if isBlockReferenced(blk) && isBlockPresent(blk) { activeBlockCount++ diff --git a/vendor/github.com/longhorn/backupstore/driver.go b/vendor/github.com/longhorn/backupstore/driver.go index d3079eaf7..443a13979 100644 --- a/vendor/github.com/longhorn/backupstore/driver.go +++ b/vendor/github.com/longhorn/backupstore/driver.go @@ -35,6 +35,10 @@ var ( log = logrus.WithFields(logrus.Fields{"pkg": "backupstore"}) ) +func GetLog() logrus.FieldLogger { + return log +} + func generateError(fields logrus.Fields, format string, v ...interface{}) error { return ErrorWithFields("backupstore", fields, format, v...) } @@ -61,14 +65,14 @@ func unregisterDriver(kind string) error { func GetBackupStoreDriver(destURL string) (BackupStoreDriver, error) { if destURL == "" { - return nil, fmt.Errorf("Destination URL hasn't been specified") + return nil, fmt.Errorf("destination URL hasn't been specified") } u, err := url.Parse(destURL) if err != nil { return nil, err } if _, exists := initializers[u.Scheme]; !exists { - return nil, fmt.Errorf("Driver %v is not supported!", u.Scheme) + return nil, fmt.Errorf("driver %v is not supported", u.Scheme) } return initializers[u.Scheme](destURL) } diff --git a/vendor/github.com/longhorn/backupstore/fsops/fsops.go b/vendor/github.com/longhorn/backupstore/fsops/fsops.go index fafe6d540..157ff9db1 100644 --- a/vendor/github.com/longhorn/backupstore/fsops/fsops.go +++ b/vendor/github.com/longhorn/backupstore/fsops/fsops.go @@ -29,10 +29,7 @@ func NewFileSystemOperator(ops FileSystemOps) *FileSystemOperator { } func (f *FileSystemOperator) preparePath(file string) error { - if err := os.MkdirAll(filepath.Dir(f.LocalPath(file)), os.ModeDir|0700); err != nil { - return err - } - return nil + return os.MkdirAll(filepath.Dir(f.LocalPath(file)), os.ModeDir|0700) } func (f *FileSystemOperator) FileSize(filePath string) int64 { @@ -140,16 +137,10 @@ func (f *FileSystemOperator) Upload(src, dst string) error { return err } _, err = util.Execute("mv", []string{f.LocalPath(tmpDst), f.LocalPath(dst)}) - if err != nil { - return err - } - return nil + return err } func (f *FileSystemOperator) Download(src, dst string) error { _, err := util.Execute("cp", []string{f.LocalPath(src), dst}) - if err != nil { - return err - } - return nil + return err } diff --git a/vendor/github.com/longhorn/backupstore/go.mod b/vendor/github.com/longhorn/backupstore/go.mod index 79f8d8cc9..019378087 100644 --- a/vendor/github.com/longhorn/backupstore/go.mod +++ b/vendor/github.com/longhorn/backupstore/go.mod @@ -11,6 +11,7 @@ require ( github.com/spf13/afero v1.5.1 github.com/stretchr/testify v1.7.0 github.com/urfave/cli v1.14.0 + golang.org/x/net v0.0.0-20200202094626-16171245cfb2 gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/vendor/github.com/longhorn/backupstore/http/client.go b/vendor/github.com/longhorn/backupstore/http/client.go index f2300e695..85fcdd5f0 100644 --- a/vendor/github.com/longhorn/backupstore/http/client.go +++ b/vendor/github.com/longhorn/backupstore/http/client.go @@ -5,6 +5,9 @@ import ( "crypto/x509" "fmt" "net/http" + "net/url" + + "golang.org/x/net/http/httpproxy" ) func getSystemCerts() *x509.CertPool { @@ -43,6 +46,9 @@ func GetClient(insecure bool, customCerts []byte) (*http.Client, error) { InsecureSkipVerify: insecure, RootCAs: certs, } + customTransport.Proxy = func(request *http.Request) (*url.URL, error) { + return httpproxy.FromEnvironment().ProxyFunc()(request.URL) + } client := &http.Client{Transport: customTransport} return client, nil } diff --git a/vendor/github.com/longhorn/backupstore/list.go b/vendor/github.com/longhorn/backupstore/list.go index dde1babf2..9c7b74f61 100644 --- a/vendor/github.com/longhorn/backupstore/list.go +++ b/vendor/github.com/longhorn/backupstore/list.go @@ -51,11 +51,11 @@ type BackupInfo struct { func addListVolume(volumeName string, driver BackupStoreDriver, volumeOnly bool) (*VolumeInfo, error) { if volumeName == "" { - return nil, fmt.Errorf("Invalid empty volume Name") + return nil, fmt.Errorf("invalid empty volume Name") } if !util.ValidateName(volumeName) { - return nil, fmt.Errorf("Invalid volume name %v", volumeName) + return nil, fmt.Errorf("invalid volume name %v", volumeName) } volumeInfo := &VolumeInfo{Messages: make(map[MessageType]string)} diff --git a/vendor/github.com/longhorn/backupstore/lock.go b/vendor/github.com/longhorn/backupstore/lock.go index e5a708856..6e3bc73bb 100644 --- a/vendor/github.com/longhorn/backupstore/lock.go +++ b/vendor/github.com/longhorn/backupstore/lock.go @@ -62,7 +62,7 @@ func (lock *FileLock) canAcquire() bool { canAcquire := true locks := getLocksForVolume(lock.volume, lock.driver) file := getLockFilePath(lock.volume, lock.Name) - log.WithField("lock", lock).Debugf("trying to acquire lock %v", file) + log.WithField("lock", lock).Debugf("Trying to acquire lock %v", file) log.Debugf("backupstore volume %v contains locks %v", lock.volume, locks) for _, serverLock := range locks { @@ -164,7 +164,7 @@ func (lock *FileLock) Unlock() error { func loadLock(volumeName string, name string, driver BackupStoreDriver) (*FileLock, error) { lock := &FileLock{} file := getLockFilePath(volumeName, name) - if err := loadConfigInBackupStore(file, driver, lock); err != nil { + if err := LoadConfigInBackupStore(file, driver, lock); err != nil { return nil, err } lock.serverTime = driver.FileTime(file) @@ -183,7 +183,7 @@ func removeLock(lock *FileLock) error { func saveLock(lock *FileLock) error { file := getLockFilePath(lock.volume, lock.Name) - if err := saveConfigInBackupStore(file, lock.driver, lock); err != nil { + if err := SaveConfigInBackupStore(file, lock.driver, lock); err != nil { return err } lock.serverTime = lock.driver.FileTime(file) @@ -226,7 +226,7 @@ func getLocksForVolume(volumeName string, driver BackupStoreDriver) []*FileLock lock, err := loadLock(volumeName, name, driver) if err != nil { file := getLockFilePath(volumeName, name) - log.Warnf("failed to load lock %v on backupstore reason %v", file, err) + log.WithError(err).Warnf("Failed to load lock %v on backupstore", file) continue } locks = append(locks, lock) diff --git a/vendor/github.com/longhorn/backupstore/logging/logging.go b/vendor/github.com/longhorn/backupstore/logging/logging.go index e57d9e96e..c040e98d9 100644 --- a/vendor/github.com/longhorn/backupstore/logging/logging.go +++ b/vendor/github.com/longhorn/backupstore/logging/logging.go @@ -18,6 +18,7 @@ const ( LogFieldLastSnapshot = "last_snapshot" LogEventBackupURL = "backup_url" LogFieldDestURL = "dest_url" + LogFieldSourceURL = "source_url" LogFieldKind = "kind" LogFieldFilepath = "filepath" diff --git a/vendor/github.com/longhorn/backupstore/nfs/nfs.go b/vendor/github.com/longhorn/backupstore/nfs/nfs.go index 15af7aea1..828f1473c 100644 --- a/vendor/github.com/longhorn/backupstore/nfs/nfs.go +++ b/vendor/github.com/longhorn/backupstore/nfs/nfs.go @@ -15,8 +15,9 @@ import ( ) var ( - log = logrus.WithFields(logrus.Fields{"pkg": "nfs"}) - MinorVersions = []string{"4.2", "4.1", "4.0"} + log = logrus.WithFields(logrus.Fields{"pkg": "nfs"}) + MinorVersions = []string{"4.2", "4.1", "4.0"} + defaultTimeout = 5 * time.Second ) type BackupStoreDriver struct { @@ -59,24 +60,24 @@ func initFunc(destURL string) (backupstore.BackupStoreDriver, error) { return nil, fmt.Errorf("NFS path must follow: nfs://server:/path/ format") } if u.Path == "" { - return nil, fmt.Errorf("Cannot find nfs path") + return nil, fmt.Errorf("cannot find nfs path") } b.serverPath = u.Host + u.Path b.mountDir = filepath.Join(MountDir, strings.TrimRight(strings.Replace(u.Host, ".", "_", -1), ":"), u.Path) - if _, err = util.ExecuteWithCustomTimeout("mkdir", []string{"-m", "700", "-p", b.mountDir}, 3*time.Second); err != nil { - return nil, fmt.Errorf("Cannot create mount directory %v for NFS server: %v", b.mountDir, err) + if _, err = util.ExecuteWithCustomTimeout("mkdir", []string{"-m", "700", "-p", b.mountDir}, defaultTimeout); err != nil { + return nil, errors.Wrapf(err, "cannot create mount directory %v for NFS server", b.mountDir) } if err := b.mount(); err != nil { - return nil, fmt.Errorf("Cannot mount nfs %v: %v", b.serverPath, err) + return nil, errors.Wrapf(err, "cannot mount nfs %v", b.serverPath) } if _, err := b.List(""); err != nil { return nil, fmt.Errorf("NFS path %v doesn't exist or is not a directory", b.serverPath) } b.destURL = KIND + "://" + b.serverPath - log.Debugf("Loaded driver for %v", b.destURL) + log.Infof("Loaded driver for %v", b.destURL) return b, nil } @@ -88,8 +89,8 @@ func (b *BackupStoreDriver) mount() (err error) { retErr := errors.New("Cannot mount using NFSv4") for _, version := range MinorVersions { - log.Debugf("attempting mount for nfs path %v with nfsvers %v", b.serverPath, version) - _, err = util.Execute("mount", []string{"-t", "nfs4", "-o", fmt.Sprintf("nfsvers=%v", version), "-o", "actimeo=1", b.serverPath, b.mountDir}) + log.Debugf("Attempting mount for nfs path %v with nfsvers %v", b.serverPath, version) + _, err = util.ExecuteWithCustomTimeout("mount", []string{"-t", "nfs4", "-o", fmt.Sprintf("nfsvers=%v", version), "-o", "actimeo=1", b.serverPath, b.mountDir}, defaultTimeout) if err == nil { return nil } diff --git a/vendor/github.com/longhorn/backupstore/s3/s3.go b/vendor/github.com/longhorn/backupstore/s3/s3.go index aef885092..9a32f70ae 100644 --- a/vendor/github.com/longhorn/backupstore/s3/s3.go +++ b/vendor/github.com/longhorn/backupstore/s3/s3.go @@ -60,13 +60,12 @@ func initFunc(destURL string) (backupstore.BackupStoreDriver, error) { } // add custom ca to http client that is used by s3 service - if customCerts := getCustomCerts(); customCerts != nil { - client, err := http.GetClientWithCustomCerts(customCerts) - if err != nil { - return nil, err - } - b.service.Client = client + customCerts := getCustomCerts() + client, err := http.GetClientWithCustomCerts(customCerts) + if err != nil { + return nil, err } + b.service.Client = client //Leading '/' can cause mystery problems for s3 b.path = strings.TrimLeft(b.path, "/") @@ -82,7 +81,7 @@ func initFunc(destURL string) (backupstore.BackupStoreDriver, error) { } b.destURL += "/" + b.path - log.Debugf("Loaded driver for %v", b.destURL) + log.Infof("Loaded driver for %v", b.destURL) return b, nil } @@ -114,7 +113,7 @@ func (s *BackupStoreDriver) List(listPath string) ([]string, error) { path := s.updatePath(listPath) + "/" contents, prefixes, err := s.service.ListObjects(path, "/") if err != nil { - log.Error("Fail to list s3: ", err) + log.WithError(err).Error("Failed to list s3") return result, err } @@ -195,11 +194,17 @@ func (s *BackupStoreDriver) Download(src, dst string) error { if _, err := os.Stat(dst); err != nil { os.Remove(dst) } + + if err := os.MkdirAll(filepath.Dir(dst), os.ModeDir|0700); err != nil { + return err + } + f, err := os.Create(dst) if err != nil { return err } defer f.Close() + path := s.updatePath(src) rc, err := s.service.GetObject(path) if err != nil { @@ -208,8 +213,5 @@ func (s *BackupStoreDriver) Download(src, dst string) error { defer rc.Close() _, err = io.Copy(f, rc) - if err != nil { - return err - } - return nil + return err } diff --git a/vendor/github.com/longhorn/backupstore/s3/s3_service.go b/vendor/github.com/longhorn/backupstore/s3/s3_service.go index 6c516dcbd..6cee12e7d 100644 --- a/vendor/github.com/longhorn/backupstore/s3/s3_service.go +++ b/vendor/github.com/longhorn/backupstore/s3/s3_service.go @@ -10,6 +10,7 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" + "github.com/pkg/errors" ) type Service struct { @@ -163,12 +164,12 @@ func (s *Service) DeleteObjects(key string) error { objects, _, err := s.ListObjects(key, "") if err != nil { - return fmt.Errorf("failed to list objects with prefix %v before removing them error: %v", key, err) + return errors.Wrapf(err, "failed to list objects with prefix %v before removing them", key) } svc, err := s.New() if err != nil { - return fmt.Errorf("failed to get a new s3 client instance before removing objects: %v", err) + return errors.Wrap(err, "failed to get a new s3 client instance before removing objects") } defer s.Close() @@ -180,7 +181,7 @@ func (s *Service) DeleteObjects(key string) error { }) if err != nil { - log.Errorf("failed to delete object: %v response: %v error: %v", + log.Errorf("Failed to delete object: %v response: %v error: %v", aws.StringValue(object.Key), resp.String(), parseAwsError(err)) deletionFailures = append(deletionFailures, aws.StringValue(object.Key)) } diff --git a/vendor/github.com/longhorn/backupstore/singlefile.go b/vendor/github.com/longhorn/backupstore/singlefile.go index 24af97cb9..a959ff197 100644 --- a/vendor/github.com/longhorn/backupstore/singlefile.go +++ b/vendor/github.com/longhorn/backupstore/singlefile.go @@ -1,10 +1,10 @@ package backupstore import ( - "fmt" "path/filepath" "github.com/longhorn/backupstore/util" + "github.com/pkg/errors" "github.com/sirupsen/logrus" . "github.com/longhorn/backupstore/logging" @@ -117,7 +117,7 @@ func DeleteSingleFileBackup(backupURL string) error { _, err = loadVolume(volumeName, driver) if err != nil { - return fmt.Errorf("Cannot find volume %v in backupstore due to: %v", volumeName, err) + return errors.Wrapf(err, "cannot find volume %v in backupstore", volumeName) } backup, err := loadBackup(backupName, volumeName, driver) @@ -129,9 +129,5 @@ func DeleteSingleFileBackup(backupURL string) error { return err } - if err := removeBackup(backup, driver); err != nil { - return err - } - - return nil + return removeBackup(backup, driver) } diff --git a/vendor/github.com/longhorn/backupstore/util/util.go b/vendor/github.com/longhorn/backupstore/util/util.go index be23b864b..ae31a1348 100644 --- a/vendor/github.com/longhorn/backupstore/util/util.go +++ b/vendor/github.com/longhorn/backupstore/util/util.go @@ -4,11 +4,13 @@ import ( "bytes" "compress/gzip" "context" + "crypto/sha256" "crypto/sha512" "encoding/hex" "fmt" "io" "io/ioutil" + "os" "os/exec" "regexp" "strings" @@ -41,6 +43,21 @@ func GetChecksum(data []byte) string { return checksum } +func GetFileChecksum(filePath string) (string, error) { + f, err := os.Open(filePath) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + + return hex.EncodeToString(h.Sum(nil)), nil +} + func CompressData(data []byte) (io.ReadSeeker, error) { var b bytes.Buffer w := gzip.NewWriter(&b) @@ -154,11 +171,11 @@ func execute(ctx context.Context, binary string, args []string) (string, error) case <-done: break case <-ctx.Done(): - return "", fmt.Errorf("Timeout executing: %v %v, output %v, error %v", binary, args, string(output), err) + return "", fmt.Errorf("timeout executing: %v %v, output %v, error %v", binary, args, string(output), err) } if err != nil { - return "", fmt.Errorf("Failed to execute: %v %v, output %v, error %v", binary, args, string(output), err) + return "", fmt.Errorf("failed to execute: %v %v, output %v, error %v", binary, args, string(output), err) } return string(output), nil diff --git a/vendor/github.com/longhorn/backupstore/vfs/vfs.go b/vendor/github.com/longhorn/backupstore/vfs/vfs.go index 74d207ddd..6fbb0b7e8 100644 --- a/vendor/github.com/longhorn/backupstore/vfs/vfs.go +++ b/vendor/github.com/longhorn/backupstore/vfs/vfs.go @@ -60,7 +60,7 @@ func initFunc(destURL string) (backupstore.BackupStoreDriver, error) { } b.destURL = KIND + "://" + b.path - log.Debugf("Loaded driver for %v", b.destURL) + log.Infof("Loaded driver for %v", b.destURL) return b, nil } diff --git a/vendor/golang.org/x/net/http/httpproxy/proxy.go b/vendor/golang.org/x/net/http/httpproxy/proxy.go new file mode 100644 index 000000000..163645b86 --- /dev/null +++ b/vendor/golang.org/x/net/http/httpproxy/proxy.go @@ -0,0 +1,370 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httpproxy provides support for HTTP proxy determination +// based on environment variables, as provided by net/http's +// ProxyFromEnvironment function. +// +// The API is not subject to the Go 1 compatibility promise and may change at +// any time. +package httpproxy + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "strings" + "unicode/utf8" + + "golang.org/x/net/idna" +) + +// Config holds configuration for HTTP proxy settings. See +// FromEnvironment for details. +type Config struct { + // HTTPProxy represents the value of the HTTP_PROXY or + // http_proxy environment variable. It will be used as the proxy + // URL for HTTP requests and HTTPS requests unless overridden by + // HTTPSProxy or NoProxy. + HTTPProxy string + + // HTTPSProxy represents the HTTPS_PROXY or https_proxy + // environment variable. It will be used as the proxy URL for + // HTTPS requests unless overridden by NoProxy. + HTTPSProxy string + + // NoProxy represents the NO_PROXY or no_proxy environment + // variable. It specifies a string that contains comma-separated values + // specifying hosts that should be excluded from proxying. Each value is + // represented by an IP address prefix (1.2.3.4), an IP address prefix in + // CIDR notation (1.2.3.4/8), a domain name, or a special DNS label (*). + // An IP address prefix and domain name can also include a literal port + // number (1.2.3.4:80). + // A domain name matches that name and all subdomains. A domain name with + // a leading "." matches subdomains only. For example "foo.com" matches + // "foo.com" and "bar.foo.com"; ".y.com" matches "x.y.com" but not "y.com". + // A single asterisk (*) indicates that no proxying should be done. + // A best effort is made to parse the string and errors are + // ignored. + NoProxy string + + // CGI holds whether the current process is running + // as a CGI handler (FromEnvironment infers this from the + // presence of a REQUEST_METHOD environment variable). + // When this is set, ProxyForURL will return an error + // when HTTPProxy applies, because a client could be + // setting HTTP_PROXY maliciously. See https://golang.org/s/cgihttpproxy. + CGI bool +} + +// config holds the parsed configuration for HTTP proxy settings. +type config struct { + // Config represents the original configuration as defined above. + Config + + // httpsProxy is the parsed URL of the HTTPSProxy if defined. + httpsProxy *url.URL + + // httpProxy is the parsed URL of the HTTPProxy if defined. + httpProxy *url.URL + + // ipMatchers represent all values in the NoProxy that are IP address + // prefixes or an IP address in CIDR notation. + ipMatchers []matcher + + // domainMatchers represent all values in the NoProxy that are a domain + // name or hostname & domain name + domainMatchers []matcher +} + +// FromEnvironment returns a Config instance populated from the +// environment variables HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the +// lowercase versions thereof). HTTPS_PROXY takes precedence over +// HTTP_PROXY for https requests. +// +// The environment values may be either a complete URL or a +// "host[:port]", in which case the "http" scheme is assumed. An error +// is returned if the value is a different form. +func FromEnvironment() *Config { + return &Config{ + HTTPProxy: getEnvAny("HTTP_PROXY", "http_proxy"), + HTTPSProxy: getEnvAny("HTTPS_PROXY", "https_proxy"), + NoProxy: getEnvAny("NO_PROXY", "no_proxy"), + CGI: os.Getenv("REQUEST_METHOD") != "", + } +} + +func getEnvAny(names ...string) string { + for _, n := range names { + if val := os.Getenv(n); val != "" { + return val + } + } + return "" +} + +// ProxyFunc returns a function that determines the proxy URL to use for +// a given request URL. Changing the contents of cfg will not affect +// proxy functions created earlier. +// +// A nil URL and nil error are returned if no proxy is defined in the +// environment, or a proxy should not be used for the given request, as +// defined by NO_PROXY. +// +// As a special case, if req.URL.Host is "localhost" (with or without a +// port number), then a nil URL and nil error will be returned. +func (cfg *Config) ProxyFunc() func(reqURL *url.URL) (*url.URL, error) { + // Preprocess the Config settings for more efficient evaluation. + cfg1 := &config{ + Config: *cfg, + } + cfg1.init() + return cfg1.proxyForURL +} + +func (cfg *config) proxyForURL(reqURL *url.URL) (*url.URL, error) { + var proxy *url.URL + if reqURL.Scheme == "https" { + proxy = cfg.httpsProxy + } + if proxy == nil { + proxy = cfg.httpProxy + if proxy != nil && cfg.CGI { + return nil, errors.New("refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy") + } + } + if proxy == nil { + return nil, nil + } + if !cfg.useProxy(canonicalAddr(reqURL)) { + return nil, nil + } + + return proxy, nil +} + +func parseProxy(proxy string) (*url.URL, error) { + if proxy == "" { + return nil, nil + } + + proxyURL, err := url.Parse(proxy) + if err != nil || + (proxyURL.Scheme != "http" && + proxyURL.Scheme != "https" && + proxyURL.Scheme != "socks5") { + // proxy was bogus. Try prepending "http://" to it and + // see if that parses correctly. If not, we fall + // through and complain about the original one. + if proxyURL, err := url.Parse("http://" + proxy); err == nil { + return proxyURL, nil + } + } + if err != nil { + return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) + } + return proxyURL, nil +} + +// useProxy reports whether requests to addr should use a proxy, +// according to the NO_PROXY or no_proxy environment variable. +// addr is always a canonicalAddr with a host and port. +func (cfg *config) useProxy(addr string) bool { + if len(addr) == 0 { + return true + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return false + } + if host == "localhost" { + return false + } + ip := net.ParseIP(host) + if ip != nil { + if ip.IsLoopback() { + return false + } + } + + addr = strings.ToLower(strings.TrimSpace(host)) + + if ip != nil { + for _, m := range cfg.ipMatchers { + if m.match(addr, port, ip) { + return false + } + } + } + for _, m := range cfg.domainMatchers { + if m.match(addr, port, ip) { + return false + } + } + return true +} + +func (c *config) init() { + if parsed, err := parseProxy(c.HTTPProxy); err == nil { + c.httpProxy = parsed + } + if parsed, err := parseProxy(c.HTTPSProxy); err == nil { + c.httpsProxy = parsed + } + + for _, p := range strings.Split(c.NoProxy, ",") { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + + if p == "*" { + c.ipMatchers = []matcher{allMatch{}} + c.domainMatchers = []matcher{allMatch{}} + return + } + + // IPv4/CIDR, IPv6/CIDR + if _, pnet, err := net.ParseCIDR(p); err == nil { + c.ipMatchers = append(c.ipMatchers, cidrMatch{cidr: pnet}) + continue + } + + // IPv4:port, [IPv6]:port + phost, pport, err := net.SplitHostPort(p) + if err == nil { + if len(phost) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + if phost[0] == '[' && phost[len(phost)-1] == ']' { + phost = phost[1 : len(phost)-1] + } + } else { + phost = p + } + // IPv4, IPv6 + if pip := net.ParseIP(phost); pip != nil { + c.ipMatchers = append(c.ipMatchers, ipMatch{ip: pip, port: pport}) + continue + } + + if len(phost) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + + // domain.com or domain.com:80 + // foo.com matches bar.foo.com + // .domain.com or .domain.com:port + // *.domain.com or *.domain.com:port + if strings.HasPrefix(phost, "*.") { + phost = phost[1:] + } + matchHost := false + if phost[0] != '.' { + matchHost = true + phost = "." + phost + } + c.domainMatchers = append(c.domainMatchers, domainMatch{host: phost, port: pport, matchHost: matchHost}) + } +} + +var portMap = map[string]string{ + "http": "80", + "https": "443", + "socks5": "1080", +} + +// canonicalAddr returns url.Host but always with a ":port" suffix +func canonicalAddr(url *url.URL) string { + addr := url.Hostname() + if v, err := idnaASCII(addr); err == nil { + addr = v + } + port := url.Port() + if port == "" { + port = portMap[url.Scheme] + } + return net.JoinHostPort(addr, port) +} + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +func idnaASCII(v string) (string, error) { + // TODO: Consider removing this check after verifying performance is okay. + // Right now punycode verification, length checks, context checks, and the + // permissible character tests are all omitted. It also prevents the ToASCII + // call from salvaging an invalid IDN, when possible. As a result it may be + // possible to have two IDNs that appear identical to the user where the + // ASCII-only version causes an error downstream whereas the non-ASCII + // version does not. + // Note that for correct ASCII IDNs ToASCII will only do considerably more + // work, but it will not cause an allocation. + if isASCII(v) { + return v, nil + } + return idna.Lookup.ToASCII(v) +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} + +// matcher represents the matching rule for a given value in the NO_PROXY list +type matcher interface { + // match returns true if the host and optional port or ip and optional port + // are allowed + match(host, port string, ip net.IP) bool +} + +// allMatch matches on all possible inputs +type allMatch struct{} + +func (a allMatch) match(host, port string, ip net.IP) bool { + return true +} + +type cidrMatch struct { + cidr *net.IPNet +} + +func (m cidrMatch) match(host, port string, ip net.IP) bool { + return m.cidr.Contains(ip) +} + +type ipMatch struct { + ip net.IP + port string +} + +func (m ipMatch) match(host, port string, ip net.IP) bool { + if m.ip.Equal(ip) { + return m.port == "" || m.port == port + } + return false +} + +type domainMatch struct { + host string + port string + + matchHost bool +} + +func (m domainMatch) match(host, port string, ip net.IP) bool { + if strings.HasSuffix(host, m.host) || (m.matchHost && host == m.host[1:]) { + return m.port == "" || m.port == port + } + return false +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 76995775a..dc61d51f2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -68,7 +68,7 @@ github.com/gorilla/handlers github.com/honestbee/jobq # github.com/jmespath/go-jmespath v0.3.0 github.com/jmespath/go-jmespath -# github.com/longhorn/backupstore v0.0.0-20220913112826-5f5c95274f2a +# github.com/longhorn/backupstore v0.0.0-20230306022849-8d5e216c3b33 github.com/longhorn/backupstore github.com/longhorn/backupstore/fsops github.com/longhorn/backupstore/http @@ -115,6 +115,7 @@ github.com/willf/bitset # golang.org/x/net v0.0.0-20200202094626-16171245cfb2 golang.org/x/net/context golang.org/x/net/http/httpguts +golang.org/x/net/http/httpproxy golang.org/x/net/http2 golang.org/x/net/http2/hpack golang.org/x/net/idna