diff --git a/go.mod b/go.mod index 01f2c23fc..08e3f00da 100644 --- a/go.mod +++ b/go.mod @@ -6,14 +6,13 @@ require ( github.com/RoaringBitmap/roaring v0.4.18 github.com/golang/protobuf v1.3.3-0.20190920234318-1680a479a2cf github.com/google/uuid v1.3.0 - github.com/longhorn/backupstore v0.0.0-20220913112826-5f5c95274f2a - github.com/longhorn/go-iscsi-helper v0.0.0-20221109111031-ebff48f3632a // indirect - github.com/longhorn/longhorn-engine v1.3.2-0.20220929032851-7aac8ae9c8b4 + github.com/longhorn/backupstore v0.0.0-20221114044558-19f4902cd4fd + github.com/longhorn/longhorn-engine v1.3.1-0.20221115055520-8179e277e475 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 github.com/tinylib/msgp v1.1.1-0.20190612170807-0573788bc2a8 // indirect github.com/urfave/cli v1.22.1 - golang.org/x/net v0.0.0-20200202094626-16171245cfb2 + golang.org/x/net v0.0.0-20201021035429-f5854403a974 google.golang.org/grpc v1.21.0 gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2 ) diff --git a/go.sum b/go.sum index 38cda9c05..073338b88 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,7 @@ github.com/golang/protobuf v1.3.3-0.20190920234318-1680a479a2cf/go.mod h1:vzj43D github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -46,20 +47,19 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/longhorn/backupstore v0.0.0-20220913112826-5f5c95274f2a h1:f+mLqp3A5M7plw1pBgf8K1nvJxSU7mrGtU7bii+W5Bk= -github.com/longhorn/backupstore v0.0.0-20220913112826-5f5c95274f2a/go.mod h1:hvIVsrpjPey7KupirAh0WoPMg0ArWnE6fA5bI30X7AI= -github.com/longhorn/go-iscsi-helper v0.0.0-20220805034259-7b59e22574bb/go.mod h1:9z/y9glKmWEdV50tjlUPxFwi1goQfIrrsoZbnMyIZbY= +github.com/longhorn/backupstore v0.0.0-20221114044558-19f4902cd4fd h1:dRGV5NNnYWoxtYHhm48YVwvhioU2U0Dote+vbgcR8WQ= +github.com/longhorn/backupstore v0.0.0-20221114044558-19f4902cd4fd/go.mod h1:hvIVsrpjPey7KupirAh0WoPMg0ArWnE6fA5bI30X7AI= github.com/longhorn/go-iscsi-helper v0.0.0-20221109111031-ebff48f3632a h1:qinBI/kkZa7Lyy7jgSpox80YA/2lGtcogBClOMLmzVM= github.com/longhorn/go-iscsi-helper v0.0.0-20221109111031-ebff48f3632a/go.mod h1:9z/y9glKmWEdV50tjlUPxFwi1goQfIrrsoZbnMyIZbY= -github.com/longhorn/longhorn-engine v1.3.2-0.20220929032851-7aac8ae9c8b4 h1:hNdnSDe6vRZ3uIi6rZIsVITdXAR0G86gLIOJVQtICCo= -github.com/longhorn/longhorn-engine v1.3.2-0.20220929032851-7aac8ae9c8b4/go.mod h1:AC5smEVsxdJKI+joa0vSUPLd0Cp/xEQiqAsh3UM3UfY= +github.com/longhorn/longhorn-engine v1.3.1-0.20221115055520-8179e277e475 h1:eAJnEq04jlVPVHd8YU7PpOgiJw3GGUKb0RBKldSDlZ4= +github.com/longhorn/longhorn-engine v1.3.1-0.20221115055520-8179e277e475/go.mod h1:OONX3QZCxAv7hbtcz+mV+g7xAf862//tyxtsqwjtgwQ= github.com/longhorn/nsfilelock v0.0.0-20200723175406-fa7c83ad0003/go.mod h1:0CLeXlf59Lg6C0kjLSDf47ft73Dh37CwymYRKWwAn04= github.com/longhorn/sparse-tools v0.0.0-20220323120706-0bd9b4129826 h1:8IeuJT9y0xFGSfSl9dOG/L6dJOGtIrS3nYqgo3eYzao= github.com/longhorn/sparse-tools v0.0.0-20220323120706-0bd9b4129826/go.mod h1:BWM7yTPb1DulG18EE/Jy20LVIySzIYoZpiOYFtAGwZo= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.5-0.20181218000649-703b5e6b11ae/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/moby/moby v1.11.1/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= +github.com/moby/moby v20.10.20+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= @@ -102,22 +102,32 @@ github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -125,6 +135,12 @@ golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -139,4 +155,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/longhorn/backupstore/config.go b/vendor/github.com/longhorn/backupstore/config.go index 9e38d3af7..4e8ed941f 100644 --- a/vendor/github.com/longhorn/backupstore/config.go +++ b/vendor/github.com/longhorn/backupstore/config.go @@ -33,7 +33,7 @@ func getBackupConfigName(id string) string { return BACKUP_CONFIG_PREFIX + id + CFG_SUFFIX } -func loadConfigInBackupStore(filePath string, driver BackupStoreDriver, v interface{}) error { +func LoadConfigInBackupStore(filePath string, driver BackupStoreDriver, v interface{}) error { if !driver.FileExists(filePath) { return fmt.Errorf("cannot find %v in backupstore", filePath) } @@ -61,7 +61,7 @@ func loadConfigInBackupStore(filePath string, driver BackupStoreDriver, v interf return nil } -func saveConfigInBackupStore(filePath string, driver BackupStoreDriver, v interface{}) error { +func SaveConfigInBackupStore(filePath string, driver BackupStoreDriver, v interface{}) error { j, err := json.Marshal(v) if err != nil { return err @@ -84,6 +84,49 @@ func saveConfigInBackupStore(filePath string, driver BackupStoreDriver, v interf return nil } +func SaveLocalFileToBackupStore(localFilePath, backupStoreFilePath string, driver BackupStoreDriver) error { + log := log.WithFields(logrus.Fields{ + LogFieldReason: LogReasonStart, + LogFieldObject: LogObjectConfig, + LogFieldKind: driver.Kind(), + LogFieldFilepath: localFilePath, + LogFieldDestURL: backupStoreFilePath, + }) + log.Debug() + + if driver.FileExists(backupStoreFilePath) { + return fmt.Errorf("%v already exists", backupStoreFilePath) + } + + if err := driver.Upload(localFilePath, backupStoreFilePath); err != nil { + return err + } + + log.WithField(LogFieldReason, LogReasonComplete).Debug() + return nil +} + +func SaveBackupStoreToLocalFile(backupStoreFileURL, localFilePath string, driver BackupStoreDriver) error { + log := log.WithFields(logrus.Fields{ + LogFieldReason: LogReasonStart, + LogFieldObject: LogObjectConfig, + LogFieldKind: driver.Kind(), + LogFieldFilepath: localFilePath, + LogFieldSourceURL: backupStoreFileURL, + }) + log.Debug() + + if err := driver.Download(backupStoreFileURL, localFilePath); err != nil { + return err + } + + log = log.WithFields(logrus.Fields{ + LogFieldReason: LogReasonComplete, + }) + log.Debug() + return nil +} + func volumeExists(volumeName string, driver BackupStoreDriver) bool { volumeFile := getVolumeFilePath(volumeName) return driver.FileExists(volumeFile) @@ -175,7 +218,7 @@ func getVolumeNames(jobQueues *jobq.WorkerDispatcher, jobQueueTimeout time.Durat func loadVolume(volumeName string, driver BackupStoreDriver) (*Volume, error) { v := &Volume{} file := getVolumeFilePath(volumeName) - if err := loadConfigInBackupStore(file, driver, v); err != nil { + if err := LoadConfigInBackupStore(file, driver, v); err != nil { return nil, err } return v, nil @@ -183,7 +226,7 @@ func loadVolume(volumeName string, driver BackupStoreDriver) (*Volume, error) { func saveVolume(v *Volume, driver BackupStoreDriver) error { file := getVolumeFilePath(v.Name) - if err := saveConfigInBackupStore(file, driver, v); err != nil { + if err := SaveConfigInBackupStore(file, driver, v); err != nil { return err } return nil @@ -219,7 +262,7 @@ func backupExists(backupName, volumeName string, bsDriver BackupStoreDriver) boo func loadBackup(backupName, volumeName string, bsDriver BackupStoreDriver) (*Backup, error) { backup := &Backup{} - if err := loadConfigInBackupStore(getBackupConfigPath(backupName, volumeName), bsDriver, backup); err != nil { + if err := LoadConfigInBackupStore(getBackupConfigPath(backupName, volumeName), bsDriver, backup); err != nil { return nil, err } return backup, nil @@ -230,7 +273,7 @@ func saveBackup(backup *Backup, bsDriver BackupStoreDriver) error { return fmt.Errorf("missing volume specifier for backup: %v", backup.Name) } filePath := getBackupConfigPath(backup.Name, backup.VolumeName) - if err := saveConfigInBackupStore(filePath, bsDriver, backup); err != nil { + if err := SaveConfigInBackupStore(filePath, bsDriver, backup); err != nil { return err } return nil diff --git a/vendor/github.com/longhorn/backupstore/driver.go b/vendor/github.com/longhorn/backupstore/driver.go index d3079eaf7..c4aee9b3e 100644 --- a/vendor/github.com/longhorn/backupstore/driver.go +++ b/vendor/github.com/longhorn/backupstore/driver.go @@ -35,6 +35,10 @@ var ( log = logrus.WithFields(logrus.Fields{"pkg": "backupstore"}) ) +func GetLog() logrus.FieldLogger { + return log +} + func generateError(fields logrus.Fields, format string, v ...interface{}) error { return ErrorWithFields("backupstore", fields, format, v...) } diff --git a/vendor/github.com/longhorn/backupstore/lock.go b/vendor/github.com/longhorn/backupstore/lock.go index e5a708856..c51e69ac6 100644 --- a/vendor/github.com/longhorn/backupstore/lock.go +++ b/vendor/github.com/longhorn/backupstore/lock.go @@ -164,7 +164,7 @@ func (lock *FileLock) Unlock() error { func loadLock(volumeName string, name string, driver BackupStoreDriver) (*FileLock, error) { lock := &FileLock{} file := getLockFilePath(volumeName, name) - if err := loadConfigInBackupStore(file, driver, lock); err != nil { + if err := LoadConfigInBackupStore(file, driver, lock); err != nil { return nil, err } lock.serverTime = driver.FileTime(file) @@ -183,7 +183,7 @@ func removeLock(lock *FileLock) error { func saveLock(lock *FileLock) error { file := getLockFilePath(lock.volume, lock.Name) - if err := saveConfigInBackupStore(file, lock.driver, lock); err != nil { + if err := SaveConfigInBackupStore(file, lock.driver, lock); err != nil { return err } lock.serverTime = lock.driver.FileTime(file) diff --git a/vendor/github.com/longhorn/backupstore/logging/logging.go b/vendor/github.com/longhorn/backupstore/logging/logging.go index e57d9e96e..c040e98d9 100644 --- a/vendor/github.com/longhorn/backupstore/logging/logging.go +++ b/vendor/github.com/longhorn/backupstore/logging/logging.go @@ -18,6 +18,7 @@ const ( LogFieldLastSnapshot = "last_snapshot" LogEventBackupURL = "backup_url" LogFieldDestURL = "dest_url" + LogFieldSourceURL = "source_url" LogFieldKind = "kind" LogFieldFilepath = "filepath" diff --git a/vendor/github.com/longhorn/backupstore/nfs/nfs.go b/vendor/github.com/longhorn/backupstore/nfs/nfs.go index 15af7aea1..a277168cb 100644 --- a/vendor/github.com/longhorn/backupstore/nfs/nfs.go +++ b/vendor/github.com/longhorn/backupstore/nfs/nfs.go @@ -15,8 +15,9 @@ import ( ) var ( - log = logrus.WithFields(logrus.Fields{"pkg": "nfs"}) - MinorVersions = []string{"4.2", "4.1", "4.0"} + log = logrus.WithFields(logrus.Fields{"pkg": "nfs"}) + MinorVersions = []string{"4.2", "4.1", "4.0"} + defaultTimeout = 5 * time.Second ) type BackupStoreDriver struct { @@ -64,7 +65,7 @@ func initFunc(destURL string) (backupstore.BackupStoreDriver, error) { b.serverPath = u.Host + u.Path b.mountDir = filepath.Join(MountDir, strings.TrimRight(strings.Replace(u.Host, ".", "_", -1), ":"), u.Path) - if _, err = util.ExecuteWithCustomTimeout("mkdir", []string{"-m", "700", "-p", b.mountDir}, 3*time.Second); err != nil { + if _, err = util.ExecuteWithCustomTimeout("mkdir", []string{"-m", "700", "-p", b.mountDir}, defaultTimeout); err != nil { return nil, fmt.Errorf("Cannot create mount directory %v for NFS server: %v", b.mountDir, err) } @@ -89,7 +90,7 @@ func (b *BackupStoreDriver) mount() (err error) { for _, version := range MinorVersions { log.Debugf("attempting mount for nfs path %v with nfsvers %v", b.serverPath, version) - _, err = util.Execute("mount", []string{"-t", "nfs4", "-o", fmt.Sprintf("nfsvers=%v", version), "-o", "actimeo=1", b.serverPath, b.mountDir}) + _, err = util.ExecuteWithCustomTimeout("mount", []string{"-t", "nfs4", "-o", fmt.Sprintf("nfsvers=%v", version), "-o", "actimeo=1", b.serverPath, b.mountDir}, defaultTimeout) if err == nil { return nil } diff --git a/vendor/github.com/longhorn/backupstore/s3/s3.go b/vendor/github.com/longhorn/backupstore/s3/s3.go index aef885092..72596f1f9 100644 --- a/vendor/github.com/longhorn/backupstore/s3/s3.go +++ b/vendor/github.com/longhorn/backupstore/s3/s3.go @@ -195,11 +195,17 @@ func (s *BackupStoreDriver) Download(src, dst string) error { if _, err := os.Stat(dst); err != nil { os.Remove(dst) } + + if err := os.MkdirAll(filepath.Dir(dst), os.ModeDir|0700); err != nil { + return err + } + f, err := os.Create(dst) if err != nil { return err } defer f.Close() + path := s.updatePath(src) rc, err := s.service.GetObject(path) if err != nil { diff --git a/vendor/github.com/longhorn/backupstore/util/util.go b/vendor/github.com/longhorn/backupstore/util/util.go index be23b864b..b1b0e623f 100644 --- a/vendor/github.com/longhorn/backupstore/util/util.go +++ b/vendor/github.com/longhorn/backupstore/util/util.go @@ -4,11 +4,13 @@ import ( "bytes" "compress/gzip" "context" + "crypto/sha256" "crypto/sha512" "encoding/hex" "fmt" "io" "io/ioutil" + "os" "os/exec" "regexp" "strings" @@ -41,6 +43,21 @@ func GetChecksum(data []byte) string { return checksum } +func GetFileChecksum(filePath string) (string, error) { + f, err := os.Open(filePath) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + + return hex.EncodeToString(h.Sum(nil)), nil +} + func CompressData(data []byte) (io.ReadSeeker, error) { var b bytes.Buffer w := gzip.NewWriter(&b) diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/backingfile/backingfile.go b/vendor/github.com/longhorn/longhorn-engine/pkg/backingfile/backingfile.go index eb939c152..ab4e7eff2 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/backingfile/backingfile.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/backingfile/backingfile.go @@ -16,8 +16,6 @@ import ( "github.com/longhorn/longhorn-engine/pkg/util" ) -const defaultSectorSize = 512 - type BackingFile struct { Size int64 SectorSize int64 @@ -102,14 +100,14 @@ func OpenBackingFile(file string) (*BackingFile, error) { if err != nil { return nil, err } - if size%defaultSectorSize != 0 { - return nil, fmt.Errorf("the backing file size %v should be a multiple of 512 bytes since Longhorn uses directIO by default", size) + if size%util.BackingImageSectorSize != 0 { + return nil, fmt.Errorf("the backing file size %v should be a multiple of %v bytes since Longhorn uses directIO by default", size, util.BackingImageSectorSize) } return &BackingFile{ Path: file, Disk: f, Size: size, - SectorSize: defaultSectorSize, + SectorSize: util.BackingImageSectorSize, }, nil } diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/controller/client/controller_client.go b/vendor/github.com/longhorn/longhorn-engine/pkg/controller/client/controller_client.go index 0a7252fc3..00f0a031b 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/controller/client/controller_client.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/controller/client/controller_client.go @@ -5,14 +5,15 @@ import ( "time" "github.com/golang/protobuf/ptypes/empty" + "github.com/pkg/errors" "golang.org/x/net/context" "google.golang.org/grpc" + healthpb "google.golang.org/grpc/health/grpc_health_v1" "github.com/longhorn/longhorn-engine/pkg/meta" "github.com/longhorn/longhorn-engine/pkg/types" "github.com/longhorn/longhorn-engine/pkg/util" "github.com/longhorn/longhorn-engine/proto/ptypes" - healthpb "google.golang.org/grpc/health/grpc_health_v1" ) type ControllerServiceContext struct { @@ -44,7 +45,7 @@ func NewControllerClient(address string) (*ControllerClient, error) { getControllerServiceContext := func(serviceUrl string) (ControllerServiceContext, error) { connection, err := grpc.Dial(serviceUrl, grpc.WithInsecure()) if err != nil { - return ControllerServiceContext{}, fmt.Errorf("cannot connect to ControllerService %v: %v", serviceUrl, err) + return ControllerServiceContext{}, errors.Wrapf(err, "cannot connect to ControllerService %v", serviceUrl) } return ControllerServiceContext{ @@ -118,7 +119,7 @@ func (c *ControllerClient) VolumeGet() (*types.VolumeInfo, error) { volume, err := controllerServiceClient.VolumeGet(ctx, &empty.Empty{}) if err != nil { - return nil, fmt.Errorf("failed to get volume %v: %v", c.serviceURL, err) + return nil, errors.Wrapf(err, "failed to get volume %v", c.serviceURL) } return GetVolumeInfo(volume), nil @@ -134,7 +135,7 @@ func (c *ControllerClient) VolumeStart(size, currentSize int64, replicas ...stri Size: size, CurrentSize: currentSize, }); err != nil { - return fmt.Errorf("failed to start volume %v: %v", c.serviceURL, err) + return errors.Wrapf(err, "failed to start volume %v", c.serviceURL) } return nil @@ -150,7 +151,7 @@ func (c *ControllerClient) VolumeSnapshot(name string, labels map[string]string) Labels: labels, }) if err != nil { - return "", fmt.Errorf("failed to create snapshot %v for volume %v: %v", name, c.serviceURL, err) + return "", errors.Wrapf(err, "failed to create snapshot %v for volume %v", name, c.serviceURL) } return reply.Name, nil @@ -164,7 +165,7 @@ func (c *ControllerClient) VolumeRevert(snapshot string) error { if _, err := controllerServiceClient.VolumeRevert(ctx, &ptypes.VolumeRevertRequest{ Name: snapshot, }); err != nil { - return fmt.Errorf("failed to revert to snapshot %v for volume %v: %v", snapshot, c.serviceURL, err) + return errors.Wrapf(err, "failed to revert to snapshot %v for volume %v", snapshot, c.serviceURL) } return nil @@ -178,7 +179,7 @@ func (c *ControllerClient) VolumeExpand(size int64) error { if _, err := controllerServiceClient.VolumeExpand(ctx, &ptypes.VolumeExpandRequest{ Size: size, }); err != nil { - return fmt.Errorf("failed to expand to size %v for volume %v: %v", size, c.serviceURL, err) + return errors.Wrapf(err, "failed to expand to size %v for volume %v", size, c.serviceURL) } return nil @@ -192,7 +193,7 @@ func (c *ControllerClient) VolumeFrontendStart(frontend string) error { if _, err := controllerServiceClient.VolumeFrontendStart(ctx, &ptypes.VolumeFrontendStartRequest{ Frontend: frontend, }); err != nil { - return fmt.Errorf("failed to start frontend %v for volume %v: %v", frontend, c.serviceURL, err) + return errors.Wrapf(err, "failed to start frontend %v for volume %v", frontend, c.serviceURL) } return nil @@ -204,7 +205,7 @@ func (c *ControllerClient) VolumeFrontendShutdown() error { defer cancel() if _, err := controllerServiceClient.VolumeFrontendShutdown(ctx, &empty.Empty{}); err != nil { - return fmt.Errorf("failed to shutdown frontend for volume %v: %v", c.serviceURL, err) + return errors.Wrapf(err, "failed to shutdown frontend for volume %v", c.serviceURL) } return nil @@ -217,7 +218,7 @@ func (c *ControllerClient) ReplicaList() ([]*types.ControllerReplicaInfo, error) reply, err := controllerServiceClient.ReplicaList(ctx, &empty.Empty{}) if err != nil { - return nil, fmt.Errorf("failed to list replicas for volume %v: %v", c.serviceURL, err) + return nil, errors.Wrapf(err, "failed to list replicas for volume %v", c.serviceURL) } replicas := []*types.ControllerReplicaInfo{} @@ -237,7 +238,7 @@ func (c *ControllerClient) ReplicaGet(address string) (*types.ControllerReplicaI Address: address, }) if err != nil { - return nil, fmt.Errorf("failed to get replica %v for volume %v: %v", address, c.serviceURL, err) + return nil, errors.Wrapf(err, "failed to get replica %v for volume %v", address, c.serviceURL) } return GetControllerReplicaInfo(cr), nil @@ -254,7 +255,7 @@ func (c *ControllerClient) ReplicaCreate(address string, snapshotRequired bool, Mode: ptypes.ReplicaModeToGRPCReplicaMode(mode), }) if err != nil { - return nil, fmt.Errorf("failed to create replica %v for volume %v: %v", address, c.serviceURL, err) + return nil, errors.Wrapf(err, "failed to create replica %v for volume %v", address, c.serviceURL) } return GetControllerReplicaInfo(cr), nil @@ -268,7 +269,7 @@ func (c *ControllerClient) ReplicaDelete(address string) error { if _, err := controllerServiceClient.ReplicaDelete(ctx, &ptypes.ReplicaAddress{ Address: address, }); err != nil { - return fmt.Errorf("failed to delete replica %v for volume %v: %v", address, c.serviceURL, err) + return errors.Wrapf(err, "failed to delete replica %v for volume %v", address, c.serviceURL) } return nil @@ -281,7 +282,7 @@ func (c *ControllerClient) ReplicaUpdate(replica *types.ControllerReplicaInfo) ( cr, err := controllerServiceClient.ReplicaUpdate(ctx, GetControllerReplica(replica)) if err != nil { - return nil, fmt.Errorf("failed to update replica %v for volume %v: %v", replica.Address, c.serviceURL, err) + return nil, errors.Wrapf(err, "failed to update replica %v for volume %v", replica.Address, c.serviceURL) } return GetControllerReplicaInfo(cr), nil @@ -296,7 +297,7 @@ func (c *ControllerClient) ReplicaPrepareRebuild(address string) ([]types.SyncFi Address: address, }) if err != nil { - return nil, fmt.Errorf("failed to prepare rebuilding replica %v for volume %v: %v", address, c.serviceURL, err) + return nil, errors.Wrapf(err, "failed to prepare rebuilding replica %v for volume %v", address, c.serviceURL) } return GetSyncFileInfoList(reply.SyncFileInfoList), nil @@ -310,7 +311,7 @@ func (c *ControllerClient) ReplicaVerifyRebuild(address string) error { if _, err := controllerServiceClient.ReplicaVerifyRebuild(ctx, &ptypes.ReplicaAddress{ Address: address, }); err != nil { - return fmt.Errorf("failed to verify rebuilt replica %v for volume %v: %v", address, c.serviceURL, err) + return errors.Wrapf(err, "failed to verify rebuilt replica %v for volume %v", address, c.serviceURL) } return nil @@ -324,7 +325,7 @@ func (c *ControllerClient) JournalList(limit int) error { if _, err := controllerServiceClient.JournalList(ctx, &ptypes.JournalListRequest{ Limit: int64(limit), }); err != nil { - return fmt.Errorf("failed to list journal for volume %v: %v", c.serviceURL, err) + return errors.Wrapf(err, "failed to list journal for volume %v", c.serviceURL) } return nil @@ -337,7 +338,7 @@ func (c *ControllerClient) VersionDetailGet() (*meta.VersionOutput, error) { reply, err := controllerServiceClient.VersionDetailGet(ctx, &empty.Empty{}) if err != nil { - return nil, fmt.Errorf("failed to get version detail: %v", err) + return nil, errors.Wrap(err, "failed to get version detail") } return &meta.VersionOutput{ @@ -357,7 +358,7 @@ func (c *ControllerClient) VersionDetailGet() (*meta.VersionOutput, error) { func (c *ControllerClient) Check() error { conn, err := grpc.Dial(c.serviceURL, grpc.WithInsecure()) if err != nil { - return fmt.Errorf("cannot connect to ControllerService %v: %v", c.serviceURL, err) + return errors.Wrapf(err, "cannot connect to ControllerService %v", c.serviceURL) } defer conn.Close() // TODO: JM we can reuse the controller service context connection for the health requests @@ -370,7 +371,7 @@ func (c *ControllerClient) Check() error { Service: "", }) if err != nil { - return fmt.Errorf("failed to check health for gRPC controller server %v: %v", c.serviceURL, err) + return errors.Wrapf(err, "failed to check health for gRPC controller server %v", c.serviceURL) } if reply.Status != healthpb.HealthCheckResponse_SERVING { diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/meta/version.go b/vendor/github.com/longhorn/longhorn-engine/pkg/meta/version.go index 84b16448e..6134ce11b 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/meta/version.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/meta/version.go @@ -2,7 +2,7 @@ package meta const ( // CLIAPIVersion used to communicate with user e.g. longhorn-manager - CLIAPIVersion = 6 + CLIAPIVersion = 7 CLIAPIMinVersion = 3 // ControllerAPIVersion used to communicate with instance-manager diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/qcow/libqcow.go b/vendor/github.com/longhorn/longhorn-engine/pkg/qcow/libqcow.go index e85ee8e96..2d60db9da 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/qcow/libqcow.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/qcow/libqcow.go @@ -20,7 +20,7 @@ func toError(e *C.libqcow_error_t) error { buf := [1024]C.char{} defer C.libqcow_error_free(&e) if C.libqcow_error_sprint(e, &buf[0], 1023) < 0 { - return fmt.Errorf("Unknown error: %v", e) + return fmt.Errorf("unknown error: %v", e) } return errors.New(C.GoString(&buf[0])) } @@ -44,7 +44,7 @@ func Open(path string) (*Qcow, error) { } func (q *Qcow) WriteAt(buf []byte, off int64) (int, error) { - return 0, errors.New("Unsupported operation") + return 0, errors.New("unsupported operation") } func (q *Qcow) ReadAt(buf []byte, off int64) (int, error) { diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/backup.go b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/backup.go index 20a4ce306..fed662ac4 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/backup.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/backup.go @@ -5,6 +5,7 @@ import ( "os" "sync" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/longhorn/longhorn-engine/pkg/backingfile" @@ -206,12 +207,12 @@ func (rb *BackupStatus) OpenSnapshot(snapID, volumeID string) error { } if rb.volumeID != "" { - return fmt.Errorf("Volume %s and snapshot %s are already open, close first", rb.volumeID, rb.SnapshotID) + return fmt.Errorf("volume %s and snapshot %s are already open, close first", rb.volumeID, rb.SnapshotID) } dir, err := os.Getwd() if err != nil { - return fmt.Errorf("Cannot get working directory: %v", err) + return errors.Wrap(err, "cannot get working directory") } r, err := NewReadOnly(dir, id, rb.backingFile) if err != nil { @@ -227,7 +228,7 @@ func (rb *BackupStatus) OpenSnapshot(snapID, volumeID string) error { func (rb *BackupStatus) assertOpen(id, volumeID string) error { if rb.volumeID != volumeID || rb.SnapshotID != id { - return fmt.Errorf("Invalid state volume [%s] and snapshot [%s] are open, not volume [%s], snapshot [%s]", rb.volumeID, rb.SnapshotID, volumeID, id) + return fmt.Errorf("invalid state volume [%s] and snapshot [%s] are open, not volume [%s], snapshot [%s]", rb.volumeID, rb.SnapshotID, volumeID, id) } return nil } @@ -288,12 +289,12 @@ func (rb *BackupStatus) CompareSnapshot(snapID, compareSnapID, volumeID string) from := rb.findIndex(id) if from < 0 { - return nil, fmt.Errorf("Failed to find snapshot %s in chain", id) + return nil, fmt.Errorf("failed to find snapshot %s in chain", id) } to := rb.findIndex(compareID) if to < 0 { - return nil, fmt.Errorf("Failed to find snapshot %s in chain", compareID) + return nil, fmt.Errorf("failed to find snapshot %s in chain", compareID) } mappings := &backupstore.Mappings{ diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/client/client.go b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/client/client.go index eb2790a0f..a788bacdf 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/client/client.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/client/client.go @@ -7,6 +7,7 @@ import ( "time" "github.com/golang/protobuf/ptypes/empty" + "github.com/pkg/errors" "golang.org/x/net/context" "google.golang.org/grpc" @@ -192,7 +193,7 @@ func (c *ReplicaClient) GetReplica() (*types.ReplicaInfo, error) { resp, err := replicaServiceClient.ReplicaGet(ctx, &empty.Empty{}) if err != nil { - return nil, fmt.Errorf("failed to get replica %v: %v", c.replicaServiceURL, err) + return nil, errors.Wrapf(err, "failed to get replica %v", c.replicaServiceURL) } return GetReplicaInfo(resp.Replica), nil @@ -207,7 +208,7 @@ func (c *ReplicaClient) OpenReplica() error { defer cancel() if _, err := replicaServiceClient.ReplicaOpen(ctx, &empty.Empty{}); err != nil { - return fmt.Errorf("failed to open replica %v: %v", c.replicaServiceURL, err) + return errors.Wrapf(err, "failed to open replica %v", c.replicaServiceURL) } return nil @@ -222,7 +223,7 @@ func (c *ReplicaClient) CloseReplica() error { defer cancel() if _, err := replicaServiceClient.ReplicaClose(ctx, &empty.Empty{}); err != nil { - return fmt.Errorf("failed to close replica %v: %v", c.replicaServiceURL, err) + return errors.Wrapf(err, "failed to close replica %v", c.replicaServiceURL) } return nil @@ -238,7 +239,7 @@ func (c *ReplicaClient) ReloadReplica() (*types.ReplicaInfo, error) { resp, err := replicaServiceClient.ReplicaReload(ctx, &empty.Empty{}) if err != nil { - return nil, fmt.Errorf("failed to reload replica %v: %v", c.replicaServiceURL, err) + return nil, errors.Wrapf(err, "failed to reload replica %v", c.replicaServiceURL) } return GetReplicaInfo(resp.Replica), nil @@ -274,7 +275,7 @@ func (c *ReplicaClient) Revert(name, created string) error { Name: name, Created: created, }); err != nil { - return fmt.Errorf("failed to revert replica %v: %v", c.replicaServiceURL, err) + return errors.Wrapf(err, "failed to revert replica %v", c.replicaServiceURL) } return nil @@ -292,7 +293,7 @@ func (c *ReplicaClient) RemoveDisk(disk string, force bool) error { Name: disk, Force: force, }); err != nil { - return fmt.Errorf("failed to remove disk %v for replica %v: %v", disk, c.replicaServiceURL, err) + return errors.Wrapf(err, "failed to remove disk %v for replica %v", disk, c.replicaServiceURL) } return nil @@ -310,7 +311,7 @@ func (c *ReplicaClient) ReplaceDisk(target, source string) error { Target: target, Source: source, }); err != nil { - return fmt.Errorf("failed to replace disk %v with %v for replica %v: %v", target, source, c.replicaServiceURL, err) + return errors.Wrapf(err, "failed to replace disk %v with %v for replica %v", target, source, c.replicaServiceURL) } return nil @@ -329,7 +330,7 @@ func (c *ReplicaClient) PrepareRemoveDisk(disk string) ([]*types.PrepareRemoveAc }) if err != nil { - return nil, fmt.Errorf("failed to prepare removing disk %v for replica %v: %v", disk, c.replicaServiceURL, err) + return nil, errors.Wrapf(err, "failed to prepare removing disk %v for replica %v", disk, c.replicaServiceURL) } operations := []*types.PrepareRemoveAction{} @@ -355,7 +356,7 @@ func (c *ReplicaClient) MarkDiskAsRemoved(disk string) error { if _, err := replicaServiceClient.DiskMarkAsRemoved(ctx, &ptypes.DiskMarkAsRemovedRequest{ Name: disk, }); err != nil { - return fmt.Errorf("failed to mark disk %v as removed for replica %v: %v", disk, c.replicaServiceURL, err) + return errors.Wrapf(err, "failed to mark disk %v as removed for replica %v", disk, c.replicaServiceURL) } return nil @@ -372,7 +373,7 @@ func (c *ReplicaClient) SetRebuilding(rebuilding bool) error { if _, err := replicaServiceClient.RebuildingSet(ctx, &ptypes.RebuildingSetRequest{ Rebuilding: rebuilding, }); err != nil { - return fmt.Errorf("failed to set rebuilding to %v for replica %v: %v", rebuilding, c.replicaServiceURL, err) + return errors.Wrapf(err, "failed to set rebuilding to %v for replica %v", rebuilding, c.replicaServiceURL) } return nil @@ -389,7 +390,7 @@ func (c *ReplicaClient) RemoveFile(file string) error { if _, err := syncAgentServiceClient.FileRemove(ctx, &ptypes.FileRemoveRequest{ FileName: file, }); err != nil { - return fmt.Errorf("failed to remove file %v: %v", file, err) + return errors.Wrapf(err, "failed to remove file %v", file) } return nil @@ -407,7 +408,7 @@ func (c *ReplicaClient) RenameFile(oldFileName, newFileName string) error { OldFileName: oldFileName, NewFileName: newFileName, }); err != nil { - return fmt.Errorf("failed to rename or replace old file %v with new file %v: %v", oldFileName, newFileName, err) + return errors.Wrapf(err, "failed to rename or replace old file %v with new file %v", oldFileName, newFileName) } return nil @@ -426,7 +427,7 @@ func (c *ReplicaClient) SendFile(from, host string, port int32) error { Host: host, Port: port, }); err != nil { - return fmt.Errorf("failed to send file %v to %v:%v: %v", from, host, port, err) + return errors.Wrapf(err, "failed to send file %v to %v:%v", from, host, port) } return nil @@ -447,7 +448,7 @@ func (c *ReplicaClient) ExportVolume(snapshotName, host string, port int32, expo Port: port, ExportBackingImageIfExist: exportBackingImageIfExist, }); err != nil { - return fmt.Errorf("failed to export snapshot %v to %v:%v: %v", snapshotName, host, port, err) + return errors.Wrapf(err, "failed to export snapshot %v to %v:%v", snapshotName, host, port) } return nil } @@ -464,7 +465,7 @@ func (c *ReplicaClient) LaunchReceiver(toFilePath string) (string, int32, error) ToFileName: toFilePath, }) if err != nil { - return "", 0, fmt.Errorf("failed to launch receiver for %v: %v", toFilePath, err) + return "", 0, errors.Wrapf(err, "failed to launch receiver for %v", toFilePath) } return c.host, reply.Port, nil @@ -483,7 +484,7 @@ func (c *ReplicaClient) SyncFiles(fromAddress string, list []types.SyncFileInfo) ToHost: c.host, SyncFileInfoList: syncFileInfoListToSyncAgentGRPCFormat(list), }); err != nil { - return fmt.Errorf("failed to sync files %+v from %v: %v", list, fromAddress, err) + return errors.Wrapf(err, "failed to sync files %+v from %v", list, fromAddress) } return nil @@ -508,7 +509,7 @@ func (c *ReplicaClient) CreateBackup(backupName, snapshot, dest, volume, backing BackupName: backupName, }) if err != nil { - return nil, fmt.Errorf("failed to create backup to %v for volume %v: %v", dest, volume, err) + return nil, errors.Wrapf(err, "failed to create backup to %v for volume %v", dest, volume) } return resp, nil @@ -544,7 +545,7 @@ func (c *ReplicaClient) RmBackup(backup string) error { if _, err := syncAgentServiceClient.BackupRemove(ctx, &ptypes.BackupRemoveRequest{ Backup: backup, }); err != nil { - return fmt.Errorf("failed to remove backup %v: %v", backup, err) + return errors.Wrapf(err, "failed to remove backup %v", backup) } return nil @@ -563,7 +564,7 @@ func (c *ReplicaClient) RestoreBackup(backup, snapshotDiskName string, credentia SnapshotDiskName: snapshotDiskName, Credential: credential, }); err != nil { - return fmt.Errorf("failed to restore backup data %v to snapshot file %v: %v", backup, snapshotDiskName, err) + return errors.Wrapf(err, "failed to restore backup data %v to snapshot file %v", backup, snapshotDiskName) } return nil @@ -578,7 +579,7 @@ func (c *ReplicaClient) Reset() error { defer cancel() if _, err := syncAgentServiceClient.Reset(ctx, &empty.Empty{}); err != nil { - return fmt.Errorf("failed to cleanup restore info in Sync Agent Server: %v", err) + return errors.Wrap(err, "failed to cleanup restore info in Sync Agent Server") } return nil @@ -594,7 +595,7 @@ func (c *ReplicaClient) RestoreStatus() (*ptypes.RestoreStatusResponse, error) { resp, err := syncAgentServiceClient.RestoreStatus(ctx, &empty.Empty{}) if err != nil { - return nil, fmt.Errorf("failed to get restore status: %v", err) + return nil, errors.Wrap(err, "failed to get restore status") } return resp, nil @@ -609,7 +610,7 @@ func (c *ReplicaClient) SnapshotPurge() error { defer cancel() if _, err := syncAgentServiceClient.SnapshotPurge(ctx, &empty.Empty{}); err != nil { - return fmt.Errorf("failed to start snapshot purge: %v", err) + return errors.Wrap(err, "failed to start snapshot purge") } return nil @@ -625,7 +626,7 @@ func (c *ReplicaClient) SnapshotPurgeStatus() (*ptypes.SnapshotPurgeStatusRespon status, err := syncAgentServiceClient.SnapshotPurgeStatus(ctx, &empty.Empty{}) if err != nil { - return nil, fmt.Errorf("failed to get snapshot purge status: %v", err) + return nil, errors.Wrap(err, "failed to get snapshot purge status") } return status, nil @@ -641,7 +642,7 @@ func (c *ReplicaClient) ReplicaRebuildStatus() (*ptypes.ReplicaRebuildStatusResp status, err := syncAgentServiceClient.ReplicaRebuildStatus(ctx, &empty.Empty{}) if err != nil { - return nil, fmt.Errorf("failed to get replica rebuild status: %v", err) + return nil, errors.Wrap(err, "failed to get replica rebuild status") } return status, nil @@ -661,7 +662,7 @@ func (c *ReplicaClient) CloneSnapshot(fromAddress, snapshotFileName string, expo SnapshotFileName: snapshotFileName, ExportBackingImageIfExist: exportBackingImageIfExist, }); err != nil { - return fmt.Errorf("failed to clone snapshot %v from replica %v to host %v: %v", snapshotFileName, fromAddress, c.host, err) + return errors.Wrapf(err, "failed to clone snapshot %v from replica %v to host %v", snapshotFileName, fromAddress, c.host) } return nil @@ -677,7 +678,7 @@ func (c *ReplicaClient) SnapshotCloneStatus() (*ptypes.SnapshotCloneStatusRespon status, err := syncAgentServiceClient.SnapshotCloneStatus(ctx, &empty.Empty{}) if err != nil { - return nil, fmt.Errorf("failed to get snapshot clone status: %v", err) + return nil, errors.Wrap(err, "failed to get snapshot clone status") } return status, nil } diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/diff_disk.go b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/diff_disk.go index 1168043cb..671d37bd0 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/diff_disk.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/diff_disk.go @@ -114,7 +114,7 @@ func (d *diffDisk) readModifyWrite(buf []byte, offset int64) (int, error) { func (d *diffDisk) fullWriteAt(buf []byte, offset int64) (int, error) { if int64(len(buf))%d.sectorSize != 0 || offset%d.sectorSize != 0 { - return 0, fmt.Errorf("Write len(%d), offset %d not a multiple of %d", len(buf), offset, d.sectorSize) + return 0, fmt.Errorf("write len(%d), offset %d not a multiple of %d", len(buf), offset, d.sectorSize) } target := byte(len(d.files) - 1) @@ -172,7 +172,7 @@ func (d *diffDisk) ReadAt(buf []byte, offset int64) (int, error) { func (d *diffDisk) fullReadAt(buf []byte, offset int64) (int, error) { if int64(len(buf))%d.sectorSize != 0 || offset%d.sectorSize != 0 { - return 0, fmt.Errorf("Read not a multiple of %d", d.sectorSize) + return 0, fmt.Errorf("read not a multiple of %d", d.sectorSize) } if len(buf) == 0 { diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/replica.go b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/replica.go index 74535d416..adac9464e 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/replica.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/replica.go @@ -30,7 +30,6 @@ const ( metadataSuffix = ".meta" imgSuffix = ".img" volumeMetaData = "volume.meta" - defaultSectorSize = 4096 headPrefix = "volume-head-" headSuffix = ".img" headName = headPrefix + "%03d" + headSuffix @@ -159,12 +158,12 @@ func New(size, sectorSize int64, dir string, backingFile *backingfile.BackingFil func NewReadOnly(dir, head string, backingFile *backingfile.BackingFile) (*Replica, error) { // size and sectorSize don't matter because they will be read from metadata - return construct(true, 0, 512, dir, head, backingFile, false) + return construct(true, 0, util.ReplicaSectorSize, dir, head, backingFile, false) } func construct(readonly bool, size, sectorSize int64, dir, head string, backingFile *backingfile.BackingFile, disableRevCounter bool) (*Replica, error) { if size%sectorSize != 0 { - return nil, fmt.Errorf("Size %d not a multiple of sector size %d", size, sectorSize) + return nil, fmt.Errorf("size %d not a multiple of sector size %d", size, sectorSize) } if err := os.Mkdir(dir, 0700); err != nil && !os.IsExist(err) { @@ -181,7 +180,7 @@ func construct(readonly bool, size, sectorSize int64, dir, head string, backingF } r.info.Size = size r.info.SectorSize = sectorSize - r.volume.sectorSize = defaultSectorSize + r.volume.sectorSize = util.VolumeSectorSize // Scan all the disks to build the disk map exists, err := r.readMetadata() @@ -205,7 +204,7 @@ func construct(readonly bool, size, sectorSize int64, dir, head string, backingF // Reference r.info.Size because it may have changed from reading // metadata locationSize := r.info.Size / r.volume.sectorSize - if size%defaultSectorSize != 0 { + if size%util.VolumeSectorSize != 0 { locationSize++ } r.volume.location = make([]byte, locationSize) @@ -365,7 +364,7 @@ func (r *Replica) RemoveDiffDisk(name string, force bool) error { defer r.Unlock() if name == r.info.Head { - return fmt.Errorf("Can not delete the active differencing disk") + return fmt.Errorf("cannot delete the active differencing disk") } if err := r.removeDiskNode(name, force); err != nil { @@ -396,11 +395,11 @@ func (r *Replica) MarkDiskAsRemoved(name string) error { } if disk == r.info.Head { - return fmt.Errorf("Can not mark the active differencing disk as removed") + return fmt.Errorf("cannot mark the active differencing disk as removed") } if err := r.markDiskAsRemoved(disk); err != nil { - return fmt.Errorf("Failed to mark disk %v as removed: %v", disk, err) + return errors.Wrapf(err, "failed to mark disk %v as removed", disk) } return nil @@ -408,18 +407,18 @@ func (r *Replica) MarkDiskAsRemoved(name string) error { func (r *Replica) hardlinkDisk(target, source string) error { if _, err := os.Stat(r.diskPath(source)); err != nil { - return fmt.Errorf("Cannot find source of replacing: %v", source) + return fmt.Errorf("cannot find source of replacing: %v", source) } if _, err := os.Stat(r.diskPath(target)); err == nil { logrus.Infof("Old file %s exists, deleting", target) if err := os.Remove(r.diskPath(target)); err != nil { - return fmt.Errorf("Failed to remove %s: %v", target, err) + return errors.Wrapf(err, "failed to remove %s", target) } } if err := os.Link(r.diskPath(source), r.diskPath(target)); err != nil { - return fmt.Errorf("Failed to link %s to %s", source, target) + return fmt.Errorf("failed to link %s to %s", source, target) } return nil } @@ -429,7 +428,7 @@ func (r *Replica) ReplaceDisk(target, source string) error { defer r.Unlock() if target == r.info.Head { - return fmt.Errorf("Can not replace the active differencing disk") + return fmt.Errorf("cannot replace the active differencing disk") } if err := r.hardlinkDisk(target, source); err != nil { @@ -478,7 +477,7 @@ func (r *Replica) removeDiskNode(name string, force bool) error { // If snapshot has more than one child, we cannot really delete it if len(children) > 1 { if !force { - return fmt.Errorf("Cannot remove snapshot %v with %v children", + return fmt.Errorf("cannot remove snapshot %v with %v children", name, len(children)) } logrus.Warnf("force delete disk %v with multiple children. Randomly choose a child to inherit", name) @@ -530,11 +529,11 @@ func (r *Replica) PrepareRemoveDisk(name string) ([]PrepareRemoveAction, error) } if disk == r.info.Head { - return nil, fmt.Errorf("Can not delete the active differencing disk") + return nil, fmt.Errorf("cannot delete the active differencing disk") } if !data.Removed { - return nil, fmt.Errorf("Disk %v hasn't been marked as removed", disk) + return nil, fmt.Errorf("disk %v hasn't been marked as removed", disk) } actions, err := r.processPrepareRemoveDisks(disk) @@ -548,7 +547,7 @@ func (r *Replica) processPrepareRemoveDisks(disk string) ([]PrepareRemoveAction, actions := []PrepareRemoveAction{} if _, exists := r.diskData[disk]; !exists { - return nil, fmt.Errorf("Wrong disk %v doesn't exist", disk) + return nil, fmt.Errorf("wrong disk %v doesn't exist", disk) } children := r.diskChildrenMap[disk] @@ -610,7 +609,7 @@ func (r *Replica) DisplayChain() ([]string, error) { for cur != "" { disk, ok := r.diskData[cur] if !ok { - return nil, fmt.Errorf("Failed to find metadata for %s", cur) + return nil, fmt.Errorf("failed to find metadata for %s", cur) } if !disk.Removed { result = append(result, cur) @@ -631,7 +630,7 @@ func (r *Replica) Chain() ([]string, error) { for cur != "" { result = append(result, cur) if _, ok := r.diskData[cur]; !ok { - return nil, fmt.Errorf("Failed to find metadata for %s", cur) + return nil, fmt.Errorf("failed to find metadata for %s", cur) } cur = r.diskData[cur].Parent } @@ -709,7 +708,7 @@ func (r *Replica) nextFile(parsePattern *regexp.Regexp, pattern, parent string) matches := parsePattern.FindStringSubmatch(parent) if matches == nil { - return "", fmt.Errorf("Invalid name %s does not match pattern: %v", parent, parsePattern) + return "", fmt.Errorf("invalid name %s does not match pattern: %v", parent, parsePattern) } index, _ := strconv.Atoi(matches[1]) @@ -793,13 +792,13 @@ func (r *Replica) linkDisk(oldname, newname string) error { func (r *Replica) markDiskAsRemoved(name string) error { disk, ok := r.diskData[name] if !ok { - return fmt.Errorf("Cannot find disk %v", name) + return fmt.Errorf("cannot find disk %v", name) } if stat, err := os.Stat(r.diskPath(name)); err != nil || stat.IsDir() { - return fmt.Errorf("Cannot find disk file %v", name) + return fmt.Errorf("cannot find disk file %v", name) } if stat, err := os.Stat(r.diskPath(name + metadataSuffix)); err != nil || stat.IsDir() { - return fmt.Errorf("Cannot find disk metafile %v", name+metadataSuffix) + return fmt.Errorf("cannot find disk metafile %v", name+metadataSuffix) } disk.Removed = true r.diskData[name] = disk @@ -860,11 +859,11 @@ func (r *Replica) createDisk(name string, userCreated bool, created string, labe log := logrus.WithFields(logrus.Fields{"disk": name}) log.Info("Starting to create disk") if r.readOnly { - return fmt.Errorf("Can not create disk on read-only replica") + return fmt.Errorf("cannot create disk on read-only replica") } if len(r.activeDiskData)+1 > maximumChainLength { - return fmt.Errorf("Too many active disks: %v", len(r.activeDiskData)+1) + return fmt.Errorf("too many active disks: %v", len(r.activeDiskData)+1) } oldHead := r.info.Head @@ -994,7 +993,7 @@ func (r *Replica) openLiveChain() error { } if len(chain) > maximumChainLength { - return fmt.Errorf("Live chain is too long: %v", len(chain)) + return fmt.Errorf("live chain is too long: %v", len(chain)) } for i := len(chain) - 1; i >= 0; i-- { @@ -1027,7 +1026,7 @@ func (r *Replica) readMetadata() (bool, error) { if err := r.unmarshalFile(file.Name(), &r.info); err != nil { return false, err } - r.volume.sectorSize = defaultSectorSize + r.volume.sectorSize = util.VolumeSectorSize r.volume.size = r.info.Size } else if strings.HasSuffix(file.Name(), metadataSuffix) { if err := r.readDiskData(file.Name()); err != nil { @@ -1114,7 +1113,7 @@ func (r *Replica) Expand(size int64) (err error) { defer r.Unlock() if r.info.Size > size { - return fmt.Errorf("Cannot expand replica to a smaller size %v", size) + return fmt.Errorf("cannot expand replica to a smaller size %v", size) } else if r.info.Size == size { logrus.Infof("Replica had been expanded to size %v", size) return nil @@ -1133,7 +1132,7 @@ func (r *Replica) Expand(size int64) (err error) { func (r *Replica) WriteAt(buf []byte, offset int64) (int, error) { if r.readOnly { - return 0, fmt.Errorf("Can not write on read-only replica") + return 0, fmt.Errorf("cannot write on read-only replica") } r.RLock() diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/revision_counter.go b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/revision_counter.go index f540088ea..d93ed74c3 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/revision_counter.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/revision_counter.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/longhorn/sparse-tools/sparse" @@ -26,11 +27,11 @@ func (r *Replica) readRevisionCounter() (int64, error) { buf := make([]byte, revisionBlockSize) _, err := r.revisionFile.ReadAt(buf, 0) if err != nil && err != io.EOF { - return 0, fmt.Errorf("failed to read from revision counter file: %v", err) + return 0, errors.Wrap(err, "failed to read from revision counter file") } counter, err := strconv.ParseInt(strings.Trim(string(buf), "\x00"), 10, 64) if err != nil { - return 0, fmt.Errorf("failed to parse revision counter file: %v", err) + return 0, errors.Wrap(err, "failed to parse revision counter file") } return counter, nil } @@ -44,7 +45,7 @@ func (r *Replica) writeRevisionCounter(counter int64) error { copy(buf, []byte(strconv.FormatInt(counter, 10))) _, err := r.revisionFile.WriteAt(buf, 0) if err != nil { - return fmt.Errorf("failed to write to revision counter file: %v", err) + return errors.Wrap(err, "failed to write to revision counter file") } return nil } diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/server.go b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/server.go index fef0039ec..46de8b0d3 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/server.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/server.go @@ -25,7 +25,7 @@ type Server struct { sync.RWMutex r *Replica dir string - defaultSectorSize int64 + sectorSize int64 backing *backingfile.BackingFile revisionCounterDisabled bool } @@ -34,7 +34,7 @@ func NewServer(dir string, backing *backingfile.BackingFile, sectorSize int64, d return &Server{ dir: dir, backing: backing, - defaultSectorSize: sectorSize, + sectorSize: sectorSize, revisionCounterDisabled: disableRevCounter, } } @@ -43,7 +43,7 @@ func (s *Server) getSectorSize() int64 { if s.backing != nil && s.backing.SectorSize > 0 { return s.backing.SectorSize } - return s.defaultSectorSize + return s.sectorSize } func (s *Server) Create(size int64) error { @@ -71,7 +71,7 @@ func (s *Server) Open() error { defer s.Unlock() if s.r != nil { - return fmt.Errorf("Replica is already open") + return fmt.Errorf("replica is already open") } _, info := s.Status() @@ -138,7 +138,7 @@ func (s *Server) SetRebuilding(rebuilding bool) error { // Must be Open/Dirty to set true or must be Rebuilding to set false if (rebuilding && state != Open && state != Dirty) || (!rebuilding && state != Rebuilding) { - return fmt.Errorf("Can not set rebuilding=%v from state %s", rebuilding, state) + return fmt.Errorf("cannot set rebuilding=%v from state %s", rebuilding, state) } return s.r.SetRebuilding(rebuilding) @@ -280,7 +280,7 @@ func (s *Server) WriteAt(buf []byte, offset int64) (int, error) { defer s.RUnlock() if s.r == nil { - return 0, fmt.Errorf("Volume no longer exist") + return 0, fmt.Errorf("volume no longer exist") } i, err := s.r.WriteAt(buf, offset) return i, err @@ -291,7 +291,7 @@ func (s *Server) ReadAt(buf []byte, offset int64) (int, error) { defer s.RUnlock() if s.r == nil { - return 0, fmt.Errorf("Volume no longer exist") + return 0, fmt.Errorf("volume no longer exist") } i, err := s.r.ReadAt(buf, offset) return i, err diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/sync/backup.go b/vendor/github.com/longhorn/longhorn-engine/pkg/sync/backup.go index 0cbc12ec7..81ce6bbf5 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/sync/backup.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/sync/backup.go @@ -44,7 +44,7 @@ func (t *Task) CreateBackup(backupName, snapshot, dest, backingImageName, backin var replica *types.ControllerReplicaInfo if snapshot == VolumeHeadName { - return nil, fmt.Errorf("can not backup the head disk in the chain") + return nil, fmt.Errorf("cannot backup the head disk in the chain") } volume, err := t.client.VolumeGet() @@ -175,7 +175,7 @@ func (t *Task) RestoreBackup(backup string, credential map[string]string) error if isRebuilding, err := t.isRebuilding(r); err != nil { taskErr.Append(NewReplicaError(r.Address, err)) } else if isRebuilding { - taskErr.Append(NewReplicaError(r.Address, fmt.Errorf("can not do restore for normal rebuilding replica"))) + taskErr.Append(NewReplicaError(r.Address, fmt.Errorf("cannot do restore for normal rebuilding replica"))) } } if taskErr.HasError() { @@ -284,7 +284,7 @@ func (t *Task) RestoreBackup(backup string, credential map[string]string) error func (t *Task) restoreBackup(replicaInController *types.ControllerReplicaInfo, backup string, snapshotFile string, credential map[string]string) error { if replicaInController.Mode == types.ERR { - return fmt.Errorf("can not restore backup from replica in mode ERR") + return fmt.Errorf("cannot restore backup from replica in mode ERR") } repClient, err := replicaClient.NewReplicaClient(replicaInController.Address) @@ -313,7 +313,7 @@ func (t *Task) Reset() error { return err } else if ok { logrus.Errorf("Replicas are rebuilding. Can't reset: %v", err) - return fmt.Errorf("can not reset Restore info as replica(%s) is rebuilding", r.Address) + return fmt.Errorf("cannot reset Restore info as replica(%s) is rebuilding", r.Address) } } diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/sync/sync.go b/vendor/github.com/longhorn/longhorn-engine/pkg/sync/sync.go index c38ad219d..e053af99b 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/sync/sync.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/sync/sync.go @@ -798,7 +798,7 @@ func CloneSnapshot(engineControllerClient, fromControllerClient *client.Controll } } if sourceReplica == nil { - return fmt.Errorf("cannot find a RW replica in the source volume for clonning") + return fmt.Errorf("cannot find a RW replica in the source volume for cloning") } replicas, err = engineControllerClient.ReplicaList() diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/types/types.go b/vendor/github.com/longhorn/longhorn-engine/pkg/types/types.go index 6209fbf52..8de1111f9 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/types/types.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/types/types.go @@ -69,7 +69,7 @@ type Backend interface { } type BackendFactory interface { - Create(address string) (Backend, error) + Create(address string, engineReplicaTimeout time.Duration) (Backend, error) } type Controller interface { diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/util/util.go b/vendor/github.com/longhorn/longhorn-engine/pkg/util/util.go index ae14350eb..311197051 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/util/util.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/util/util.go @@ -19,10 +19,10 @@ import ( "github.com/google/uuid" "github.com/gorilla/handlers" - iutil "github.com/longhorn/go-iscsi-helper/util" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" + iutil "github.com/longhorn/go-iscsi-helper/util" "github.com/longhorn/longhorn-engine/pkg/types" ) @@ -30,12 +30,14 @@ var ( MaximumVolumeNameSize = 64 validVolumeName = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.-]+$`) - cmdTimeout = time.Minute // one minute by default - HostProc = "/host/proc" ) const ( + VolumeSectorSize = 4096 + ReplicaSectorSize = 512 + BackingImageSectorSize = 512 + BlockSizeLinux = 512 ) diff --git a/vendor/golang.org/x/net/http2/client_conn_pool.go b/vendor/golang.org/x/net/http2/client_conn_pool.go index f4d9b5ece..3a67636fe 100644 --- a/vendor/golang.org/x/net/http2/client_conn_pool.go +++ b/vendor/golang.org/x/net/http2/client_conn_pool.go @@ -107,6 +107,7 @@ func (p *clientConnPool) getClientConn(req *http.Request, addr string, dialOnMis // dialCall is an in-flight Transport dial call to a host. type dialCall struct { + _ incomparable p *clientConnPool done chan struct{} // closed when done res *ClientConn // valid after done is closed @@ -180,6 +181,7 @@ func (p *clientConnPool) addConnIfNeeded(key string, t *Transport, c *tls.Conn) } type addConnCall struct { + _ incomparable p *clientConnPool done chan struct{} // closed when done err error @@ -200,12 +202,6 @@ func (c *addConnCall) run(t *Transport, key string, tc *tls.Conn) { close(c.done) } -func (p *clientConnPool) addConn(key string, cc *ClientConn) { - p.mu.Lock() - p.addConnLocked(key, cc) - p.mu.Unlock() -} - // p.mu must be held func (p *clientConnPool) addConnLocked(key string, cc *ClientConn) { for _, v := range p.conns[key] { diff --git a/vendor/golang.org/x/net/http2/flow.go b/vendor/golang.org/x/net/http2/flow.go index cea601fcd..b51f0e0cf 100644 --- a/vendor/golang.org/x/net/http2/flow.go +++ b/vendor/golang.org/x/net/http2/flow.go @@ -8,6 +8,8 @@ package http2 // flow is the flow control window's size. type flow struct { + _ incomparable + // n is the number of DATA bytes we're allowed to send. // A flow is kept both on a conn and a per-stream. n int32 diff --git a/vendor/golang.org/x/net/http2/hpack/huffman.go b/vendor/golang.org/x/net/http2/hpack/huffman.go index b412a96c5..a1ab2f056 100644 --- a/vendor/golang.org/x/net/http2/hpack/huffman.go +++ b/vendor/golang.org/x/net/http2/hpack/huffman.go @@ -105,7 +105,14 @@ func huffmanDecode(buf *bytes.Buffer, maxLen int, v []byte) error { return nil } +// incomparable is a zero-width, non-comparable type. Adding it to a struct +// makes that struct also non-comparable, and generally doesn't add +// any size (as long as it's first). +type incomparable [0]func() + type node struct { + _ incomparable + // children is non-nil for internal nodes children *[256]*node diff --git a/vendor/golang.org/x/net/http2/http2.go b/vendor/golang.org/x/net/http2/http2.go index bdaba1d46..5571ccfd2 100644 --- a/vendor/golang.org/x/net/http2/http2.go +++ b/vendor/golang.org/x/net/http2/http2.go @@ -19,7 +19,6 @@ package http2 // import "golang.org/x/net/http2" import ( "bufio" "crypto/tls" - "errors" "fmt" "io" "net/http" @@ -173,11 +172,6 @@ func (s SettingID) String() string { return fmt.Sprintf("UNKNOWN_SETTING_%d", uint16(s)) } -var ( - errInvalidHeaderFieldName = errors.New("http2: invalid header field name") - errInvalidHeaderFieldValue = errors.New("http2: invalid header field value") -) - // validWireHeaderFieldName reports whether v is a valid header field // name (key). See httpguts.ValidHeaderName for the base rules. // @@ -247,6 +241,7 @@ func (cw closeWaiter) Wait() { // Its buffered writer is lazily allocated as needed, to minimize // idle memory usage with many connections. type bufferedWriter struct { + _ incomparable w io.Writer // immutable bw *bufio.Writer // non-nil when data is buffered } @@ -319,6 +314,7 @@ func bodyAllowedForStatus(status int) bool { } type httpError struct { + _ incomparable msg string timeout bool } @@ -382,3 +378,8 @@ func (s *sorter) SortStrings(ss []string) { func validPseudoPath(v string) bool { return (len(v) > 0 && v[0] == '/') || v == "*" } + +// incomparable is a zero-width, non-comparable type. Adding it to a struct +// makes that struct also non-comparable, and generally doesn't add +// any size (as long as it's first). +type incomparable [0]func() diff --git a/vendor/golang.org/x/net/http2/server.go b/vendor/golang.org/x/net/http2/server.go index de31d72b2..2aa859f76 100644 --- a/vendor/golang.org/x/net/http2/server.go +++ b/vendor/golang.org/x/net/http2/server.go @@ -581,13 +581,10 @@ type stream struct { cancelCtx func() // owned by serverConn's serve loop: - bodyBytes int64 // body bytes seen so far - declBodyBytes int64 // or -1 if undeclared - flow flow // limits writing from Handler to client - inflow flow // what the client is allowed to POST/etc to us - parent *stream // or nil - numTrailerValues int64 - weight uint8 + bodyBytes int64 // body bytes seen so far + declBodyBytes int64 // or -1 if undeclared + flow flow // limits writing from Handler to client + inflow flow // what the client is allowed to POST/etc to us state streamState resetQueued bool // RST_STREAM queued for write; set by sc.resetStream gotTrailerHeader bool // HEADER frame for trailers was seen @@ -764,6 +761,7 @@ func (sc *serverConn) readFrames() { // frameWriteResult is the message passed from writeFrameAsync to the serve goroutine. type frameWriteResult struct { + _ incomparable wr FrameWriteRequest // what was written (or attempted) err error // result of the writeFrame call } @@ -774,7 +772,7 @@ type frameWriteResult struct { // serverConn. func (sc *serverConn) writeFrameAsync(wr FrameWriteRequest) { err := wr.write.writeFrame(sc) - sc.wroteFrameCh <- frameWriteResult{wr, err} + sc.wroteFrameCh <- frameWriteResult{wr: wr, err: err} } func (sc *serverConn) closeAllStreamsOnConnClose() { @@ -1164,7 +1162,7 @@ func (sc *serverConn) startFrameWrite(wr FrameWriteRequest) { if wr.write.staysWithinBuffer(sc.bw.Available()) { sc.writingFrameAsync = false err := wr.write.writeFrame(sc) - sc.wroteFrame(frameWriteResult{wr, err}) + sc.wroteFrame(frameWriteResult{wr: wr, err: err}) } else { sc.writingFrameAsync = true go sc.writeFrameAsync(wr) @@ -1696,6 +1694,7 @@ func (sc *serverConn) processData(f *DataFrame) error { if len(data) > 0 { wrote, err := st.body.Write(data) if err != nil { + sc.sendWindowUpdate(nil, int(f.Length)-wrote) return streamError(id, ErrCodeStreamClosed) } if wrote != len(data) { @@ -2022,7 +2021,11 @@ func (sc *serverConn) newWriterAndRequest(st *stream, f *MetaHeadersFrame) (*res } if bodyOpen { if vv, ok := rp.header["Content-Length"]; ok { - req.ContentLength, _ = strconv.ParseInt(vv[0], 10, 64) + if cl, err := strconv.ParseUint(vv[0], 10, 63); err == nil { + req.ContentLength = int64(cl) + } else { + req.ContentLength = 0 + } } else { req.ContentLength = -1 } @@ -2060,7 +2063,7 @@ func (sc *serverConn) newWriterAndRequestNoBody(st *stream, rp requestParam) (*r var trailer http.Header for _, v := range rp.header["Trailer"] { for _, key := range strings.Split(v, ",") { - key = http.CanonicalHeaderKey(strings.TrimSpace(key)) + key = http.CanonicalHeaderKey(textproto.TrimString(key)) switch key { case "Transfer-Encoding", "Trailer", "Content-Length": // Bogus. (copy of http1 rules) @@ -2278,6 +2281,7 @@ func (sc *serverConn) sendWindowUpdate32(st *stream, n int32) { // requestBody is the Handler's Request.Body type. // Read and Close may be called concurrently. type requestBody struct { + _ incomparable stream *stream conn *serverConn closed bool // for use by Close only @@ -2404,9 +2408,8 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) { var ctype, clen string if clen = rws.snapHeader.Get("Content-Length"); clen != "" { rws.snapHeader.Del("Content-Length") - clen64, err := strconv.ParseInt(clen, 10, 64) - if err == nil && clen64 >= 0 { - rws.sentContentLen = clen64 + if cl, err := strconv.ParseUint(clen, 10, 63); err == nil { + rws.sentContentLen = int64(cl) } else { clen = "" } diff --git a/vendor/golang.org/x/net/http2/transport.go b/vendor/golang.org/x/net/http2/transport.go index d948e96ee..4182f52b4 100644 --- a/vendor/golang.org/x/net/http2/transport.go +++ b/vendor/golang.org/x/net/http2/transport.go @@ -108,6 +108,19 @@ type Transport struct { // waiting for their turn. StrictMaxConcurrentStreams bool + // ReadIdleTimeout is the timeout after which a health check using ping + // frame will be carried out if no frame is received on the connection. + // Note that a ping response will is considered a received frame, so if + // there is no other traffic on the connection, the health check will + // be performed every ReadIdleTimeout interval. + // If zero, no health check is performed. + ReadIdleTimeout time.Duration + + // PingTimeout is the timeout after which the connection will be closed + // if a response to Ping is not received. + // Defaults to 15s. + PingTimeout time.Duration + // t1, if non-nil, is the standard library Transport using // this transport. Its settings are used (but not its // RoundTrip method, etc). @@ -131,6 +144,14 @@ func (t *Transport) disableCompression() bool { return t.DisableCompression || (t.t1 != nil && t.t1.DisableCompression) } +func (t *Transport) pingTimeout() time.Duration { + if t.PingTimeout == 0 { + return 15 * time.Second + } + return t.PingTimeout + +} + // ConfigureTransport configures a net/http HTTP/1 Transport to use HTTP/2. // It returns an error if t1 has already been HTTP/2-enabled. func ConfigureTransport(t1 *http.Transport) error { @@ -668,6 +689,7 @@ func (t *Transport) newClientConn(c net.Conn, singleUse bool) (*ClientConn, erro cc.inflow.add(transportDefaultConnFlow + initialWindowSize) cc.bw.Flush() if cc.werr != nil { + cc.Close() return nil, cc.werr } @@ -675,6 +697,20 @@ func (t *Transport) newClientConn(c net.Conn, singleUse bool) (*ClientConn, erro return cc, nil } +func (cc *ClientConn) healthCheck() { + pingTimeout := cc.t.pingTimeout() + // We don't need to periodically ping in the health check, because the readLoop of ClientConn will + // trigger the healthCheck again if there is no frame received. + ctx, cancel := context.WithTimeout(context.Background(), pingTimeout) + defer cancel() + err := cc.Ping(ctx) + if err != nil { + cc.closeForLostPing() + cc.t.connPool().MarkDead(cc) + return + } +} + func (cc *ClientConn) setGoAway(f *GoAwayFrame) { cc.mu.Lock() defer cc.mu.Unlock() @@ -846,14 +882,12 @@ func (cc *ClientConn) sendGoAway() error { return nil } -// Close closes the client connection immediately. -// -// In-flight requests are interrupted. For a graceful shutdown, use Shutdown instead. -func (cc *ClientConn) Close() error { +// closes the client connection immediately. In-flight requests are interrupted. +// err is sent to streams. +func (cc *ClientConn) closeForError(err error) error { cc.mu.Lock() defer cc.cond.Broadcast() defer cc.mu.Unlock() - err := errors.New("http2: client connection force closed via ClientConn.Close") for id, cs := range cc.streams { select { case cs.resc <- resAndError{err: err}: @@ -866,6 +900,20 @@ func (cc *ClientConn) Close() error { return cc.tconn.Close() } +// Close closes the client connection immediately. +// +// In-flight requests are interrupted. For a graceful shutdown, use Shutdown instead. +func (cc *ClientConn) Close() error { + err := errors.New("http2: client connection force closed via ClientConn.Close") + return cc.closeForError(err) +} + +// closes the client connection immediately. In-flight requests are interrupted. +func (cc *ClientConn) closeForLostPing() error { + err := errors.New("http2: client connection lost") + return cc.closeForError(err) +} + const maxAllocFrameSize = 512 << 10 // frameBuffer returns a scratch buffer suitable for writing DATA frames. @@ -916,7 +964,7 @@ func commaSeparatedTrailers(req *http.Request) (string, error) { k = http.CanonicalHeaderKey(k) switch k { case "Transfer-Encoding", "Trailer", "Content-Length": - return "", &badStringError{"invalid Trailer key", k} + return "", fmt.Errorf("invalid Trailer key %q", k) } keys = append(keys, k) } @@ -1033,6 +1081,15 @@ func (cc *ClientConn) roundTrip(req *http.Request) (res *http.Response, gotErrAf bodyWriter := cc.t.getBodyWriterState(cs, body) cs.on100 = bodyWriter.on100 + defer func() { + cc.wmu.Lock() + werr := cc.werr + cc.wmu.Unlock() + if werr != nil { + cc.Close() + } + }() + cc.wmu.Lock() endStream := !hasBody && !hasTrailers werr := cc.writeHeaders(cs.ID, endStream, int(cc.maxFrameSize), hdrs) @@ -1394,13 +1451,6 @@ func (cs *clientStream) awaitFlowControl(maxBytes int) (taken int32, err error) } } -type badStringError struct { - what string - str string -} - -func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) } - // requires cc.mu be held. func (cc *ClientConn) encodeHeaders(req *http.Request, addGzipHeader bool, trailers string, contentLength int64) ([]byte, error) { cc.hbuf.Reset() @@ -1616,6 +1666,7 @@ func (cc *ClientConn) writeHeader(name, value string) { } type resAndError struct { + _ incomparable res *http.Response err error } @@ -1663,6 +1714,7 @@ func (cc *ClientConn) streamByID(id uint32, andRemove bool) *clientStream { // clientConnReadLoop is the state owned by the clientConn's frame-reading readLoop. type clientConnReadLoop struct { + _ incomparable cc *ClientConn closeWhenIdle bool } @@ -1742,8 +1794,17 @@ func (rl *clientConnReadLoop) run() error { rl.closeWhenIdle = cc.t.disableKeepAlives() || cc.singleUse gotReply := false // ever saw a HEADERS reply gotSettings := false + readIdleTimeout := cc.t.ReadIdleTimeout + var t *time.Timer + if readIdleTimeout != 0 { + t = time.AfterFunc(readIdleTimeout, cc.healthCheck) + defer t.Stop() + } for { f, err := cc.fr.ReadFrame() + if t != nil { + t.Reset(readIdleTimeout) + } if err != nil { cc.vlogf("http2: Transport readFrame error on conn %p: (%T) %v", cc, err, err) } @@ -1892,7 +1953,9 @@ func (rl *clientConnReadLoop) handleResponse(cs *clientStream, f *MetaHeadersFra return nil, errors.New("malformed response from server: malformed non-numeric status pseudo header") } - header := make(http.Header) + regularFields := f.RegularFields() + strs := make([]string, len(regularFields)) + header := make(http.Header, len(regularFields)) res := &http.Response{ Proto: "HTTP/2.0", ProtoMajor: 2, @@ -1900,7 +1963,7 @@ func (rl *clientConnReadLoop) handleResponse(cs *clientStream, f *MetaHeadersFra StatusCode: statusCode, Status: status + " " + http.StatusText(statusCode), } - for _, hf := range f.RegularFields() { + for _, hf := range regularFields { key := http.CanonicalHeaderKey(hf.Name) if key == "Trailer" { t := res.Trailer @@ -1912,7 +1975,18 @@ func (rl *clientConnReadLoop) handleResponse(cs *clientStream, f *MetaHeadersFra t[http.CanonicalHeaderKey(v)] = nil }) } else { - header[key] = append(header[key], hf.Value) + vv := header[key] + if vv == nil && len(strs) > 0 { + // More than likely this will be a single-element key. + // Most headers aren't multi-valued. + // Set the capacity on strs[0] to 1, so any future append + // won't extend the slice into the other strings. + vv, strs = strs[:1:1], strs[1:] + vv[0] = hf.Value + header[key] = vv + } else { + header[key] = append(vv, hf.Value) + } } } @@ -1942,8 +2016,8 @@ func (rl *clientConnReadLoop) handleResponse(cs *clientStream, f *MetaHeadersFra if !streamEnded || isHead { res.ContentLength = -1 if clens := res.Header["Content-Length"]; len(clens) == 1 { - if clen64, err := strconv.ParseInt(clens[0], 10, 64); err == nil { - res.ContentLength = clen64 + if cl, err := strconv.ParseUint(clens[0], 10, 63); err == nil { + res.ContentLength = int64(cl) } else { // TODO: care? unlike http/1, it won't mess up our framing, so it's // more safe smuggling-wise to ignore. @@ -2198,8 +2272,6 @@ func (rl *clientConnReadLoop) processData(f *DataFrame) error { return nil } -var errInvalidTrailers = errors.New("http2: invalid trailers") - func (rl *clientConnReadLoop) endStream(cs *clientStream) { // TODO: check that any declared content-length matches, like // server.go's (*stream).endStream method. @@ -2430,7 +2502,6 @@ func (cc *ClientConn) writeStreamReset(streamID uint32, code ErrCode, err error) var ( errResponseHeaderListSize = errors.New("http2: response header list larger than advertised limit") errRequestHeaderListSize = errors.New("http2: request header list larger than peer's advertised limit") - errPseudoTrailers = errors.New("http2: invalid pseudo header in trailers") ) func (cc *ClientConn) logf(format string, args ...interface{}) { @@ -2464,11 +2535,13 @@ func strSliceContains(ss []string, s string) bool { type erringRoundTripper struct{ err error } +func (rt erringRoundTripper) RoundTripErr() error { return rt.err } func (rt erringRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { return nil, rt.err } // gzipReader wraps a response body so it can lazily // call gzip.NewReader on the first call to Read type gzipReader struct { + _ incomparable body io.ReadCloser // underlying Response.Body zr *gzip.Reader // lazily-initialized gzip reader zerr error // sticky error diff --git a/vendor/modules.txt b/vendor/modules.txt index 264483a85..8e93c586d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -68,7 +68,7 @@ github.com/gorilla/handlers github.com/honestbee/jobq # github.com/jmespath/go-jmespath v0.3.0 github.com/jmespath/go-jmespath -# github.com/longhorn/backupstore v0.0.0-20220913112826-5f5c95274f2a +# github.com/longhorn/backupstore v0.0.0-20221114044558-19f4902cd4fd github.com/longhorn/backupstore github.com/longhorn/backupstore/fsops github.com/longhorn/backupstore/http @@ -79,7 +79,7 @@ github.com/longhorn/backupstore/util github.com/longhorn/backupstore/vfs # github.com/longhorn/go-iscsi-helper v0.0.0-20221109111031-ebff48f3632a github.com/longhorn/go-iscsi-helper/util -# github.com/longhorn/longhorn-engine v1.3.2-0.20220929032851-7aac8ae9c8b4 +# github.com/longhorn/longhorn-engine v1.3.1-0.20221115055520-8179e277e475 github.com/longhorn/longhorn-engine/pkg/backingfile github.com/longhorn/longhorn-engine/pkg/controller/client github.com/longhorn/longhorn-engine/pkg/meta @@ -112,7 +112,7 @@ github.com/tinylib/msgp/msgp github.com/urfave/cli # github.com/willf/bitset v1.1.10 github.com/willf/bitset -# golang.org/x/net v0.0.0-20200202094626-16171245cfb2 +# golang.org/x/net v0.0.0-20201021035429-f5854403a974 golang.org/x/net/context golang.org/x/net/http/httpguts golang.org/x/net/http2