diff --git a/go.mod b/go.mod index 04a55ee6..b03b674f 100644 --- a/go.mod +++ b/go.mod @@ -2,16 +2,16 @@ module github.com/longhorn/backing-image-manager go 1.22 -toolchain go1.22.3 +toolchain go1.22.4 require ( - github.com/RoaringBitmap/roaring v1.9.3 + github.com/RoaringBitmap/roaring v1.9.4 github.com/golang/protobuf v1.5.4 github.com/gorilla/mux v1.8.1 - github.com/longhorn/backupstore v0.0.0-20240509144945-3bce6e69af15 + github.com/longhorn/backupstore v0.0.0-20240603134532-5ec9a300bef4 github.com/longhorn/longhorn-engine v1.6.0-dev-20231217.0.20240418025706-519598108463 github.com/longhorn/sparse-tools v0.0.0-20240513025352-ed49dd3f93eb - github.com/longhorn/types v0.0.0-20240510221052-ab949bbedea3 + github.com/longhorn/types v0.0.0-20240605091135-ef450e1c04cd github.com/pkg/errors v0.9.1 github.com/rancher/go-fibmap v0.0.0-20160418233256-5fc9f8c1ed47 github.com/sirupsen/logrus v1.9.3 diff --git a/go.sum b/go.sum index d7072aa2..96bbd8e4 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSu github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= -github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= +github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= +github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/aws/aws-sdk-go v1.34.2 h1:9vCknCdTAmmV4ht7lPuda7aJXzllXwEQyCMZKJHjBrM= github.com/aws/aws-sdk-go v1.34.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -55,14 +55,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/longhorn/backupstore v0.0.0-20240509144945-3bce6e69af15 h1:D838/RWPKmP9ozjOM2Al8s11NbTGLC2m+ZKT3JAogfY= -github.com/longhorn/backupstore v0.0.0-20240509144945-3bce6e69af15/go.mod h1:n210xpMUVrSn/W4Za/9BZhyXLCTVfAOq5lNdLrRSyz8= +github.com/longhorn/backupstore v0.0.0-20240603134532-5ec9a300bef4 h1:FI2oBT6397aaQfspA3tDlYjkDaqhtXnwID4dMgVic0g= +github.com/longhorn/backupstore v0.0.0-20240603134532-5ec9a300bef4/go.mod h1:n210xpMUVrSn/W4Za/9BZhyXLCTVfAOq5lNdLrRSyz8= github.com/longhorn/longhorn-engine v1.6.0-dev-20231217.0.20240418025706-519598108463 h1:KxddgUYC9InOhe8MxfbxzOL5v9q7f5DyThxQGGKvqVw= github.com/longhorn/longhorn-engine v1.6.0-dev-20231217.0.20240418025706-519598108463/go.mod h1:WNiZl2l51I36/c8dewxkyWd0yBA5Anznzki0knKO88U= github.com/longhorn/sparse-tools v0.0.0-20240513025352-ed49dd3f93eb h1:Kh89s6i5T1W6BT1Aq9W1YHXojbbcTXlDieWC5KWAs/E= github.com/longhorn/sparse-tools v0.0.0-20240513025352-ed49dd3f93eb/go.mod h1:vFvENahNfkr2VAyROaGHnZrLWuXwZJ0DHj8mBIh2miI= -github.com/longhorn/types v0.0.0-20240510221052-ab949bbedea3 h1:yCn2uYikI3xW3i1HHpKytitJ2lc7S5obDMKIa7ZIuc8= -github.com/longhorn/types v0.0.0-20240510221052-ab949bbedea3/go.mod h1:1oEh1cnDDqNSuFh/dH/lvJ3Ssq83SOweTAAPLRY4PMI= +github.com/longhorn/types v0.0.0-20240605091135-ef450e1c04cd h1:gzvHnEc4vdHmOtxwgjC/7YmChbzDsfYiY0wpI3RgB1A= +github.com/longhorn/types v0.0.0-20240605091135-ef450e1c04cd/go.mod h1:1oEh1cnDDqNSuFh/dH/lvJ3Ssq83SOweTAAPLRY4PMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= diff --git a/vendor/github.com/RoaringBitmap/roaring/README.md b/vendor/github.com/RoaringBitmap/roaring/README.md index acd3058b..f6705dfe 100644 --- a/vendor/github.com/RoaringBitmap/roaring/README.md +++ b/vendor/github.com/RoaringBitmap/roaring/README.md @@ -10,7 +10,7 @@ This is a go version of the Roaring bitmap data structure. Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and -[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [anacrolix/torrent][anacrolix/torrent], [Whoosh][whoosh], [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. The YouTube SQL Engine, [Google Procella](https://research.google/pubs/pub48388/), uses Roaring bitmaps for indexing. +[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [anacrolix/torrent][anacrolix/torrent], [Whoosh][whoosh], [Redpanda](https://github.com/redpanda-data/redpanda), [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. The YouTube SQL Engine, [Google Procella](https://research.google/pubs/pub48388/), uses Roaring bitmaps for indexing. [lucene]: https://lucene.apache.org/ [solr]: https://lucene.apache.org/solr/ diff --git a/vendor/github.com/longhorn/backupstore/s3/s3.go b/vendor/github.com/longhorn/backupstore/s3/s3.go index 73d232ee..bdb813bb 100644 --- a/vendor/github.com/longhorn/backupstore/s3/s3.go +++ b/vendor/github.com/longhorn/backupstore/s3/s3.go @@ -13,7 +13,6 @@ import ( "github.com/sirupsen/logrus" "github.com/longhorn/backupstore" - "github.com/longhorn/backupstore/http" ) var ( @@ -23,7 +22,7 @@ var ( type BackupStoreDriver struct { destURL string path string - service Service + service *service } const ( @@ -37,8 +36,6 @@ func init() { } func initFunc(destURL string) (backupstore.BackupStoreDriver, error) { - b := &BackupStoreDriver{} - u, err := url.Parse(destURL) if err != nil { return nil, err @@ -48,26 +45,17 @@ func initFunc(destURL string) (backupstore.BackupStoreDriver, error) { return nil, fmt.Errorf("BUG: Why dispatch %v to %v?", u.Scheme, KIND) } - if u.User != nil { - b.service.Region = u.Host - b.service.Bucket = u.User.Username() - } else { - //We would depends on AWS_REGION environment variable - b.service.Bucket = u.Host + b := &BackupStoreDriver{} + b.service, err = newService(u) + if err != nil { + return nil, err } + b.path = u.Path if b.service.Bucket == "" || b.path == "" { return nil, fmt.Errorf("invalid URL. Must be either s3://bucket@region/path/, or s3://bucket/path") } - // add custom ca to http client that is used by s3 service - customCerts := getCustomCerts() - client, err := http.GetClientWithCustomCerts(customCerts) - if err != nil { - return nil, err - } - b.service.Client = client - //Leading '/' can cause mystery problems for s3 b.path = strings.TrimLeft(b.path, "/") diff --git a/vendor/github.com/longhorn/backupstore/s3/s3_service.go b/vendor/github.com/longhorn/backupstore/s3/s3_service.go index 1b5eee6d..e5aa7366 100644 --- a/vendor/github.com/longhorn/backupstore/s3/s3_service.go +++ b/vendor/github.com/longhorn/backupstore/s3/s3_service.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "time" @@ -13,9 +14,11 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/pkg/errors" + + bhttp "github.com/longhorn/backupstore/http" ) -type Service struct { +type service struct { Region string Bucket string Client *http.Client @@ -25,7 +28,28 @@ const ( VirtualHostedStyle = "VIRTUAL_HOSTED_STYLE" ) -func (s *Service) New() (*s3.S3, error) { +func newService(u *url.URL) (*service, error) { + s := service{} + if u.User != nil { + s.Region = u.Host + s.Bucket = u.User.Username() + } else { + //We would depends on AWS_REGION environment variable + s.Bucket = u.Host + } + + // add custom ca to http client that is used by s3 service + customCerts := getCustomCerts() + client, err := bhttp.GetClientWithCustomCerts(customCerts) + if err != nil { + return nil, err + } + s.Client = client + + return &s, nil +} + +func (s *service) newInstance() (*s3.S3, error) { // get custom endpoint endpoints := os.Getenv("AWS_ENDPOINTS") config := &aws.Config{Region: &s.Region, MaxRetries: aws.Int(3)} @@ -58,7 +82,7 @@ func (s *Service) New() (*s3.S3, error) { return s3.New(ses), nil } -func (s *Service) Close() { +func (s *service) Close() { } func parseAwsError(err error) error { @@ -72,8 +96,8 @@ func parseAwsError(err error) error { return err } -func (s *Service) ListObjects(key, delimiter string) ([]*s3.Object, []*s3.CommonPrefix, error) { - svc, err := s.New() +func (s *service) ListObjects(key, delimiter string) ([]*s3.Object, []*s3.CommonPrefix, error) { + svc, err := s.newInstance() if err != nil { return nil, nil, err } @@ -102,8 +126,8 @@ func (s *Service) ListObjects(key, delimiter string) ([]*s3.Object, []*s3.Common return objects, commonPrefixs, nil } -func (s *Service) HeadObject(key string) (*s3.HeadObjectOutput, error) { - svc, err := s.New() +func (s *service) HeadObject(key string) (*s3.HeadObjectOutput, error) { + svc, err := s.newInstance() if err != nil { return nil, err } @@ -120,8 +144,8 @@ func (s *Service) HeadObject(key string) (*s3.HeadObjectOutput, error) { return resp, nil } -func (s *Service) PutObject(key string, reader io.ReadSeeker) error { - svc, err := s.New() +func (s *service) PutObject(key string, reader io.ReadSeeker) error { + svc, err := s.newInstance() if err != nil { return err } @@ -149,8 +173,8 @@ func (s *Service) PutObject(key string, reader io.ReadSeeker) error { return nil } -func (s *Service) GetObject(key string) (io.ReadCloser, error) { - svc, err := s.New() +func (s *service) GetObject(key string) (io.ReadCloser, error) { + svc, err := s.newInstance() if err != nil { return nil, err } @@ -170,14 +194,14 @@ func (s *Service) GetObject(key string) (io.ReadCloser, error) { return resp.Body, nil } -func (s *Service) DeleteObjects(key string) error { +func (s *service) DeleteObjects(key string) error { objects, _, err := s.ListObjects(key, "") if err != nil { return errors.Wrapf(err, "failed to list objects with prefix %v before removing them", key) } - svc, err := s.New() + svc, err := s.newInstance() if err != nil { return errors.Wrap(err, "failed to get a new s3 client instance before removing objects") } diff --git a/vendor/modules.txt b/vendor/modules.txt index 453b7644..c4afe053 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -20,7 +20,7 @@ github.com/Azure/azure-sdk-for-go/sdk/internal/uuid ## explicit; go 1.16 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/internal -# github.com/RoaringBitmap/roaring v1.9.3 +# github.com/RoaringBitmap/roaring v1.9.4 ## explicit; go 1.14 github.com/RoaringBitmap/roaring github.com/RoaringBitmap/roaring/internal @@ -117,7 +117,7 @@ github.com/kr/pretty # github.com/kr/text v0.2.0 ## explicit github.com/kr/text -# github.com/longhorn/backupstore v0.0.0-20240509144945-3bce6e69af15 +# github.com/longhorn/backupstore v0.0.0-20240603134532-5ec9a300bef4 ## explicit; go 1.21 github.com/longhorn/backupstore github.com/longhorn/backupstore/azblob @@ -145,7 +145,7 @@ github.com/longhorn/sparse-tools/sparse github.com/longhorn/sparse-tools/sparse/rest github.com/longhorn/sparse-tools/types github.com/longhorn/sparse-tools/util -# github.com/longhorn/types v0.0.0-20240510221052-ab949bbedea3 +# github.com/longhorn/types v0.0.0-20240605091135-ef450e1c04cd ## explicit; go 1.21 github.com/longhorn/types/pkg/generated/bimrpc github.com/longhorn/types/pkg/generated/enginerpc