From 8a8cff7fa6a30ed80cdffd4adf39215a3ab25cea Mon Sep 17 00:00:00 2001 From: "renovate-sh-app[bot]" <219655108+renovate-sh-app[bot]@users.noreply.github.com> Date: Sat, 8 Nov 2025 09:34:59 +0000 Subject: [PATCH] fix(deps): update module github.com/baidubce/bce-sdk-go to v0.9.251 | datasource | package | from | to | | ---------- | ------------------------------ | -------- | -------- | | go | github.com/baidubce/bce-sdk-go | v0.9.245 | v0.9.251 | Signed-off-by: renovate-sh-app[bot] <219655108+renovate-sh-app[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 +- .../baidubce/bce-sdk-go/bce/client.go | 63 ++- .../baidubce/bce-sdk-go/bce/config.go | 4 +- .../baidubce/bce-sdk-go/bce/limiter.go | 43 ++ .../baidubce/bce-sdk-go/http/client.go | 65 ++- .../baidubce/bce-sdk-go/http/constants.go | 1 + .../baidubce/bce-sdk-go/http/request.go | 18 +- .../baidubce/bce-sdk-go/rate/rate.go | 427 ++++++++++++++++++ .../baidubce/bce-sdk-go/rate/sometimes.go | 69 +++ .../bce-sdk-go/services/bos/api/model.go | 82 +++- .../bce-sdk-go/services/bos/api/multipart.go | 29 +- .../bce-sdk-go/services/bos/api/object.go | 238 ++++------ .../bce-sdk-go/services/bos/api/option.go | 17 +- .../bce-sdk-go/services/bos/api/util.go | 4 +- .../bce-sdk-go/services/bos/client.go | 106 ++--- vendor/modules.txt | 3 +- 17 files changed, 904 insertions(+), 271 deletions(-) create mode 100644 vendor/github.com/baidubce/bce-sdk-go/bce/limiter.go create mode 100644 vendor/github.com/baidubce/bce-sdk-go/rate/rate.go create mode 100644 vendor/github.com/baidubce/bce-sdk-go/rate/sometimes.go diff --git a/go.mod b/go.mod index b4a6cf2262bf3..cec401ef3714d 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.39.2 github.com/aws/aws-sdk-go-v2/config v1.30.0 github.com/aws/aws-sdk-go-v2/credentials v1.18.16 - github.com/baidubce/bce-sdk-go v0.9.245 + github.com/baidubce/bce-sdk-go v0.9.251 github.com/bmatcuk/doublestar/v4 v4.9.1 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/cespare/xxhash/v2 v2.3.0 diff --git a/go.sum b/go.sum index 41e9911809f1b..08a863322ece7 100644 --- a/go.sum +++ b/go.sum @@ -253,8 +253,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= -github.com/baidubce/bce-sdk-go v0.9.245 h1:Xngza8UtIQ1w470JXJUdQKQASVIYb3grkLWSj3mnBM4= -github.com/baidubce/bce-sdk-go v0.9.245/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= +github.com/baidubce/bce-sdk-go v0.9.251 h1:gjNzXhu9FtHjwPZqmXtCiOJnX8TbpwvpjTY6mWzl/uM= +github.com/baidubce/bce-sdk-go v0.9.251/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps= github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= diff --git a/vendor/github.com/baidubce/bce-sdk-go/bce/client.go b/vendor/github.com/baidubce/bce-sdk-go/bce/client.go index 52c2054a4ba9b..cb396a3651fdc 100644 --- a/vendor/github.com/baidubce/bce-sdk-go/bce/client.go +++ b/vendor/github.com/baidubce/bce-sdk-go/bce/client.go @@ -42,6 +42,7 @@ import ( "fmt" "io" "io/ioutil" + net_http "net/http" "time" "github.com/baidubce/bce-sdk-go/auth" @@ -60,8 +61,10 @@ type Client interface { // BceClient defines the general client to access the BCE services. type BceClient struct { - Config *BceClientConfiguration - Signer auth.Signer // the sign algorithm + Config *BceClientConfiguration + Signer auth.Signer // the sign algorithm + RateLimiters RateLimiters + HTTPClient *net_http.Client } // BuildHttpRequest - the helper method for the client to build http request @@ -98,6 +101,9 @@ func (c *BceClient) buildHttpRequest(request *BceRequest) { if c.Config.Credentials != nil { c.Signer.Sign(&request.Request, c.Config.Credentials, c.Config.SignOption) } + if c.HTTPClient != nil { + request.SetHTTPClient(c.HTTPClient) + } } // SendRequest - the client performs sending the http request with retry policy and receive the @@ -157,7 +163,9 @@ func (c *BceClient) SendRequest(req *BceRequest, resp *BceResponse) error { log.Infof("receive http response: status: %s, debugId: %s, requestId: %s, elapsed: %v", resp.StatusText(), resp.DebugId(), resp.RequestId(), resp.ElapsedTime()) - if resp.ElapsedTime().Milliseconds() > DEFAULT_WARN_LOG_TIMEOUT_IN_MILLS { + // not print this warn log with upload/download rate limit + if resp.ElapsedTime().Milliseconds() > DEFAULT_WARN_LOG_TIMEOUT_IN_MILLS && + (c.Config.UploadRatelimit == nil && c.Config.DownloadRatelimit == nil) { log.Warnf("request time more than 5 second, debugId: %s, requestId: %s, elapsed: %v", resp.DebugId(), resp.RequestId(), resp.ElapsedTime()) } @@ -250,6 +258,50 @@ func (c *BceClient) GetBceClientConfig() *BceClientConfiguration { return c.Config } +func NewBceClientWithExclusiveHTTPClient(conf *BceClientConfiguration, sign auth.Signer) *BceClient { + clientConfig := &http.ClientConfig{ + RedirectDisabled: conf.RedirectDisabled, + DisableKeepAlives: conf.DisableKeepAlives, + NoVerifySSL: conf.NoVerifySSL, + DialTimeout: conf.DialTimeout, + KeepAlive: conf.KeepAlive, + ReadTimeout: conf.ReadTimeout, + WriteTimeout: conf.WriteTimeOut, + TLSHandshakeTimeout: conf.TLSHandshakeTimeout, + IdleConnectionTimeout: conf.IdleConnectionTimeout, + ResponseHeaderTimeout: conf.ResponseHeaderTimeout, + HTTPClientTimeout: conf.HTTPClientTimeout, + } + + bceClient := &BceClient{ + Config: conf, + Signer: sign, + } + if conf.UploadRatelimit != nil { + value := *conf.UploadRatelimit * 1024 + tb := newRateLimiter(value) + clientConfig.PostWrite = append(clientConfig.PostWrite, func(n int, _ error) { + tb.LimitBandwidth(n) + }) + bceClient.RateLimiters[RateLimiterSlotTx] = tb + } + if conf.DownloadRatelimit != nil { + value := *conf.DownloadRatelimit * 1024 + tb := newRateLimiter(value) + clientConfig.PostRead = append(clientConfig.PostRead, func(n int, _ error) { + tb.LimitBandwidth(n) + }) + bceClient.RateLimiters[RateLimiterSlotRx] = tb + } + + if conf.HTTPClient != nil { + bceClient.HTTPClient = conf.HTTPClient + } else { + bceClient.HTTPClient = http.InitExclusiveHTTPClient(clientConfig) + } + return bceClient +} + func NewBceClientWithTimeout(conf *BceClientConfiguration, sign auth.Signer) *BceClient { clientConfig := &http.ClientConfig{ RedirectDisabled: conf.RedirectDisabled, @@ -263,11 +315,10 @@ func NewBceClientWithTimeout(conf *BceClientConfiguration, sign auth.Signer) *Bc IdleConnectionTimeout: conf.IdleConnectionTimeout, ResponseHeaderTimeout: conf.ResponseHeaderTimeout, HTTPClientTimeout: conf.HTTPClientTimeout, - HTTPClient: conf.HTTPClient, } http.InitClientWithTimeout(clientConfig) - return &BceClient{conf, sign} + return &BceClient{Config: conf, Signer: sign} } func NewBceClient(conf *BceClientConfiguration, sign auth.Signer) *BceClient { @@ -276,7 +327,7 @@ func NewBceClient(conf *BceClientConfiguration, sign auth.Signer) *BceClient { DisableKeepAlives: conf.DisableKeepAlives, } http.InitClient(clientConfig) - return &BceClient{conf, sign} + return &BceClient{Config: conf, Signer: sign} } func NewBceClientWithAkSk(ak, sk, endPoint string) (*BceClient, error) { diff --git a/vendor/github.com/baidubce/bce-sdk-go/bce/config.go b/vendor/github.com/baidubce/bce-sdk-go/bce/config.go index 8a8fbb3f37573..dda67b3b4a14b 100644 --- a/vendor/github.com/baidubce/bce-sdk-go/bce/config.go +++ b/vendor/github.com/baidubce/bce-sdk-go/bce/config.go @@ -28,7 +28,7 @@ import ( // Constants and default values for the package bce const ( - SDK_VERSION = "0.9.245" + SDK_VERSION = "0.9.251" URI_PREFIX = "/" // now support uri without prefix "v1" so just set root path DEFAULT_DOMAIN = "baidubce.com" DEFAULT_PROTOCOL = "http" @@ -77,6 +77,8 @@ type BceClientConfiguration struct { ResponseHeaderTimeout *time.Duration // http.Transport.ResponseHeaderTimeout HTTPClientTimeout *time.Duration // http.Client.Timeout HTTPClient *http.Client // customized http client + UploadRatelimit *int64 // the limit of upload rate, unit:KB/s + DownloadRatelimit *int64 // the limit of download rate, unit:KB/s } func (c *BceClientConfiguration) String() string { diff --git a/vendor/github.com/baidubce/bce-sdk-go/bce/limiter.go b/vendor/github.com/baidubce/bce-sdk-go/bce/limiter.go new file mode 100644 index 0000000000000..2e964afb8e1b5 --- /dev/null +++ b/vendor/github.com/baidubce/bce-sdk-go/bce/limiter.go @@ -0,0 +1,43 @@ +package bce + +import ( + "context" + "time" + + "github.com/baidubce/bce-sdk-go/rate" +) + +const ( + RateLimiterSlotRx int = iota + RateLimiterSlotTx + RateLimiterSlots +) + +type RateLimiter struct { + Bandwidth int64 // Byte/S + Limiter *rate.Limiter +} + +type RateLimiters [RateLimiterSlots]*RateLimiter + +func newRateLimiter(bandwidth int64) *RateLimiter { + return &RateLimiter{ + Bandwidth: bandwidth, + Limiter: newTokenBucket(bandwidth), + } +} + +func newTokenBucket(bandwidth int64) *rate.Limiter { + const defaultMaxBurstSize = 4 * 1024 * 1024 + maxBurstSize := (bandwidth * defaultMaxBurstSize) / (256 * 1024 * 1024) + if maxBurstSize < defaultMaxBurstSize { + maxBurstSize = defaultMaxBurstSize + } + tb := rate.NewLimiter(rate.Limit(bandwidth), int(maxBurstSize)) + tb.AllowN(time.Now(), int(maxBurstSize)) + return tb +} + +func (rl *RateLimiter) LimitBandwidth(n int) { + rl.Limiter.WaitN(context.Background(), n) +} diff --git a/vendor/github.com/baidubce/bce-sdk-go/http/client.go b/vendor/github.com/baidubce/bce-sdk-go/http/client.go index 5cd487a6804a3..a2d114fd7c1bb 100644 --- a/vendor/github.com/baidubce/bce-sdk-go/http/client.go +++ b/vendor/github.com/baidubce/bce-sdk-go/http/client.go @@ -27,6 +27,8 @@ import ( "net/url" "sync" "time" + + "github.com/baidubce/bce-sdk-go/util/log" ) var ( @@ -48,10 +50,17 @@ var ( transport *http.Transport ) +var defaultHTTPClient = &http.Client{ + Timeout: defaultHTTPClientTimeout, + Transport: NewTransportCustom(&DefaultClientConfig), +} + type Dialer struct { net.Dialer ReadTimeout *time.Duration WriteTimeout *time.Duration + postRead []func(n int, err error) + postWrite []func(n int, err error) } func NewDialer(config *ClientConfig) *Dialer { @@ -62,6 +71,8 @@ func NewDialer(config *ClientConfig) *Dialer { }, ReadTimeout: config.ReadTimeout, WriteTimeout: config.WriteTimeout, + postRead: config.PostRead, + postWrite: config.PostWrite, } return dialer } @@ -81,6 +92,7 @@ func (d *Dialer) DialContext(ctx context.Context, network, address string) (net. largeInterval: defaultLargeInterval, readTimeout: d.ReadTimeout, writeTimeout: d.WriteTimeout, + dialer: d, } if tc.readTimeout != nil { err = tc.SetReadDeadline(time.Now().Add(*tc.readTimeout)) @@ -107,6 +119,7 @@ type timeoutConn struct { largeInterval time.Duration readTimeout *time.Duration writeTimeout *time.Duration + dialer *Dialer } func (c *timeoutConn) Read(b []byte) (n int, err error) { @@ -125,6 +138,11 @@ func (c *timeoutConn) Read(b []byte) (n int, err error) { } else { err = c.SetReadDeadline(time.Now().Add(*c.readTimeout)) } + if c.dialer != nil { + for _, fn := range c.dialer.postRead { + fn(n, err) + } + } return n, err } func (c *timeoutConn) Write(b []byte) (n int, err error) { @@ -143,6 +161,11 @@ func (c *timeoutConn) Write(b []byte) (n int, err error) { } else { err = c.SetWriteDeadline(time.Now().Add(*c.writeTimeout)) } + if c.dialer != nil { + for _, fn := range c.dialer.postWrite { + fn(n, err) + } + } return n, err } func (c *timeoutConn) Close() error { return c.conn.Close() } @@ -164,7 +187,8 @@ type ClientConfig struct { IdleConnectionTimeout *time.Duration ResponseHeaderTimeout *time.Duration HTTPClientTimeout *time.Duration - HTTPClient *http.Client + PostRead []func(n int, err error) + PostWrite []func(n int, err error) } var DefaultClientConfig = ClientConfig{ @@ -208,6 +232,12 @@ func (cfg *ClientConfig) Copy(src *ClientConfig) { if src.HTTPClientTimeout != nil { cfg.HTTPClientTimeout = src.HTTPClientTimeout } + if len(src.PostRead) > 0 { + cfg.PostRead = append(cfg.PostRead, src.PostRead...) + } + if len(src.PostWrite) > 0 { + cfg.PostWrite = append(cfg.PostWrite, src.PostWrite...) + } } func MergeWithDefaultConfig(cfgs ...*ClientConfig) *ClientConfig { @@ -276,12 +306,23 @@ func NewTransportCustom(config *ClientConfig) *http.Transport { return transport } +func InitExclusiveHTTPClient(config *ClientConfig) *http.Client { + config = MergeWithDefaultConfig(config) + transport = NewTransportCustom(config) + myHTTPClient := &http.Client{ + Timeout: *config.HTTPClientTimeout, + Transport: transport, + } + if config.RedirectDisabled { + myHTTPClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + } + return myHTTPClient +} + func InitClientWithTimeout(config *ClientConfig) { customizeInit.Do(func() { - if config.HTTPClient != nil { - httpClient = config.HTTPClient - return - } config = MergeWithDefaultConfig(config) transport = NewTransportCustom(config) httpClient = &http.Client{ @@ -312,11 +353,21 @@ func Execute(request *Request) (*Response, error) { ProtoMinor: 1, } + // get http client + curHTTPClient := httpClient + if request.HTTPClient() != nil { + curHTTPClient = request.HTTPClient() + } + if curHTTPClient == nil { + log.Infof("use default http client to execute request") + curHTTPClient = defaultHTTPClient + } + if request.Context() != nil { httpRequest = httpRequest.WithContext(request.Context()) } else { // Set the connection timeout for current request - httpClient.Timeout = time.Duration(request.Timeout()) * time.Second + curHTTPClient.Timeout = time.Duration(request.Timeout()) * time.Second } // Set the request method @@ -361,7 +412,7 @@ func Execute(request *Request) (*Response, error) { // that may continue sending request's data subsequently. start := time.Now() - httpResponse, err := httpClient.Do(httpRequest) + httpResponse, err := curHTTPClient.Do(httpRequest) end := time.Now() if err != nil { diff --git a/vendor/github.com/baidubce/bce-sdk-go/http/constants.go b/vendor/github.com/baidubce/bce-sdk-go/http/constants.go index d28ba2f17171f..4f9383875927d 100644 --- a/vendor/github.com/baidubce/bce-sdk-go/http/constants.go +++ b/vendor/github.com/baidubce/bce-sdk-go/http/constants.go @@ -98,6 +98,7 @@ const ( BCE_SERVER_SIDE_ENCRYPTION_KEY_ID = "x-bce-server-side-encryption-bos-kms-key-id" BCE_OBJECT_RETENTION_DATE = "x-bce-object-rentention-date" BCE_TAGGING_COUNT = "x-bce-tagging-count" + BCE_CONTENT_CRC64ECMA = "x-bce-content-crc64ecma" //post field SUCCESS_ACTION_REDIRECT = "success-action-redirect" SUCCESS_ACTION_STATUS = "success-action-status" diff --git a/vendor/github.com/baidubce/bce-sdk-go/http/request.go b/vendor/github.com/baidubce/bce-sdk-go/http/request.go index 21b5eb0517ae5..a4cbe20c2f1cd 100644 --- a/vendor/github.com/baidubce/bce-sdk-go/http/request.go +++ b/vendor/github.com/baidubce/bce-sdk-go/http/request.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "io" + "net/http" "strconv" "strings" @@ -39,9 +40,20 @@ type Request struct { params map[string]string // Optional body and length fields to set the body stream and content length - body io.ReadCloser - length int64 - ctx context.Context + body io.ReadCloser + length int64 + ctx context.Context + httpClient *http.Client +} + +func (r *Request) HTTPClient() *http.Client { + return r.httpClient +} + +func (r *Request) SetHTTPClient(client *http.Client) { + if client != nil { + r.httpClient = client + } } func (r *Request) Context() context.Context { diff --git a/vendor/github.com/baidubce/bce-sdk-go/rate/rate.go b/vendor/github.com/baidubce/bce-sdk-go/rate/rate.go new file mode 100644 index 0000000000000..563270c15494f --- /dev/null +++ b/vendor/github.com/baidubce/bce-sdk-go/rate/rate.go @@ -0,0 +1,427 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package rate provides a rate limiter. +package rate + +import ( + "context" + "fmt" + "math" + "sync" + "time" +) + +// Limit defines the maximum frequency of some events. +// Limit is represented as number of events per second. +// A zero Limit allows no events. +type Limit float64 + +// Inf is the infinite rate limit; it allows all events (even if burst is zero). +const Inf = Limit(math.MaxFloat64) + +// Every converts a minimum time interval between events to a Limit. +func Every(interval time.Duration) Limit { + if interval <= 0 { + return Inf + } + return 1 / Limit(interval.Seconds()) +} + +// A Limiter controls how frequently events are allowed to happen. +// It implements a "token bucket" of size b, initially full and refilled +// at rate r tokens per second. +// Informally, in any large enough time interval, the Limiter limits the +// rate to r tokens per second, with a maximum burst size of b events. +// As a special case, if r == Inf (the infinite rate), b is ignored. +// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets. +// +// The zero value is a valid Limiter, but it will reject all events. +// Use NewLimiter to create non-zero Limiters. +// +// Limiter has three main methods, Allow, Reserve, and Wait. +// Most callers should use Wait. +// +// Each of the three methods consumes a single token. +// They differ in their behavior when no token is available. +// If no token is available, Allow returns false. +// If no token is available, Reserve returns a reservation for a future token +// and the amount of time the caller must wait before using it. +// If no token is available, Wait blocks until one can be obtained +// or its associated context.Context is canceled. +// +// The methods AllowN, ReserveN, and WaitN consume n tokens. +// +// Limiter is safe for simultaneous use by multiple goroutines. +type Limiter struct { + mu sync.Mutex + limit Limit + burst int + tokens float64 + // last is the last time the limiter's tokens field was updated + last time.Time + // lastEvent is the latest time of a rate-limited event (past or future) + lastEvent time.Time +} + +// Limit returns the maximum overall event rate. +func (lim *Limiter) Limit() Limit { + lim.mu.Lock() + defer lim.mu.Unlock() + return lim.limit +} + +// Burst returns the maximum burst size. Burst is the maximum number of tokens +// that can be consumed in a single call to Allow, Reserve, or Wait, so higher +// Burst values allow more events to happen at once. +// A zero Burst allows no events, unless limit == Inf. +func (lim *Limiter) Burst() int { + lim.mu.Lock() + defer lim.mu.Unlock() + return lim.burst +} + +// TokensAt returns the number of tokens available at time t. +func (lim *Limiter) TokensAt(t time.Time) float64 { + lim.mu.Lock() + tokens := lim.advance(t) // does not mutate lim + lim.mu.Unlock() + return tokens +} + +// Tokens returns the number of tokens available now. +func (lim *Limiter) Tokens() float64 { + return lim.TokensAt(time.Now()) +} + +// NewLimiter returns a new Limiter that allows events up to rate r and permits +// bursts of at most b tokens. +func NewLimiter(r Limit, b int) *Limiter { + return &Limiter{ + limit: r, + burst: b, + tokens: float64(b), + } +} + +// Allow reports whether an event may happen now. +func (lim *Limiter) Allow() bool { + return lim.AllowN(time.Now(), 1) +} + +// AllowN reports whether n events may happen at time t. +// Use this method if you intend to drop / skip events that exceed the rate limit. +// Otherwise use Reserve or Wait. +func (lim *Limiter) AllowN(t time.Time, n int) bool { + return lim.reserveN(t, n, 0).ok +} + +// A Reservation holds information about events that are permitted by a Limiter to happen after a delay. +// A Reservation may be canceled, which may enable the Limiter to permit additional events. +type Reservation struct { + ok bool + lim *Limiter + tokens int + timeToAct time.Time + // This is the Limit at reservation time, it can change later. + limit Limit +} + +// OK returns whether the limiter can provide the requested number of tokens +// within the maximum wait time. If OK is false, Delay returns InfDuration, and +// Cancel does nothing. +func (r *Reservation) OK() bool { + return r.ok +} + +// Delay is shorthand for DelayFrom(time.Now()). +func (r *Reservation) Delay() time.Duration { + return r.DelayFrom(time.Now()) +} + +// InfDuration is the duration returned by Delay when a Reservation is not OK. +const InfDuration = time.Duration(math.MaxInt64) + +// DelayFrom returns the duration for which the reservation holder must wait +// before taking the reserved action. Zero duration means act immediately. +// InfDuration means the limiter cannot grant the tokens requested in this +// Reservation within the maximum wait time. +func (r *Reservation) DelayFrom(t time.Time) time.Duration { + if !r.ok { + return InfDuration + } + delay := r.timeToAct.Sub(t) + if delay < 0 { + return 0 + } + return delay +} + +// Cancel is shorthand for CancelAt(time.Now()). +func (r *Reservation) Cancel() { + r.CancelAt(time.Now()) +} + +// CancelAt indicates that the reservation holder will not perform the reserved action +// and reverses the effects of this Reservation on the rate limit as much as possible, +// considering that other reservations may have already been made. +func (r *Reservation) CancelAt(t time.Time) { + if !r.ok { + return + } + + r.lim.mu.Lock() + defer r.lim.mu.Unlock() + + if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(t) { + return + } + + // calculate tokens to restore + // The duration between lim.lastEvent and r.timeToAct tells us how many tokens were reserved + // after r was obtained. These tokens should not be restored. + restoreTokens := float64(r.tokens) - r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct)) + if restoreTokens <= 0 { + return + } + // advance time to now + tokens := r.lim.advance(t) + // calculate new number of tokens + tokens += restoreTokens + if burst := float64(r.lim.burst); tokens > burst { + tokens = burst + } + // update state + r.lim.last = t + r.lim.tokens = tokens + if r.timeToAct.Equal(r.lim.lastEvent) { + prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens))) + if !prevEvent.Before(t) { + r.lim.lastEvent = prevEvent + } + } +} + +// Reserve is shorthand for ReserveN(time.Now(), 1). +func (lim *Limiter) Reserve() *Reservation { + return lim.ReserveN(time.Now(), 1) +} + +// ReserveN returns a Reservation that indicates how long the caller must wait before n events happen. +// The Limiter takes this Reservation into account when allowing future events. +// The returned Reservation’s OK() method returns false if n exceeds the Limiter's burst size. +// Usage example: +// +// r := lim.ReserveN(time.Now(), 1) +// if !r.OK() { +// // Not allowed to act! Did you remember to set lim.burst to be > 0 ? +// return +// } +// time.Sleep(r.Delay()) +// Act() +// +// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events. +// If you need to respect a deadline or cancel the delay, use Wait instead. +// To drop or skip events exceeding rate limit, use Allow instead. +func (lim *Limiter) ReserveN(t time.Time, n int) *Reservation { + r := lim.reserveN(t, n, InfDuration) + return &r +} + +// Wait is shorthand for WaitN(ctx, 1). +func (lim *Limiter) Wait(ctx context.Context) (err error) { + return lim.WaitN(ctx, 1) +} + +// WaitN blocks until lim permits n events to happen. +// It returns an error if n exceeds the Limiter's burst size, the Context is +// canceled, or the expected wait time exceeds the Context's Deadline. +// The burst limit is ignored if the rate limit is Inf. +func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) { + // The test code calls lim.wait with a fake timer generator. + // This is the real timer generator. + newTimer := func(d time.Duration) (<-chan time.Time, func() bool, func()) { + timer := time.NewTimer(d) + return timer.C, timer.Stop, func() {} + } + + return lim.wait(ctx, n, time.Now(), newTimer) +} + +// wait is the internal implementation of WaitN. +func (lim *Limiter) wait(ctx context.Context, n int, t time.Time, newTimer func(d time.Duration) (<-chan time.Time, func() bool, func())) error { + lim.mu.Lock() + burst := lim.burst + limit := lim.limit + lim.mu.Unlock() + + if n > burst && limit != Inf { + return fmt.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", n, burst) + } + // Check if ctx is already cancelled + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + // Determine wait limit + waitLimit := InfDuration + if deadline, ok := ctx.Deadline(); ok { + waitLimit = deadline.Sub(t) + } + // Reserve + r := lim.reserveN(t, n, waitLimit) + if !r.ok { + return fmt.Errorf("rate: Wait(n=%d) would exceed context deadline", n) + } + // Wait if necessary + delay := r.DelayFrom(t) + if delay == 0 { + return nil + } + ch, stop, advance := newTimer(delay) + defer stop() + advance() // only has an effect when testing + select { + case <-ch: + // We can proceed. + return nil + case <-ctx.Done(): + // Context was canceled before we could proceed. Cancel the + // reservation, which may permit other events to proceed sooner. + r.Cancel() + return ctx.Err() + } +} + +// SetLimit is shorthand for SetLimitAt(time.Now(), newLimit). +func (lim *Limiter) SetLimit(newLimit Limit) { + lim.SetLimitAt(time.Now(), newLimit) +} + +// SetLimitAt sets a new Limit for the limiter. The new Limit, and Burst, may be violated +// or underutilized by those which reserved (using Reserve or Wait) but did not yet act +// before SetLimitAt was called. +func (lim *Limiter) SetLimitAt(t time.Time, newLimit Limit) { + lim.mu.Lock() + defer lim.mu.Unlock() + + tokens := lim.advance(t) + + lim.last = t + lim.tokens = tokens + lim.limit = newLimit +} + +// SetBurst is shorthand for SetBurstAt(time.Now(), newBurst). +func (lim *Limiter) SetBurst(newBurst int) { + lim.SetBurstAt(time.Now(), newBurst) +} + +// SetBurstAt sets a new burst size for the limiter. +func (lim *Limiter) SetBurstAt(t time.Time, newBurst int) { + lim.mu.Lock() + defer lim.mu.Unlock() + + tokens := lim.advance(t) + + lim.last = t + lim.tokens = tokens + lim.burst = newBurst +} + +// reserveN is a helper method for AllowN, ReserveN, and WaitN. +// maxFutureReserve specifies the maximum reservation wait duration allowed. +// reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN. +func (lim *Limiter) reserveN(t time.Time, n int, maxFutureReserve time.Duration) Reservation { + lim.mu.Lock() + defer lim.mu.Unlock() + + if lim.limit == Inf { + return Reservation{ + ok: true, + lim: lim, + tokens: n, + timeToAct: t, + } + } + + tokens := lim.advance(t) + + // Calculate the remaining number of tokens resulting from the request. + tokens -= float64(n) + + // Calculate the wait duration + var waitDuration time.Duration + if tokens < 0 { + waitDuration = lim.limit.durationFromTokens(-tokens) + } + + // Decide result + ok := n <= lim.burst && waitDuration <= maxFutureReserve + + // Prepare reservation + r := Reservation{ + ok: ok, + lim: lim, + limit: lim.limit, + } + if ok { + r.tokens = n + r.timeToAct = t.Add(waitDuration) + + // Update state + lim.last = t + lim.tokens = tokens + lim.lastEvent = r.timeToAct + } + + return r +} + +// advance calculates and returns an updated number of tokens for lim +// resulting from the passage of time. +// lim is not changed. +// advance requires that lim.mu is held. +func (lim *Limiter) advance(t time.Time) (newTokens float64) { + last := lim.last + if t.Before(last) { + last = t + } + + // Calculate the new number of tokens, due to time that passed. + elapsed := t.Sub(last) + delta := lim.limit.tokensFromDuration(elapsed) + tokens := lim.tokens + delta + if burst := float64(lim.burst); tokens > burst { + tokens = burst + } + return tokens +} + +// durationFromTokens is a unit conversion function from the number of tokens to the duration +// of time it takes to accumulate them at a rate of limit tokens per second. +func (limit Limit) durationFromTokens(tokens float64) time.Duration { + if limit <= 0 { + return InfDuration + } + + duration := (tokens / float64(limit)) * float64(time.Second) + + // Cap the duration to the maximum representable int64 value, to avoid overflow. + if duration > float64(math.MaxInt64) { + return InfDuration + } + + return time.Duration(duration) +} + +// tokensFromDuration is a unit conversion function from a time duration to the number of tokens +// which could be accumulated during that duration at a rate of limit tokens per second. +func (limit Limit) tokensFromDuration(d time.Duration) float64 { + if limit <= 0 { + return 0 + } + return d.Seconds() * float64(limit) +} diff --git a/vendor/github.com/baidubce/bce-sdk-go/rate/sometimes.go b/vendor/github.com/baidubce/bce-sdk-go/rate/sometimes.go new file mode 100644 index 0000000000000..9b83932692f0f --- /dev/null +++ b/vendor/github.com/baidubce/bce-sdk-go/rate/sometimes.go @@ -0,0 +1,69 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rate + +import ( + "sync" + "time" +) + +// Sometimes will perform an action occasionally. The First, Every, and +// Interval fields govern the behavior of Do, which performs the action. +// A zero Sometimes value will perform an action exactly once. +// +// # Example: logging with rate limiting +// +// var sometimes = rate.Sometimes{First: 3, Interval: 10*time.Second} +// func Spammy() { +// sometimes.Do(func() { log.Info("here I am!") }) +// } +type Sometimes struct { + First int // if non-zero, the first N calls to Do will run f. + Every int // if non-zero, every Nth call to Do will run f. + Interval time.Duration // if non-zero and Interval has elapsed since f's last run, Do will run f. + + mu sync.Mutex + count int // number of Do calls + last time.Time // last time f was run +} + +// Do runs the function f as allowed by First, Every, and Interval. +// +// The model is a union (not intersection) of filters. The first call to Do +// always runs f. Subsequent calls to Do run f if allowed by First or Every or +// Interval. +// +// A non-zero First:N causes the first N Do(f) calls to run f. +// +// A non-zero Every:M causes every Mth Do(f) call, starting with the first, to +// run f. +// +// A non-zero Interval causes Do(f) to run f if Interval has elapsed since +// Do last ran f. +// +// Specifying multiple filters produces the union of these execution streams. +// For example, specifying both First:N and Every:M causes the first N Do(f) +// calls and every Mth Do(f) call, starting with the first, to run f. See +// Examples for more. +// +// If Do is called multiple times simultaneously, the calls will block and run +// serially. Therefore, Do is intended for lightweight operations. +// +// Because a call to Do may block until f returns, if f causes Do to be called, +// it will deadlock. +func (s *Sometimes) Do(f func()) { + s.mu.Lock() + defer s.mu.Unlock() + if s.count == 0 || + (s.First > 0 && s.count < s.First) || + (s.Every > 0 && s.count%s.Every == 0) || + (s.Interval > 0 && time.Since(s.last) >= s.Interval) { + f() + if s.Interval > 0 { + s.last = time.Now() + } + } + s.count++ +} diff --git a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/model.go b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/model.go index 9bf4d424a8053..86a2a05592687 100644 --- a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/model.go +++ b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/model.go @@ -106,10 +106,26 @@ type AclRefererType struct { StringEquals []string `json:"stringEquals,omitempty"` } +type AclTimeCond struct { + DateLessThan string `json:"dateLessThan,omitempty"` + DateLessThanEquals string `json:"dateLessThanEquals,omitempty"` + DateGreaterThan string `json:"dateGreaterThan,omitempty"` + DateGreaterThanEquals string `json:"dateGreaterThanEquals,omitempty"` +} + +type AclUserAgentType struct { + StringLike []string `json:"stringLike,omitempty"` + StringEquals []string `json:"stringEquals,omitempty"` +} + type AclCondType struct { - IpAddress []string `json:"ipAddress,omitempty"` - Referer AclRefererType `json:"referer,omitempty"` - VpcId []string `json:"vpcId,omitempty"` + IpAddress []string `json:"ipAddress,omitempty"` + NotIpAddress []string `json:"notIpAddress,omitempty"` + Referer AclRefererType `json:"referer,omitempty"` + SecureTransport bool `json:"secureTransport,omitempty"` + CurrentTime AclTimeCond `json:"currentTime,omitempty"` + UserAgent AclUserAgentType `json:"userAgent,omitempty"` + VpcId []string `json:"vpcId,omitempty"` } // GrantType defines the grant struct in ACL setting @@ -293,6 +309,7 @@ type PutObjectArgs struct { ContentEncoding string ForbidOverwrite bool Encryption SSEHeaders + ContentCrc64ECMA string // please set other header/params of http request By Option // alternative Options please refer to service/bos/api/option.go } @@ -332,6 +349,7 @@ type CopyObjectArgs struct { ContentCrc32c string ContentCrc32cFlag bool ObjectExpires int + ContentCrc64ECMA string // please set other header/params of http request By Option // alternative Options please refer to service/bos/api/option.go } @@ -348,6 +366,7 @@ type MultiCopyObjectArgs struct { GrantFullControl []string ObjectExpires int UserMeta map[string]string + ContentCrc64ECMA string } type CallbackResult struct { @@ -361,6 +380,7 @@ type PutObjectResult struct { StorageClass string `json:"-"` VersionId string `json:"-"` ServerSideEncryption string `json:"-"` + ContentCrc64ECMA string `json:"-"` } type PostObjectResult struct { @@ -379,11 +399,6 @@ type CopyObjectResult struct { RequestId string `json:"requestId,omitempty"` } -type SetObjectMetaArgs struct { - ObjectMeta - ObjectExpires int -} - type ObjectMeta struct { CacheControl string ContentDisposition string @@ -409,6 +424,7 @@ type ObjectMeta struct { Encryption SSEHeaders RetentionDate string objectTagCount int64 + ContentCrc64ECMA string } // GetObjectResult defines the result data of the get object api. @@ -514,6 +530,7 @@ type AppendObjectArgs struct { ContentCrc32cFlag bool ObjectExpires int ContentEncoding string + ContentCrc64ECMA string } // AppendObjectResult defines the result data structure for appending object. @@ -523,6 +540,7 @@ type AppendObjectResult struct { ContentCrc32 string ETag string ContentCrc32c string + ContentCrc64ECMA string } // DeleteObjectArgs defines the input args structure for a single object. @@ -578,6 +596,7 @@ type UploadPartArgs struct { TrafficLimit int64 ContentCrc32c string ContentCrc32cFlag bool + ContentCrc64ECMA string } // UploadPartCopyArgs defines the optional arguments of UploadPartCopy. @@ -590,6 +609,7 @@ type UploadPartCopyArgs struct { TrafficLimit int64 ContentCrc32c string ContentCrc32cFlag bool + ContentCrc64ECMA string } type PutSymlinkArgs struct { @@ -614,17 +634,19 @@ type CompleteMultipartUploadArgs struct { ContentCrc32c string `json:"-"` ContentCrc32cFlag bool `json:"-"` ObjectExpires int `json:"-"` + ContentCrc64ECMA string `json:"-"` } // CompleteMultipartUploadResult defines the result structure of CompleteMultipartUpload. type CompleteMultipartUploadResult struct { - Location string `json:"location"` - Bucket string `json:"bucket"` - Key string `json:"key"` - ETag string `json:"eTag"` - ContentCrc32 string `json:"-"` - ContentCrc32c string `json:"-"` - VersionId string `json:"-"` + Location string `json:"location"` + Bucket string `json:"bucket"` + Key string `json:"key"` + ETag string `json:"eTag"` + ContentCrc32 string `json:"-"` + ContentCrc32c string `json:"-"` + VersionId string `json:"-"` + ContentCrc64ECMA string `json:"-"` } // ListPartsArgs defines the input optional arguments of listing parts information. @@ -706,20 +728,34 @@ type EncryptionKey struct { } type PutBucketNotificationSt struct { - Id string `json:"id"` - Name string `json:"name"` - AppId string `json:"appId"` - Status string `json:"status"` - Encryption EncryptionKey `json:"encryption"` - Resources []string `json:"resources"` - Events []string `json:"events"` - Apps []PutBucketNotificationAppsSt `json:"apps"` + Id string `json:"id"` + Name string `json:"name"` + AppId string `json:"appId"` + Status string `json:"status"` + Encryption EncryptionKey `json:"encryption"` + Resources []string `json:"resources"` + Events []string `json:"events"` + Apps []PutBucketNotificationAppsSt `json:"apps"` + ContentType NotificationContentTypeSt `json:"contentType,omitempty"` + Quota NotificationQuotaSt `json:"quota,omitempty"` +} + +type NotificationQuotaSt struct { + QuotaDay int64 `json:"quotaDay,omitempty"` + QuotaSec float64 `json:"quotaSec,omitempty"` } type PutBucketNotificationAppsSt struct { Id string `json:"id"` EventUrl string `json:"eventUrl"` XVars string `json:"xVars"` + Cfc string `json:"cfc"` + XParams string `json:"xParams"` +} + +type NotificationContentTypeSt struct { + Extensions []string `json:"extensions,omitempty"` + MimeTypes []string `json:"mimeTypes,omitempty"` } type MirrorConfigurationRule struct { diff --git a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/multipart.go b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/multipart.go index 97059f8320c72..26a30d186d571 100644 --- a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/multipart.go +++ b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/multipart.go @@ -25,6 +25,7 @@ import ( "github.com/baidubce/bce-sdk-go/bce" "github.com/baidubce/bce-sdk-go/http" "github.com/baidubce/bce-sdk-go/util" + "github.com/baidubce/bce-sdk-go/util/log" ) // InitiateMultipartUpload - initiate a multipart upload to get a upload ID @@ -163,6 +164,7 @@ func UploadPart(cli bce.Client, bucket, object, uploadId string, partNumber int, http.BCE_CONTENT_CRC32: args.ContentCrc32, http.BCE_CONTENT_CRC32C: args.ContentCrc32c, http.BCE_CONTENT_CRC32C_FLAG: strconv.FormatBool(args.ContentCrc32cFlag), + http.BCE_CONTENT_CRC64ECMA: args.ContentCrc64ECMA, }) //set traffic-limit if args.TrafficLimit > 0 { @@ -260,6 +262,7 @@ func UploadPartFromBytes(cli bce.Client, bucket, object, uploadId string, partNu http.BCE_CONTENT_CRC32: args.ContentCrc32, http.BCE_CONTENT_CRC32C: args.ContentCrc32c, http.BCE_CONTENT_CRC32C_FLAG: strconv.FormatBool(args.ContentCrc32cFlag), + http.BCE_CONTENT_CRC64ECMA: args.ContentCrc64ECMA, }) //set traffic-limit if args.TrafficLimit > 0 { @@ -332,6 +335,7 @@ func UploadPartCopy(cli bce.Client, bucket, object, source, uploadId string, par http.BCE_COPY_SOURCE_IF_UNMODIFIED_SINCE: args.IfUnmodifiedSince, http.BCE_CONTENT_CRC32C: args.ContentCrc32c, http.BCE_CONTENT_CRC32C_FLAG: strconv.FormatBool(args.ContentCrc32cFlag), + http.BCE_CONTENT_CRC64ECMA: args.ContentCrc64ECMA, }) //set traffic-limit if args.TrafficLimit > 0 { @@ -357,6 +361,9 @@ func UploadPartCopy(cli bce.Client, bucket, object, source, uploadId string, par if err := resp.ParseJsonBody(result); err != nil { return nil, err } + if resp.Header(http.BCE_REQUEST_ID) != "" { + result.RequestId = resp.Header(http.BCE_REQUEST_ID) + } return result, nil } @@ -395,18 +402,13 @@ func CompleteMultipartUpload(cli bce.Client, bucket, object, uploadId string, bo return nil, err } } - if len(args.Process) != 0 { - req.SetHeader(http.BCE_PROCESS, args.Process) - } - if len(args.ContentCrc32) != 0 { - req.SetHeader(http.BCE_CONTENT_CRC32, args.ContentCrc32) - } - if len(args.ContentCrc32c) != 0 { - req.SetHeader(http.BCE_CONTENT_CRC32C, args.ContentCrc32c) - } - if args.ContentCrc32cFlag { - req.SetHeader(http.BCE_CONTENT_CRC32C_FLAG, strconv.FormatBool(args.ContentCrc32cFlag)) - } + setOptionalNullHeaders(req, map[string]string{ + http.BCE_PROCESS: args.Process, + http.BCE_CONTENT_CRC32: args.ContentCrc32, + http.BCE_CONTENT_CRC32C: args.ContentCrc32c, + http.BCE_CONTENT_CRC32C_FLAG: strconv.FormatBool(args.ContentCrc32cFlag), + http.BCE_CONTENT_CRC64ECMA: args.ContentCrc64ECMA, + }) if args.ObjectExpires > 0 { req.SetHeader(http.BCE_OBJECT_EXPIRES, fmt.Sprintf("%d", args.ObjectExpires)) } @@ -431,9 +433,10 @@ func CompleteMultipartUpload(cli bce.Client, bucket, object, uploadId string, bo getHeader(http.BCE_CONTENT_CRC32, &result.ContentCrc32), getHeader(http.BCE_CONTENT_CRC32C, &result.ContentCrc32c), getHeader(http.BCE_VERSION_ID, &result.VersionId), + getHeader(http.BCE_CONTENT_CRC64ECMA, &result.ContentCrc64ECMA), } if err := handleGetOptions(resp, getOptions); err != nil { - return nil, bce.NewBceClientError(fmt.Sprintf("Handle get options error: %s", err)) + log.Warnf("Handle get options error: %s", err) } if args != nil && args.ContentCrc32c != "" && args.ContentCrc32c != result.ContentCrc32c { errMsg := fmt.Sprintf(BOS_CRC32C_CHECK_ERROR_MSG, args.ContentCrc32c, result.ContentCrc32c) diff --git a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/object.go b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/object.go index b159180ab2851..ec9dae14085cc 100644 --- a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/object.go +++ b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/object.go @@ -37,6 +37,7 @@ import ( "github.com/baidubce/bce-sdk-go/bce" "github.com/baidubce/bce-sdk-go/http" "github.com/baidubce/bce-sdk-go/util" + "github.com/baidubce/bce-sdk-go/util/log" ) // PutObject - put the object from the string or the stream @@ -83,6 +84,7 @@ func PutObject(cli bce.Client, bucket, object string, body *bce.Body, args *PutO http.BCE_SERVER_SIDE_ENCRYPTION_KEY_ID: args.Encryption.SSEKmsKeyId, http.BCE_SERVER_SIDE_ENCRYPTION_KEY_MD5: args.Encryption.SSECKeyMD5, http.BCE_FORBID_OVERWRITE: strconv.FormatBool(args.ForbidOverwrite), + http.BCE_CONTENT_CRC64ECMA: args.ContentCrc64ECMA, }) if args.ObjectExpires > 0 { req.SetHeader(http.BCE_OBJECT_EXPIRES, fmt.Sprintf("%d", args.ObjectExpires)) @@ -166,6 +168,7 @@ func PutObject(cli bce.Client, bucket, object string, body *bce.Body, args *PutO getHeader(http.BCE_CONTENT_CRC32, &jsonBody.ContentCrc32), getHeader(http.BCE_CONTENT_CRC32C, &jsonBody.ContentCrc32c), getHeader(http.BCE_SERVER_SIDE_ENCRYPTION, &jsonBody.ServerSideEncryption), + getHeader(http.BCE_CONTENT_CRC64ECMA, &jsonBody.ContentCrc64ECMA), } if err := handleGetOptions(resp, getOptions); err != nil { return "", nil, bce.NewBceClientError(fmt.Sprintf("Handle get options error: %s", err)) @@ -394,6 +397,7 @@ func CopyObject(cli bce.Client, bucket, object, source string, args *CopyObjectA http.BCE_SERVER_SIDE_ENCRYPTION_KEY: args.Encryption.SSECKey, http.BCE_SERVER_SIDE_ENCRYPTION_KEY_ID: args.Encryption.SSEKmsKeyId, http.BCE_SERVER_SIDE_ENCRYPTION_KEY_MD5: args.Encryption.SSECKeyMD5, + http.BCE_CONTENT_CRC64ECMA: args.ContentCrc64ECMA, }) if args.ObjectExpires > 0 { req.SetHeader(http.BCE_OBJECT_EXPIRES, fmt.Sprintf("%d", args.ObjectExpires)) @@ -457,6 +461,9 @@ func CopyObject(cli bce.Client, bucket, object, source string, args *CopyObjectA if resp.Header(http.BCE_VERSION_ID) != "" { jsonBody.VersionId = resp.Header(http.BCE_VERSION_ID) } + if resp.Header(http.BCE_REQUEST_ID) != "" { + jsonBody.RequestId = resp.Header(http.BCE_REQUEST_ID) + } return jsonBody, nil } @@ -510,55 +517,45 @@ func GetObject(cli bce.Client, bucket, object string, ctx *BosContext, args map[ if resp.IsFail() { return nil, resp.ServiceError() } - headers := resp.Headers() + result := &GetObjectResult{} - if val, ok := headers[http.CACHE_CONTROL]; ok { - result.CacheControl = val - } - if val, ok := headers[http.CONTENT_DISPOSITION]; ok { - result.ContentDisposition = val - } - if val, ok := headers[http.CONTENT_LENGTH]; ok { - if length, err := strconv.ParseInt(val, 10, 64); err == nil { - result.ContentLength = length - } - } - if val, ok := headers[http.CONTENT_RANGE]; ok { - result.ContentRange = val - } - if val, ok := headers[http.CONTENT_TYPE]; ok { - result.ContentType = val - } - if val, ok := headers[http.CONTENT_MD5]; ok { - result.ContentMD5 = val - } - if val, ok := headers[http.EXPIRES]; ok { - result.Expires = val + getOptions := []GetOption{ + getHeader(http.CACHE_CONTROL, &result.CacheControl), + getHeader(http.CONTENT_DISPOSITION, &result.ContentDisposition), + getHeader(http.CONTENT_LENGTH, &result.ContentLength), + getHeader(http.CONTENT_RANGE, &result.ContentRange), + getHeader(http.CONTENT_TYPE, &result.ContentType), + getHeader(http.CONTENT_MD5, &result.ContentMD5), + getHeader(http.EXPIRES, &result.Expires), + getHeader(http.LAST_MODIFIED, &result.LastModified), + getHeader(http.CONTENT_LANGUAGE, &result.ContentLanguage), + getHeader(http.CONTENT_ENCODING, &result.ContentEncoding), + getHeader(http.BCE_CONTENT_SHA256, &result.ContentSha256), + getHeader(http.BCE_CONTENT_CRC32, &result.ContentCrc32), + getHeader(http.BCE_STORAGE_CLASS, &result.StorageClass), + getHeader(http.BCE_VERSION_ID, &result.VersionId), + getHeader(http.BCE_OBJECT_TYPE, &result.ObjectType), + getHeader(http.BCE_NEXT_APPEND_OFFSET, &result.NextAppendOffset), + getHeader(http.BCE_CONTENT_CRC32C, &result.ContentCrc32c), + getHeader(http.BCE_EXPIRATION_DATE, &result.ExpirationDate), + getHeader(http.BCE_SERVER_SIDE_ENCRYPTION, &result.Encryption.ServerSideEncryption), + getHeader(http.BCE_SERVER_SIDE_ENCRYPTION_KEY, &result.Encryption.SSECKey), + getHeader(http.BCE_SERVER_SIDE_ENCRYPTION_KEY_MD5, &result.Encryption.SSECKeyMD5), + getHeader(http.BCE_SERVER_SIDE_ENCRYPTION_KEY_ID, &result.Encryption.SSEKmsKeyId), + getHeader(http.BCE_OBJECT_RETENTION_DATE, &result.RetentionDate), + getHeader(http.BCE_TAGGING_COUNT, &result.objectTagCount), + getHeader(http.BCE_CONTENT_CRC64ECMA, &result.ContentCrc64ECMA), } - if val, ok := headers[http.LAST_MODIFIED]; ok { - result.LastModified = val + + if err := handleGetOptions(resp, getOptions); err != nil { + log.Warnf("Handle get options error: %s", err) } + + headers := resp.Headers() if val, ok := headers[http.ETAG]; ok { result.ETag = strings.Trim(val, "\"") } - if val, ok := headers[http.CONTENT_LANGUAGE]; ok { - result.ContentLanguage = val - } - if val, ok := headers[http.CONTENT_ENCODING]; ok { - result.ContentEncoding = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_CONTENT_SHA256)]; ok { - result.ContentSha256 = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_CONTENT_CRC32)]; ok { - result.ContentCrc32 = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_STORAGE_CLASS)]; ok { - result.StorageClass = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_VERSION_ID)]; ok { - result.VersionId = val - } + bcePrefix := toHttpHeaderKey(http.BCE_USER_METADATA_PREFIX) for k, v := range headers { if strings.Index(k, bcePrefix) == 0 { @@ -568,36 +565,6 @@ func GetObject(cli bce.Client, bucket, object string, ctx *BosContext, args map[ result.UserMeta[k[len(bcePrefix):]] = v } } - if val, ok := headers[toHttpHeaderKey(http.BCE_OBJECT_TYPE)]; ok { - result.ObjectType = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_NEXT_APPEND_OFFSET)]; ok { - result.NextAppendOffset = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_CONTENT_CRC32C)]; ok { - result.ContentCrc32c = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_EXPIRATION_DATE)]; ok { - result.ExpirationDate = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_SERVER_SIDE_ENCRYPTION)]; ok { - result.Encryption.ServerSideEncryption = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_SERVER_SIDE_ENCRYPTION_KEY)]; ok { - result.Encryption.SSECKey = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_SERVER_SIDE_ENCRYPTION_KEY_MD5)]; ok { - result.Encryption.SSECKeyMD5 = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_SERVER_SIDE_ENCRYPTION_KEY_ID)]; ok { - result.Encryption.SSEKmsKeyId = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_OBJECT_RETENTION_DATE)]; ok { - result.RetentionDate = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_TAGGING_COUNT)]; ok { - result.objectTagCount, _ = strconv.ParseInt(val, 10, 64) - } result.Body = resp.Body() return result, nil } @@ -631,56 +598,40 @@ func GetObjectMeta(cli bce.Client, bucket, object string, ctx *BosContext, optio } headers := resp.Headers() result := &GetObjectMetaResult{} - if val, ok := headers[http.CACHE_CONTROL]; ok { - result.CacheControl = val - } - if val, ok := headers[http.CONTENT_DISPOSITION]; ok { - result.ContentDisposition = val - } - if val, ok := headers[http.CONTENT_LENGTH]; ok { - if length, err := strconv.ParseInt(val, 10, 64); err == nil { - result.ContentLength = length - } - } - if val, ok := headers[http.CONTENT_RANGE]; ok { - result.ContentRange = val - } - if val, ok := headers[http.CONTENT_TYPE]; ok { - result.ContentType = val - } - if val, ok := headers[http.CONTENT_MD5]; ok { - result.ContentMD5 = val - } - if val, ok := headers[http.EXPIRES]; ok { - result.Expires = val + getOptions := []GetOption{ + getHeader(http.CACHE_CONTROL, &result.CacheControl), + getHeader(http.CONTENT_DISPOSITION, &result.ContentDisposition), + getHeader(http.CONTENT_LENGTH, &result.ContentLength), + getHeader(http.CONTENT_RANGE, &result.ContentRange), + getHeader(http.CONTENT_TYPE, &result.ContentType), + getHeader(http.CONTENT_MD5, &result.ContentMD5), + getHeader(http.EXPIRES, &result.Expires), + getHeader(http.LAST_MODIFIED, &result.LastModified), + getHeader(http.CONTENT_ENCODING, &result.ContentEncoding), + getHeader(http.BCE_CONTENT_SHA256, &result.ContentSha256), + getHeader(http.BCE_CONTENT_CRC32, &result.ContentCrc32), + getHeader(http.BCE_STORAGE_CLASS, &result.StorageClass), + getHeader(http.BCE_RESTORE, &result.BceRestore), + getHeader(http.BCE_VERSION_ID, &result.VersionId), + getHeader(http.BCE_OBJECT_TYPE, &result.ObjectType), + getHeader(http.BCE_NEXT_APPEND_OFFSET, &result.NextAppendOffset), + getHeader(http.BCE_CONTENT_CRC32C, &result.ContentCrc32c), + getHeader(http.BCE_EXPIRATION_DATE, &result.ExpirationDate), + getHeader(http.BCE_SERVER_SIDE_ENCRYPTION, &result.Encryption.ServerSideEncryption), + getHeader(http.BCE_SERVER_SIDE_ENCRYPTION_KEY, &result.Encryption.SSECKey), + getHeader(http.BCE_SERVER_SIDE_ENCRYPTION_KEY_MD5, &result.Encryption.SSECKeyMD5), + getHeader(http.BCE_SERVER_SIDE_ENCRYPTION_KEY_ID, &result.Encryption.SSEKmsKeyId), + getHeader(http.BCE_OBJECT_RETENTION_DATE, &result.RetentionDate), + getHeader(http.BCE_TAGGING_COUNT, &result.objectTagCount), + getHeader(http.BCE_CONTENT_CRC64ECMA, &result.ContentCrc64ECMA), } - if val, ok := headers[http.LAST_MODIFIED]; ok { - result.LastModified = val + + if err := handleGetOptions(resp, getOptions); err != nil { + log.Warnf("Handle get options error: %s", err) } if val, ok := headers[http.ETAG]; ok { result.ETag = strings.Trim(val, "\"") } - if val, ok := headers[http.CONTENT_ENCODING]; ok { - result.ContentEncoding = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_CONTENT_SHA256)]; ok { - result.ContentSha256 = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_CONTENT_CRC32)]; ok { - result.ContentCrc32 = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_STORAGE_CLASS)]; ok { - result.StorageClass = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_RESTORE)]; ok { - result.BceRestore = val - } - if val, ok := headers[http.BCE_OBJECT_TYPE]; ok { - result.BceObjectType = val - } - if val, ok := headers[http.BCE_VERSION_ID]; ok { - result.VersionId = val - } bcePrefix := toHttpHeaderKey(http.BCE_USER_METADATA_PREFIX) for k, v := range headers { if strings.Index(k, bcePrefix) == 0 { @@ -690,36 +641,6 @@ func GetObjectMeta(cli bce.Client, bucket, object string, ctx *BosContext, optio result.UserMeta[k[len(bcePrefix):]] = v } } - if val, ok := headers[toHttpHeaderKey(http.BCE_OBJECT_TYPE)]; ok { - result.ObjectType = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_NEXT_APPEND_OFFSET)]; ok { - result.NextAppendOffset = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_CONTENT_CRC32C)]; ok { - result.ContentCrc32c = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_EXPIRATION_DATE)]; ok { - result.ExpirationDate = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_SERVER_SIDE_ENCRYPTION)]; ok { - result.Encryption.ServerSideEncryption = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_SERVER_SIDE_ENCRYPTION_KEY)]; ok { - result.Encryption.SSECKey = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_SERVER_SIDE_ENCRYPTION_KEY_MD5)]; ok { - result.Encryption.SSECKeyMD5 = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_SERVER_SIDE_ENCRYPTION_KEY_ID)]; ok { - result.Encryption.SSEKmsKeyId = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_OBJECT_RETENTION_DATE)]; ok { - result.RetentionDate = val - } - if val, ok := headers[toHttpHeaderKey(http.BCE_TAGGING_COUNT)]; ok { - result.objectTagCount, _ = strconv.ParseInt(val, 10, 64) - } defer func() { resp.Body().Close() }() return result, nil } @@ -888,6 +809,7 @@ func AppendObject(cli bce.Client, bucket, object string, content *bce.Body, args http.BCE_CONTENT_CRC32C: args.ContentCrc32c, http.BCE_CONTENT_CRC32C_FLAG: strconv.FormatBool(args.ContentCrc32cFlag), http.CONTENT_ENCODING: args.ContentEncoding, + http.BCE_CONTENT_CRC64ECMA: args.ContentCrc64ECMA, }) if args.ObjectExpires > 0 { req.SetHeader(http.BCE_OBJECT_EXPIRES, fmt.Sprintf("%d", args.ObjectExpires)) @@ -926,8 +848,14 @@ func AppendObject(cli bce.Client, bucket, object string, content *bce.Body, args defer func() { resp.Body().Close() }() headers := resp.Headers() result := &AppendObjectResult{} - if val, ok := headers[http.CONTENT_MD5]; ok { - result.ContentMD5 = val + getOptions := []GetOption{ + getHeader(http.CONTENT_MD5, &result.ContentMD5), + getHeader(http.BCE_CONTENT_CRC32, &result.ContentCrc32), + getHeader(http.BCE_CONTENT_CRC32C, &result.ContentCrc32c), + getHeader(http.BCE_CONTENT_CRC64ECMA, &result.ContentCrc64ECMA), + } + if err := handleGetOptions(resp, getOptions); err != nil { + log.Warnf("Handle get options error: %s", err) } if val, ok := headers[toHttpHeaderKey(http.BCE_NEXT_APPEND_OFFSET)]; ok { nextOffset, offsetErr := strconv.ParseInt(val, 10, 64) @@ -938,15 +866,9 @@ func AppendObject(cli bce.Client, bucket, object string, content *bce.Body, args } else { result.NextAppendOffset = content.Size() } - if val, ok := headers[toHttpHeaderKey(http.BCE_CONTENT_CRC32)]; ok { - result.ContentCrc32 = val - } if val, ok := headers[http.ETAG]; ok { result.ETag = strings.Trim(val, "\"") } - if val, ok := headers[toHttpHeaderKey(http.BCE_CONTENT_CRC32C)]; ok { - result.ContentCrc32c = val - } return result, nil } @@ -1060,6 +982,10 @@ func GeneratePresignedUrlInternal(conf *bce.BceClientConfiguration, signer auth. if len(method) == 0 { method = http.GET } + if method == http.GET && (object == "" || object == "v1") { + log.Warnf("objectKey '%s' is invalid, cannot generate presigned url.", object) + return "" + } req.SetMethod(method) req.SetEndpoint(conf.Endpoint) if req.Protocol() == "" { diff --git a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/option.go b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/option.go index 39070349c1001..f0df08b4ce12d 100644 --- a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/option.go +++ b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/option.go @@ -447,11 +447,22 @@ func handleGetOptions(response *BosResponse, options []GetOption) error { *vReal = val } if vReal, ok := v.(*bool); ok { - *vReal, _ = strconv.ParseBool(val) + vbool, err := strconv.ParseBool(val) + if err == nil { + *vReal = vbool + } } if vReal, ok := v.(*int); ok { - vint, _ := strconv.ParseInt(val, 10, 64) - *vReal = int(vint) + vint, err := strconv.ParseInt(val, 10, 64) + if err == nil { + *vReal = int(vint) + } + } + if vReal, ok := v.(*int64); ok { + vint64, err := strconv.ParseInt(val, 10, 64) + if err == nil { + *vReal = vint64 + } } if vReal, ok := v.(*[]string); ok { *vReal = strings.Split(val, ",") diff --git a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/util.go b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/util.go index b75cc00cdec00..1f8e90ec6e206 100644 --- a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/util.go +++ b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/util.go @@ -33,7 +33,6 @@ import ( const ( METADATA_DIRECTIVE_COPY = "copy" METADATA_DIRECTIVE_REPLACE = "replace" - METADATA_DIRECTIVE_UPDATE = "update" STORAGE_CLASS_STANDARD = "STANDARD" STORAGE_CLASS_STANDARD_IA = "STANDARD_IA" @@ -148,8 +147,7 @@ func getCnameUri(uri string) string { func validMetadataDirective(val string) bool { if val == METADATA_DIRECTIVE_COPY || - val == METADATA_DIRECTIVE_REPLACE || - val == METADATA_DIRECTIVE_UPDATE { + val == METADATA_DIRECTIVE_REPLACE { return true } return false diff --git a/vendor/github.com/baidubce/bce-sdk-go/services/bos/client.go b/vendor/github.com/baidubce/bce-sdk-go/services/bos/client.go index 1a319113d00f0..96788de62b53d 100644 --- a/vendor/github.com/baidubce/bce-sdk-go/services/bos/client.go +++ b/vendor/github.com/baidubce/bce-sdk-go/services/bos/client.go @@ -70,6 +70,7 @@ type BosClientConfiguration struct { PathStyleEnable bool DisableKeepAlives bool NoVerifySSL bool + ExclusiveHTTPClient bool retryPolicy bce.RetryPolicy DialTimeout *time.Duration // timeout of building a connection KeepAlive *time.Duration // interval between keep-alive probes for an active connection @@ -80,18 +81,21 @@ type BosClientConfiguration struct { ResponseHeaderTimeout *time.Duration // http.Transport.ResponseHeaderTimeout HTTPClientTimeout *time.Duration // http.Client.Timeout HTTPClient *http.Client // customized http client to send request + UploadRatelimit *int64 // the limit of upload rate, unit:KB/s + DownloadRatelimit *int64 // the limit of download rate, unit:KB/s } func NewBosClientConfig(ak, sk, endpoint string) *BosClientConfiguration { return &BosClientConfiguration{ - Ak: ak, - Sk: sk, - Endpoint: endpoint, - RedirectDisabled: false, - PathStyleEnable: false, - DisableKeepAlives: false, - NoVerifySSL: false, - retryPolicy: bce.DEFAULT_RETRY_POLICY, + Ak: ak, + Sk: sk, + Endpoint: endpoint, + RedirectDisabled: false, + PathStyleEnable: false, + DisableKeepAlives: false, + NoVerifySSL: false, + ExclusiveHTTPClient: true, + retryPolicy: bce.DEFAULT_RETRY_POLICY, } } @@ -180,6 +184,21 @@ func (cfg *BosClientConfiguration) WithHttpClient(val http.Client) *BosClientCon return cfg } +func (cfg *BosClientConfiguration) WithUploadRateLimit(val int64) *BosClientConfiguration { + cfg.UploadRatelimit = &val + return cfg +} + +func (cfg *BosClientConfiguration) WithDownloadRateLimit(val int64) *BosClientConfiguration { + cfg.DownloadRatelimit = &val + return cfg +} + +func (cfg *BosClientConfiguration) WithExclusiveHTTPClient(val bool) *BosClientConfiguration { + cfg.ExclusiveHTTPClient = val + return cfg +} + // NewClient make the BOS service client with default configuration. // Use `cli.Config.xxx` to access the config or change it to non-default value. func NewClient(ak, sk, endpoint string) (*Client, error) { @@ -218,6 +237,7 @@ func NewStsClient(ak, sk, endpoint string, expiration int) (*Client, error) { func NewClientWithConfig(config *BosClientConfiguration) (*Client, error) { var credentials *auth.BceCredentials + var bceClient *bce.BceClient var err error ak, sk, endpoint := config.Ak, config.Sk, config.Endpoint if len(ak) == 0 && len(sk) == 0 { // to support public-read-write request @@ -257,14 +277,23 @@ func NewClientWithConfig(config *BosClientConfiguration) (*Client, error) { ResponseHeaderTimeout: config.ResponseHeaderTimeout, HTTPClientTimeout: config.HTTPClientTimeout, HTTPClient: config.HTTPClient, + UploadRatelimit: config.UploadRatelimit, + DownloadRatelimit: config.DownloadRatelimit, + } + if config.HTTPClientTimeout != nil { + defaultConf.ConnectionTimeoutInMillis = int(config.HTTPClientTimeout.Milliseconds()) } v1Signer := &auth.BceV1Signer{} defaultContext := &api.BosContext{ PathStyleEnable: config.PathStyleEnable, } - client := &Client{bce.NewBceClientWithTimeout(defaultConf, v1Signer), - DEFAULT_MAX_PARALLEL, DEFAULT_MULTIPART_SIZE, defaultContext} - return client, nil + if config.ExclusiveHTTPClient || config.HTTPClient != nil || + config.UploadRatelimit != nil || config.DownloadRatelimit != nil { + bceClient = bce.NewBceClientWithExclusiveHTTPClient(defaultConf, v1Signer) + } else { + bceClient = bce.NewBceClientWithTimeout(defaultConf, v1Signer) + } + return &Client{bceClient, DEFAULT_MAX_PARALLEL, DEFAULT_MULTIPART_SIZE, defaultContext}, nil } // ListBuckets - list all buckets @@ -1465,45 +1494,6 @@ func (c *Client) GetObjectToFileWithContext(ctx context.Context, bucket, object, return nil } -// SetObjectMeta - set the given object metadata -// -// PARAMS: -// - bucket: the name of the bucket -// - object: the name of the object -// - args: new object meta value for changing -// - options: option func to set various requeset headers -// -// RETURNS: -// - error: any error if it occurs -func (c *Client) SetObjectMeta(bucket, object string, args *api.SetObjectMetaArgs, - options ...api.Option) error { - cpArgs := &api.CopyObjectArgs{ - ObjectMeta: args.ObjectMeta, - ObjectExpires: args.ObjectExpires, - MetadataDirective: api.METADATA_DIRECTIVE_UPDATE, - } - source := fmt.Sprintf("/%s/%s", bucket, object) - _, err := api.CopyObject(c, bucket, object, source, cpArgs, c.BosContext, options...) - return err -} - -// SetObjectMetaWithContext - support to cancel request by context.Context -func (c *Client) SetObjectMetaWithContext(ctx context.Context, bucket, object string, - args *api.SetObjectMetaArgs, options ...api.Option) error { - cpArgs := &api.CopyObjectArgs{ - ObjectMeta: args.ObjectMeta, - ObjectExpires: args.ObjectExpires, - MetadataDirective: api.METADATA_DIRECTIVE_UPDATE, - } - source := fmt.Sprintf("/%s/%s", bucket, object) - bosContext := &api.BosContext{ - PathStyleEnable: c.BosContext.PathStyleEnable, - Ctx: ctx, - } - _, err := api.CopyObject(c, bucket, object, source, cpArgs, bosContext, options...) - return err -} - // GetObjectMeta - get the given object metadata // // PARAMS: @@ -2624,8 +2614,13 @@ func (c *Client) ParallelUpload(bucket string, object string, filename string, c } completeArgs := &api.CompleteMultipartUploadArgs{ - Parts: partEtags, - ObjectExpires: args.ObjectExpires, + Parts: partEtags, + } + + if args != nil { + if args.ObjectExpires > 0 { + completeArgs.ObjectExpires = args.ObjectExpires + } } completeMultipartUploadResult, err := c.CompleteMultipartUploadFromStruct(bucket, object, initiateMultipartUploadResult.UploadId, completeArgs) @@ -2666,6 +2661,10 @@ func (c *Client) parallelPartUpload(bucket string, object string, filename strin partSize = (partSize + MULTIPART_ALIGN - 1) / MULTIPART_ALIGN * MULTIPART_ALIGN partNum = (fileSize + partSize - 1) / partSize } + // 文件大小为 0 时,至少执行一次 UploadPart + if partNum == 0 { + partNum = 1 + } parallelChan := make(chan int, c.MaxParallel) @@ -2835,6 +2834,9 @@ func (c *Client) ParallelCopy(srcBucketName string, srcObjectName string, if args.ObjectExpires > 0 { completeArgs.ObjectExpires = args.ObjectExpires } + if len(args.ContentCrc64ECMA) > 0 { + completeArgs.ContentCrc64ECMA = args.ContentCrc64ECMA + } } completeMultipartUploadResult, err := c.CompleteMultipartUploadFromStruct(destBucketName, destObjectName, initiateMultipartUploadResult.UploadId, completeArgs) diff --git a/vendor/modules.txt b/vendor/modules.txt index 72d4ec014a257..c77edc4d9b87b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -564,11 +564,12 @@ github.com/axiomhq/hyperloglog # github.com/aymanbagabas/go-osc52/v2 v2.0.1 ## explicit; go 1.16 github.com/aymanbagabas/go-osc52/v2 -# github.com/baidubce/bce-sdk-go v0.9.245 +# github.com/baidubce/bce-sdk-go v0.9.251 ## explicit; go 1.11 github.com/baidubce/bce-sdk-go/auth github.com/baidubce/bce-sdk-go/bce github.com/baidubce/bce-sdk-go/http +github.com/baidubce/bce-sdk-go/rate github.com/baidubce/bce-sdk-go/services/bos github.com/baidubce/bce-sdk-go/services/bos/api github.com/baidubce/bce-sdk-go/services/sts