From a68f819ba19a847a64b88d5dbae80dbaced9938a Mon Sep 17 00:00:00 2001 From: cqc Date: Thu, 8 Jun 2023 11:47:48 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=96=87=E4=BB=B6=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E4=B8=8B=E8=BD=BD=E9=99=90=E9=80=9F=E3=80=81=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E7=94=9F=E6=88=904=E7=A7=8D=E5=A4=96=E9=93=BE?= =?UTF-8?q?=E4=B8=94=E6=94=AF=E6=8C=81=E9=99=90=E9=80=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- aws/request.go | 13 +---- internal/signer/v2/v2.go | 3 +- internal/signer/v4/v4.go | 2 +- internal/signer/v4/v4_test.go | 27 +--------- service/s3/api.go | 77 +++++++++++---------------- service/s3/cors.go | 20 -------- service/s3/lifecycle.go | 13 ----- test/objectsample_test.go | 97 +++++++++++------------------------ 8 files changed, 62 insertions(+), 190 deletions(-) diff --git a/aws/request.go b/aws/request.go index a6d8062..b6b1ff6 100755 --- a/aws/request.go +++ b/aws/request.go @@ -17,7 +17,7 @@ type Request struct { *Service Handlers Handlers Time time.Time - ExpireTime time.Duration + ExpireTime int64 Operation *Operation HTTPRequest *http.Request HTTPResponse *http.Response @@ -135,17 +135,6 @@ func (r *Request) SetReaderBody(reader io.ReadSeeker) { r.Body = reader } -// Presign returns the request's signed URL. Error will be returned -// if the signing fails. -func (r *Request) Presign(expireTime time.Duration) (string, error) { - r.ExpireTime = expireTime - r.Sign() - if r.Error != nil { - return "", r.Error - } - return r.HTTPRequest.URL.String(), nil -} - // Build will build the request's object so it can be signed and sent // to the service. Build will also validate all the request's parameters. // Anny additional build Handlers set on this request will be run diff --git a/internal/signer/v2/v2.go b/internal/signer/v2/v2.go index 1623c7e..3e2b954 100644 --- a/internal/signer/v2/v2.go +++ b/internal/signer/v2/v2.go @@ -62,7 +62,7 @@ type signer struct { Service *aws.Service Request *http.Request Time time.Time - ExpireTime time.Duration + ExpireTime int64 ServiceName string Region string CredValues credentials.Value @@ -305,7 +305,6 @@ func (v2 *signer) buildStringToSign() { if len(typelist) > 0 { contenttype = v2.Request.Header["Content-Type"][0] } - signItems := []string{v2.Request.Method, md5, contenttype} if v2.isPresign { signItems = append(signItems, v2.Query["Expires"][0]) diff --git a/internal/signer/v4/v4.go b/internal/signer/v4/v4.go index 0ce8515..a048ab9 100755 --- a/internal/signer/v4/v4.go +++ b/internal/signer/v4/v4.go @@ -36,7 +36,7 @@ var ignoredHeaders = map[string]bool{ type signer struct { Request *http.Request Time time.Time - ExpireTime time.Duration + ExpireTime int64 ServiceName string Region string CredValues credentials.Value diff --git a/internal/signer/v4/v4_test.go b/internal/signer/v4/v4_test.go index fd1e7f2..ac7def5 100755 --- a/internal/signer/v4/v4_test.go +++ b/internal/signer/v4/v4_test.go @@ -175,7 +175,7 @@ func TestIgnorePreResignRequestWithValidCreds(t *testing.T) { nil, nil, ) - r.ExpireTime = time.Minute * 10 + r.ExpireTime = 10 Sign(r) sig := r.HTTPRequest.Header.Get("X-Amz-Signature") @@ -205,31 +205,6 @@ func TestResignRequestExpiredCreds(t *testing.T) { assert.NotEqual(t, querySig, r.HTTPRequest.Header.Get("Authorization")) } -func TestPreResignRequestExpiredCreds(t *testing.T) { - provider := &credentials.StaticProvider{credentials.Value{"AKID", "SECRET", "SESSION"}} - creds := credentials.NewCredentials(provider) - r := aws.NewRequest( - aws.NewService(&aws.Config{Credentials: creds}), - &aws.Operation{ - Name: "BatchGetItem", - HTTPMethod: "POST", - HTTPPath: "/", - }, - nil, - nil, - ) - r.ExpireTime = time.Minute * 10 - - Sign(r) - querySig := r.HTTPRequest.URL.Query().Get("X-Amz-Signature") - - creds.Expire() - r.Time = time.Now().Add(time.Hour * 48) - - Sign(r) - assert.NotEqual(t, querySig, r.HTTPRequest.URL.Query().Get("X-Amz-Signature")) -} - func BenchmarkPresignRequest(b *testing.B) { signer := buildSigner("dynamodb", "us-east-1", time.Now(), 300*time.Second, "{}") for i := 0; i < b.N; i++ { diff --git a/service/s3/api.go b/service/s3/api.go index 5a2ec9c..c8d8857 100755 --- a/service/s3/api.go +++ b/service/s3/api.go @@ -18,10 +18,8 @@ import ( "io" "io/ioutil" "net/http" - "net/url" "os" "sort" - "strconv" "strings" "sync" "time" @@ -928,15 +926,6 @@ func (c *S3) GetObjectToFile(bucket, objectKey, filePath, Range string) error { return os.Rename(tempFilePath, filePath) } -func (c *S3) GetObjectPresignedUrl(input *GetObjectInput, expires time.Duration) (*url.URL, error) { - req, _ := c.GetObjectRequest(input) - req.ExpireTime = expires - err := req.Sign() - if *input.TrafficLimit > 0 { - req.HTTPRequest.URL.RawQuery += "&x-kss-traffic-limit=" + strconv.FormatInt(*input.TrafficLimit, 10) - } - return req.HTTPRequest.URL, err -} var opGetObject *aws.Operation @@ -1605,42 +1594,41 @@ func (c *S3) PutBucketWebsiteRequest(input *PutBucketWebsiteInput) (req *aws.Req return } +type HTTPMethod string + +const ( + PUT HTTPMethod = "PUT" + GET HTTPMethod = "GET" + DELETE HTTPMethod = "DELETE" + HEAD HTTPMethod = "HEAD" +) + +type metadataGeneratePresignedUrlInput struct { + SDKShapeTraits bool `type:"structure" payload:"GeneratePresignedUrlInput"` +} type GeneratePresignedUrlInput struct { - HTTPMethod string + // The canned ACL to apply to the object. ACL *string `location:"header" locationName:"x-amz-acl" type:"string"` Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - // Specifies what content encodings have been applied to the object and thus - // what decoding mechanisms must be applied to obtain the media-type referenced - // by the Content-Type header field. - ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` - - // The language the content is in. - ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"` - - // Size of the body in bytes. This parameter is useful when the size of the - // body cannot be determined automatically. - ContentLength *int64 `location:"header" locationName:"Content-Length" type:"integer"` - // A standard MIME type describing the format of the object data. ContentType *string `location:"header" locationName:"Content-Type" type:"string"` - // The date and time at which the object is no longer cacheable. - Expires time.Duration - Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` // A map of metadata to store with the object in S3. Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` - CallbackUrl *string `location:"header" locationName:"x-kss-callbackurl" type:"string"` - CallbackBody *string `location:"header" locationName:"x-kss-callbackbody" type:"string"` + TrafficLimit *int64 `location:"querystring" locationName:"x-kss-traffic-limit" type:"integer"` - TrafficLimit *int64 `location:"header" locationName:"x-kss-traffic-limit" type:"integer"` + HTTPMethod HTTPMethod - metadataPutObjectInput `json:"-" xml:"-"` + // The date and time at which the object is no longer cacheable. + Expires int64 + + metadataGeneratePresignedUrlInput `json:"-" xml:"-"` } type GeneratePresignedUrlOutput struct { url *string @@ -1717,31 +1705,24 @@ func (c *S3) PutObject(input *PutObjectInput) (*PutObjectOutput, error) { func (c *S3) GeneratePresignedUrlInput(input *GeneratePresignedUrlInput) (url string) { opPutObject = &aws.Operation{ - Name: "PutObject", - HTTPMethod: input.HTTPMethod, + HTTPMethod: string(input.HTTPMethod), HTTPPath: "/{Bucket}/{Key+}", } - output := &GeneratePresignedUrlOutput{} req := c.newRequest(opPutObject, input, output) - req.ExpireTime = input.Expires - req.Sign() - if *input.TrafficLimit > 0 { - req.HTTPRequest.URL.RawQuery += "&x-kss-traffic-limit=" + strconv.FormatInt(*input.TrafficLimit, 10) + now := time.Now().Unix() + if c.Config.SignerVersion == "V2" { + req.ExpireTime = input.Expires + now + } else { + req.ExpireTime = input.Expires } + req.Sign() + //if input.TrafficLimit != nil && *input.TrafficLimit > 0 { + // req.HTTPRequest.URL.RawQuery += "&x-amz-traffic-limit=" + strconv.FormatInt(*input.TrafficLimit, 10) + //} return req.HTTPRequest.URL.String() } -func (c *S3) PutObjectPresignedUrl(input *PutObjectInput, expires time.Duration) (*url.URL, error) { - req, _ := c.PutObjectRequest(input) - req.ExpireTime = expires - err := req.Sign() - if *input.TrafficLimit > 0 { - req.HTTPRequest.URL.RawQuery += "&x-kss-traffic-limit=" + strconv.FormatInt(*input.TrafficLimit, 10) - } - return req.HTTPRequest.URL, err -} - func (c *S3) PutObjectPreassignedInput(input *PutObjectInput) (*http.Request, error) { req, _ := c.PutObjectRequest(input) err := req.Sign() diff --git a/service/s3/cors.go b/service/s3/cors.go index a3fa46e..d3d01d9 100644 --- a/service/s3/cors.go +++ b/service/s3/cors.go @@ -2,8 +2,6 @@ package s3 import ( "github.com/ks3sdklib/aws-sdk-go/aws" - "net/url" - "time" ) // GetBucketCORSRequest generates a request for the GetBucketCORS operation. @@ -35,12 +33,6 @@ func (c *S3) GetBucketCORS(input *GetBucketCORSInput) (*GetBucketCORSOutput, err err := req.Send() return out, err } -func (c *S3) GetBucketCORSPresignedUrl(input *GetBucketCORSInput, expires time.Duration) (*url.URL, error) { - req, _ := c.GetBucketCORSRequest(input) - req.ExpireTime = expires - err := req.Sign() - return req.HTTPRequest.URL, err -} var opGetBucketCORS *aws.Operation @@ -73,12 +65,6 @@ func (c *S3) DeleteBucketCORS(input *DeleteBucketCORSInput) (*DeleteBucketCORSOu err := req.Send() return out, err } -func (c *S3) DeleteBucketCORSPresignedUrl(input *DeleteBucketCORSInput, expires time.Duration) (*url.URL, error) { - req, _ := c.DeleteBucketCORSRequest(input) - req.ExpireTime = expires - err := req.Sign() - return req.HTTPRequest.URL, err -} var opDeleteBucketCORS *aws.Operation @@ -110,12 +96,6 @@ func (c *S3) PutBucketCORS(input *PutBucketCORSInput) (*PutBucketCORSOutput, err err := req.Send() return out, err } -func (c *S3) PutBucketCORSPresignedUrl(input *PutBucketCORSInput, expires time.Duration) (*url.URL, error) { - req, _ := c.PutBucketCORSRequest(input) - req.ExpireTime = expires - err := req.Sign() - return req.HTTPRequest.URL, err -} var opPutBucketCORS *aws.Operation diff --git a/service/s3/lifecycle.go b/service/s3/lifecycle.go index 3b63ad8..d136bd0 100644 --- a/service/s3/lifecycle.go +++ b/service/s3/lifecycle.go @@ -2,7 +2,6 @@ package s3 import ( "github.com/ks3sdklib/aws-sdk-go/aws" - "net/url" "time" ) @@ -181,12 +180,6 @@ func (c *S3) DeleteBucketLifecycle(input *DeleteBucketLifecycleInput) (*DeleteBu err := req.Send() return out, err } -func (c *S3) DeleteBucketLifecyclePresignedUrl(input *DeleteBucketLifecycleInput, expires time.Duration) (*url.URL, error) { - req, _ := c.DeleteBucketLifecycleRequest(input) - req.ExpireTime = expires - err := req.Sign() - return req.HTTPRequest.URL, err -} var opDeleteBucketLifecycle *aws.Operation @@ -219,12 +212,6 @@ func (c *S3) GetBucketLifecycle(input *GetBucketLifecycleInput) (*GetBucketLifec err := req.Send() return out, err } -func (c *S3) GetBucketLifecyclePresignedUrl(input *GetBucketLifecycleInput, expires time.Duration) (*url.URL, error) { - req, _ := c.GetBucketLifecycleRequest(input) - req.ExpireTime = expires - err := req.Sign() - return req.HTTPRequest.URL, err -} var opGetBucketLifecycle *aws.Operation diff --git a/test/objectsample_test.go b/test/objectsample_test.go index 4655358..2e4dfee 100644 --- a/test/objectsample_test.go +++ b/test/objectsample_test.go @@ -2,14 +2,16 @@ package lib import ( "bytes" + "crypto/md5" + "encoding/hex" "fmt" "github.com/ks3sdklib/aws-sdk-go/aws" "github.com/ks3sdklib/aws-sdk-go/aws/awserr" "github.com/ks3sdklib/aws-sdk-go/aws/awsutil" - "github.com/ks3sdklib/aws-sdk-go/internal/util/utilfile" "github.com/ks3sdklib/aws-sdk-go/service/s3" "github.com/ks3sdklib/aws-sdk-go/service/s3/s3manager" . "gopkg.in/check.v1" + "io" "net/url" "os" "time" @@ -47,16 +49,16 @@ func (s *Ks3utilCommandSuite) TestPutObject(c *C) { XAmzTagging := v.Encode() object := randLowStr(10) - s.createFile(object, content, c) - fd, _ := os.Open(content) - md5, _ := utilfile.GetFileMD5(content) + createFile(object, 1024*1024*1) + fd, _ := os.Open(object) + //md5, _ := utilfile.GetFileMD5(content) input := s3.PutObjectInput{ Bucket: aws.String(bucket), - Key: aws.String("/data/ss/hls_vod/data/sdtv2019/jimo/�k�%2/1677600325430_71915152/1677600326671-10000.ts"), + Key: aws.String(object), ACL: aws.String("public-read"), Body: fd, XAmzTagging: aws.String(XAmzTagging), - ContentMD5: aws.String(md5), + //ContentMD5: aws.String(md5), } resp, _ := client.PutObject(&input) fmt.Println("result：\n", awsutil.StringValue(resp)) @@ -68,7 +70,7 @@ func (s *Ks3utilCommandSuite) TestPutObject(c *C) { */ func (s *Ks3utilCommandSuite) TestPutObjectByLimit(c *C) { - MIN_BANDWIDTH := 1024 * 100 * 8 //100K bits/s + MIN_BANDWIDTH := 1024 * 1024 * 100 * 8 //100K bits/s createFile(content, 1024*1024*100) fd, _ := os.Open(content) input := s3.PutObjectInput{ @@ -152,68 +154,19 @@ func (s *Ks3utilCommandSuite) TestDelObject(c *C) { fmt.Println("result：\n", awsutil.StringValue(resp)) } -// 生成下载外链 -func (s *Ks3utilCommandSuite) TestGetObjectPresignedUrl(c *C) { - - expirationTime := time.Now().Add(time.Hour) // 设置外链过期时间为当前时间加上一小时 - urlExpiration := expirationTime.Unix() // 将过期时间转换为 Unix 时间戳 - - MIN_BANDWIDTH := 1024 * 100 * 8 // 100K bits/s - - // 设置 GetObjectInput 参数，并指定 TrafficLimit 限制上传速度 - params := &s3.GetObjectInput{ - Bucket: aws.String(bucket), // 设置 bucket 名称 - Key: aws.String("a.txt"), // 设置 object key - TrafficLimit: aws.Long(int64(MIN_BANDWIDTH)), // 设置上传速度限制 - } - //注意：v4签名情况下 这个时间多少秒后过期 - resp, err := client.GetObjectPresignedUrl(params, time.Duration(urlExpiration)) - if err != nil { - // 处理错误 - fmt.Println("Error:", err) - return - } - fmt.Println("Result:\n", awsutil.StringValue(resp.String())) -} - -// 生成上传外链 -func (s *Ks3utilCommandSuite) TestPutObjectPresignedUrl(c *C) { - - //注意：v4签名情况下 这个时间多少秒后过期 - expirationTime := time.Now().Add(time.Hour) // 设置外链过期时间为当前时间加上一小时 - urlExpiration := expirationTime.Unix() // 将过期时间转换为 Unix 时间戳 - - params := &s3.PutObjectInput{ - Bucket: aws.String(bucket), // 设置 bucket 名称 - Key: aws.String("a.txt"), // 设置 object key - TrafficLimit: aws.Long(1000), // 设置上传速度限制 - ContentType: aws.String("image/jpeg"), // 设置 content-type - } - - resp, err := client.PutObjectPresignedUrl(params, time.Duration(urlExpiration)) - if err != nil { - // 处理错误 - fmt.Println("Error:", err) - return - } - fmt.Println("Result:\n", awsutil.StringValue(resp.String())) -} - //根据方法生成外链 func (s *Ks3utilCommandSuite) TestGeneratePresignedUrl(c *C) { - //注意：v4签名情况下 这个时间多少秒后过期 - expirationTime := time.Now().Add(time.Hour) // 设置外链过期时间为当前时间加上一小时 - urlExpiration := expirationTime.Unix() // 将过期时间转换为 Unix 时间戳 - params := &s3.GeneratePresignedUrlInput{ - Bucket: aws.String(bucket), // 设置 bucket 名称 - Key: aws.String("a.txt"), // 设置 object key - TrafficLimit: aws.Long(1000), // 设置速度限制 - //ContentType: aws.String("image/jpeg"), // 设置 content-type - Expires: time.Duration(urlExpiration), + Bucket: aws.String(bucket), // 设置 bucket 名称 + Key: aws.String("test"), // 设置 object key + TrafficLimit: aws.Long(1000), // 设置速度限制 + //如果是PUT方法，需要设置content-type + //ContentType: aws.String("image/jpeg"), + //多久后过期 + Expires: 3600, //可选值有 PUT, GET, DELETE, HEAD - HTTPMethod: "GET", + HTTPMethod: s3.GET, } url := client.GeneratePresignedUrlInput(params) fmt.Println("Result:\n", url) @@ -373,8 +326,8 @@ func (s *Ks3utilCommandSuite) TestModifyObjectMeta(c *C) { //注意: 当你启动分块上传后，并开始上传分块，你必须完成或者放弃上传任务，才能终止因为存储造成的收费。 func (s *Ks3utilCommandSuite) TestMultipartUpload(c *C) { - MIN_BANDWIDTH := 1024 * 100 * 8 //100K bits/s - createFile(content, 1024*1024*100) + //MIN_BANDWIDTH := 1024 * 100 * 8 //100K bits/s + createFile(content, 1024*1024*10) initRet, _ := client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{ Bucket: aws.String(bucket), Key: aws.String(key), @@ -394,7 +347,7 @@ func (s *Ks3utilCommandSuite) TestMultipartUpload(c *C) { defer f.Close() var i int64 = 1 //组装分块参数 - compParts := []*s3.CompletedPart{} + var compParts []*s3.CompletedPart partsNum := []int64{0} sc := make([]byte, 52428800) @@ -414,6 +367,13 @@ func (s *Ks3utilCommandSuite) TestMultipartUpload(c *C) { //块的数量可以是1到10,000中的任意一个（包含1和10,000）。块序号用于标识一个块以及其在对象创建时的位置。如果你上传一个新的块，使用之前已经使用的序列号，那么之前的那个块将会被覆盖。当所有块总大小大于5M时，除了最后一个块没有大小限制外，其余的块的大小均要求在5MB以上。当所有块总大小小于5M时，除了最后一个块没有大小限制外，其余的块的大小均要求在100K以上。如果不符合上述要求，会返回413状态码。 // //为了保证数据在传输过程中没有损坏，请使用 Content-MD5 头部。当使用此头部时，KS3会自动计算出MD5，并根据用户提供的MD5进行校验，如果不匹配，将会返回错误信息。 + //计算sc[:nr]的md5值 + md5Ctx := md5.New() + reader := bytes.NewReader(sc[0:nr]) + io.Copy(md5Ctx, reader) + cipherStr := md5Ctx.Sum(nil) + md5Str := hex.EncodeToString(cipherStr) + resp, _ := client.UploadPart(&s3.UploadPartInput{ Bucket: aws.String(bucket), Key: aws.String(key), @@ -421,7 +381,8 @@ func (s *Ks3utilCommandSuite) TestMultipartUpload(c *C) { UploadID: aws.String(uploadId), Body: bytes.NewReader(sc[0:nr]), ContentLength: aws.Long(int64(len(sc[0:nr]))), - TrafficLimit: aws.Long(int64(MIN_BANDWIDTH)), + //TrafficLimit: aws.Long(int64(MIN_BANDWIDTH)), + ContentMD5: aws.String(md5Str), }) partsNum = append(partsNum, i) compParts = append(compParts, &s3.CompletedPart{PartNumber: &partsNum[i], ETag: resp.ETag})