From cf30fc99ca9f55284787fe735455c3cd397bbf82 Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Thu, 19 Sep 2024 15:06:21 +0800 Subject: [PATCH 01/13] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8Dhtml=E5=93=8D?= =?UTF-8?q?=E5=BA=94=E8=A7=A3=E6=9E=90=E5=A4=B1=E8=B4=A5=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 1 + go.sum | 44 ++++++++++++++ internal/protocol/query/unmarshal_error.go | 54 +++++++++++++++-- test/objectsample_test.go | 70 ++++++++++++---------- 4 files changed, 133 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index 03b673a..7b8293f 100644 --- a/go.mod +++ b/go.mod @@ -4,5 +4,6 @@ go 1.16 require ( github.com/stretchr/testify v1.8.1 + golang.org/x/net v0.24.0 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 ) diff --git a/go.sum b/go.sum index 2ec90f7..162c901 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,50 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/protocol/query/unmarshal_error.go b/internal/protocol/query/unmarshal_error.go index ee9a08e..78a6e5c 100755 --- a/internal/protocol/query/unmarshal_error.go +++ b/internal/protocol/query/unmarshal_error.go @@ -1,10 +1,11 @@ package query import ( + "bytes" "encoding/xml" + "golang.org/x/net/html" "io" - "io/ioutil" - "log" + "strings" "github.com/ks3sdklib/aws-sdk-go/aws" "github.com/ks3sdklib/aws-sdk-go/internal/apierr" @@ -13,25 +14,48 @@ import ( type XmlErrorResponse struct { XMLName xml.Name `xml:"Error"` Code string `xml:"Code"` - StatusCode int `"StatusCode"` + StatusCode int `xml:"StatusCode"` Message string `xml:"Message"` Resource string `xml:"Resource"` RequestID string `xml:"RequestId"` } -// UnmarshalError unmarshals an error response for an AWS Query service. +// UnmarshalError unmarshal an error response for an AWS Query service. func UnmarshalError(r *aws.Request) { defer r.HTTPResponse.Body.Close() resp := &XmlErrorResponse{} - body, err := ioutil.ReadAll(r.HTTPResponse.Body) + body, err := io.ReadAll(r.HTTPResponse.Body) if err != nil { - log.Printf("read body err, %v\n", err) + r.Error = apierr.New("Unmarshal", "failed to read body", err) return } + + // 如果响应类型是html,则解析html文本 + if strings.Contains(r.HTTPResponse.Header.Get("Content-Type"), "html") { + // 解析HTML文本 + doc, err := html.Parse(bytes.NewReader(body)) + if err != nil { + r.Error = apierr.New("Unmarshal", "failed to parse html", err) + return + } + title := findTitle(doc) + r.Error = apierr.NewRequestError( + apierr.New(title, "", nil), + r.HTTPResponse.StatusCode, + "", + ) + return + } + err = xml.Unmarshal(body, &resp) resp.StatusCode = r.HTTPResponse.StatusCode + // head请求无法从body中获取request id,如果是head请求,则从header中获取 + if resp.RequestID == "" && r.HTTPRequest.Method == "HEAD" { + resp.RequestID = r.HTTPResponse.Header.Get("X-KSS-Request-Id") + } + if err != nil && err != io.EOF { r.Error = apierr.New("Unmarshal", "failed to decode query XML error response", err) } else { @@ -42,3 +66,21 @@ func UnmarshalError(r *aws.Request) { ) } } + +// findTitle 提取HTML文档中<title>标签的内容 +func findTitle(doc *html.Node) string { + var title string + + var traverse func(*html.Node) + traverse = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil { + title = n.FirstChild.Data + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + traverse(c) + } + } + + traverse(doc) + return title +} diff --git a/test/objectsample_test.go b/test/objectsample_test.go index 65499aa..ec13ba2 100644 --- a/test/objectsample_test.go +++ b/test/objectsample_test.go @@ -56,18 +56,17 @@ func (s *Ks3utilCommandSuite) TestPutObject(c *C) { // TestPutObjectByLimit 上传示例 -限速 func (s *Ks3utilCommandSuite) TestPutObjectByLimit(c *C) { - MIN_BANDWIDTH := 1024 * 100 * 8 // 100KB/s + minBandwidth := 1024 * 100 * 8 // 100KB/s object := randLowStr(10) createFile(object, 1024*1024*1) // 1MB大小的文件 fd, _ := os.Open(object) // 记录开始时间 startTime := time.Now() _, err := client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - Body: fd, - //设置上传速度 - TrafficLimit: aws.Long(int64(MIN_BANDWIDTH)), + Bucket: aws.String(bucket), + Key: aws.String(object), + Body: fd, + TrafficLimit: aws.Long(int64(minBandwidth)), //限制上传速度 }) c.Assert(err, IsNil) // 计算上传耗时 @@ -79,7 +78,7 @@ func (s *Ks3utilCommandSuite) TestPutObjectByLimit(c *C) { // TestGetObjectByLimit 下载限速示例 func (s *Ks3utilCommandSuite) TestGetObjectByLimit(c *C) { - MIN_BANDWIDTH := 1024 * 100 * 8 // 100KB/s + minBandwidth := 1024 * 100 * 8 // 100KB/s _, err := client.PutObject(&s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), @@ -92,23 +91,15 @@ func (s *Ks3utilCommandSuite) TestGetObjectByLimit(c *C) { _, err = client.GetObject(&s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), - TrafficLimit: aws.Long(int64(MIN_BANDWIDTH)), + TrafficLimit: aws.Long(int64(minBandwidth)), //限制下载速度 }) c.Assert(err, IsNil) } // TestGetObject 下载示例 func (s *Ks3utilCommandSuite) TestGetObject(c *C) { - _, err := client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - ACL: aws.String("public-read"), - Body: strings.NewReader(content), - }) - c.Assert(err, IsNil) - - //下载 - _, err = client.GetObject(&s3.GetObjectInput{ + s.PutObject(key, c) + _, err := client.GetObject(&s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) @@ -117,15 +108,8 @@ func (s *Ks3utilCommandSuite) TestGetObject(c *C) { // TestDeleteObject 删除对象 func (s *Ks3utilCommandSuite) TestDeleteObject(c *C) { - _, err := client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - ACL: aws.String("public-read"), - Body: strings.NewReader(content), - }) - c.Assert(err, IsNil) - - _, err = client.DeleteObject(&s3.DeleteObjectInput{ + s.PutObject(key, c) + _, err := client.DeleteObject(&s3.DeleteObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) @@ -233,7 +217,6 @@ func (s *Ks3utilCommandSuite) TestCopyObject(c *C) { // TestUploadPartCopy 分块拷贝用例 func (s *Ks3utilCommandSuite) TestUploadPartCopy(c *C) { s.PutObject(key, c) - dstKey := "xxx/copy/" + key //初始化分块 initResp, err := client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{ @@ -242,7 +225,7 @@ func (s *Ks3utilCommandSuite) TestUploadPartCopy(c *C) { }) c.Assert(err, IsNil) - uploadPartCopyresp, err := client.UploadPartCopy(&s3.UploadPartCopyInput{ + uploadPartCopyResp, err := client.UploadPartCopy(&s3.UploadPartCopyInput{ Bucket: aws.String(bucket), Key: aws.String(dstKey), CopySource: aws.String("/" + bucket + "/" + key), @@ -261,7 +244,7 @@ func (s *Ks3utilCommandSuite) TestUploadPartCopy(c *C) { Parts: []*s3.CompletedPart{ { PartNumber: aws.Long(1), - ETag: uploadPartCopyresp.CopyPartResult.ETag, + ETag: uploadPartCopyResp.CopyPartResult.ETag, }, }, }, @@ -742,3 +725,30 @@ func (s *Ks3utilCommandSuite) TestUploadPartProgress(c *C) { c.Assert(err, IsNil) os.Remove(object) } + +// TestPutObject10GB 上传10GB文件,报413 Request Entity Too Large错误,错误类型为html +func (s *Ks3utilCommandSuite) TestPutObject10GB(c *C) { + object := randLowStr(10) + createFile(object, 1024*1024*1) + fd, _ := os.Open(object) + _, err := client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + Body: fd, + ContentLength: aws.Long(1024 * 1024 * 1024 * 10), + }) + c.Assert(err, NotNil) + c.Assert(strings.Contains(err.Error(), "413 Request Entity Too Large"), Equals, true) + os.Remove(object) +} + +// TestHeadNotExistsObject head不存在的对象,报404错误,request id不为空 +func (s *Ks3utilCommandSuite) TestHeadNotExistsObject(c *C) { + object := randLowStr(10) + _, err := client.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + }) + c.Assert(err, NotNil) + c.Assert(strings.Index(err.Error(), "[")+1 != strings.Index(err.Error(), "]"), Equals, true) +} From c3d99e8ceda3090d19b4dcd84a23e6c831183679 Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Fri, 20 Sep 2024 16:42:00 +0800 Subject: [PATCH 02/13] =?UTF-8?q?feat:=20=E7=94=9F=E6=88=90=E5=A4=96?= =?UTF-8?q?=E9=93=BE=E6=94=AF=E6=8C=81=E8=87=AA=E5=AE=9A=E4=B9=89=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E5=A4=B4=E5=92=8C=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/protocol/query/unmarshal_error.go | 5 +- internal/protocol/rest/build.go | 17 +- service/s3/api.go | 67 +----- service/s3/const.go | 26 ++- service/s3/service.go | 12 +- service/s3/{s3util => }/util.go | 40 +++- test/object_encryption_test.go | 57 +++-- test/objectsample_test.go | 254 ++++++++++++++++++++- 8 files changed, 365 insertions(+), 113 deletions(-) rename service/s3/{s3util => }/util.go (53%) diff --git a/internal/protocol/query/unmarshal_error.go b/internal/protocol/query/unmarshal_error.go index 78a6e5c..88fed23 100755 --- a/internal/protocol/query/unmarshal_error.go +++ b/internal/protocol/query/unmarshal_error.go @@ -3,12 +3,11 @@ package query import ( "bytes" "encoding/xml" + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" "golang.org/x/net/html" "io" "strings" - - "github.com/ks3sdklib/aws-sdk-go/aws" - "github.com/ks3sdklib/aws-sdk-go/internal/apierr" ) type XmlErrorResponse struct { diff --git a/internal/protocol/rest/build.go b/internal/protocol/rest/build.go index 55189d9..022a45f 100755 --- a/internal/protocol/rest/build.go +++ b/internal/protocol/rest/build.go @@ -75,6 +75,8 @@ func buildLocationElements(r *aws.Request, v reflect.Value) { buildURI(r, m, name) case "querystring": buildQueryString(r, m, name, query) + case "parameters": + buildParameters(r, m, query) } } if r.Error != nil { @@ -149,6 +151,19 @@ func buildQueryString(r *aws.Request, v reflect.Value, name string, query url.Va r.Error = apierr.New("Marshal", "failed to encode REST request", err) } else if str != nil { query.Set(name, *str) + } else if str == nil { + query.Set(name, "") + } +} + +func buildParameters(r *aws.Request, v reflect.Value, query url.Values) { + for _, key := range v.MapKeys() { + str, err := convertType(v.MapIndex(key)) + if err != nil { + r.Error = apierr.New("Marshal", "failed to encode REST request", err) + } else { + buildQueryString(r, reflect.ValueOf(str), key.String(), query) + } } } @@ -230,7 +245,7 @@ func convertType(v reflect.Value) (*string, error) { case time.Time: str = value.UTC().Format(RFC822) default: - err := fmt.Errorf("Unsupported value for param %v (%s)", v.Interface(), v.Type()) + err := fmt.Errorf("unsupported value for param %v (%s)", v.Interface(), v.Type()) return nil, err } return &str, nil diff --git a/service/s3/api.go b/service/s3/api.go index 63f410a..0d12e04 100755 --- a/service/s3/api.go +++ b/service/s3/api.go @@ -9,7 +9,6 @@ import ( "github.com/ks3sdklib/aws-sdk-go/aws/awserr" "github.com/ks3sdklib/aws-sdk-go/internal/apierr" "github.com/ks3sdklib/aws-sdk-go/internal/crc" - "github.com/ks3sdklib/aws-sdk-go/service/s3/s3util" "hash" "io" "net/http" @@ -123,7 +122,7 @@ func (c *S3) CopyObjectRequest(input *CopyObjectInput) (req *aws.Request, output // URL encode the copy source if input.CopySource == nil { - input.CopySource = aws.String(s3util.BuildCopySource(input.SourceBucket, input.SourceKey)) + input.CopySource = aws.String(BuildCopySource(input.SourceBucket, input.SourceKey)) } req = c.newRequest(opCopyObject, input, output) output = &CopyObjectOutput{} @@ -651,6 +650,7 @@ func (c *S3) GetBucketLocationRequest(input *GetBucketLocationInput) (req *aws.R } req = c.newRequest(opGetBucketLocation, input, output) + req.Handlers.Unmarshal.PushFront(buildGetBucketLocation) output = &GetBucketLocationOutput{} req.Data = output return @@ -1933,15 +1933,6 @@ func (c *S3) PutBucketWebsiteRequest(input *PutBucketWebsiteInput) (req *aws.Req return } -type HTTPMethod string - -const ( - PUT HTTPMethod = "PUT" - GET HTTPMethod = "GET" - DELETE HTTPMethod = "DELETE" - HEAD HTTPMethod = "HEAD" -) - type metadataGeneratePresignedUrlInput struct { SDKShapeTraits bool `type:"structure" payload:"GeneratePresignedUrlInput"` } @@ -1987,6 +1978,10 @@ type GeneratePresignedUrlInput struct { // Sets the Expires header of the response. ResponseExpires *time.Time `location:"querystring" locationName:"response-expires" type:"timestamp" timestampFormat:"iso8601"` + Headers map[string]*string `location:"headers" type:"map"` + + Parameters map[string]*string `location:"parameters" type:"map"` + metadataGeneratePresignedUrlInput `json:"-" xml:"-"` } type GeneratePresignedUrlOutput struct { @@ -2294,7 +2289,7 @@ func (c *S3) UploadPartCopyRequest(input *UploadPartCopyInput) (req *aws.Request // URL encode the copy source if input.CopySource == nil { - input.CopySource = aws.String(s3util.BuildCopySource(input.SourceBucket, input.SourceKey)) + input.CopySource = aws.String(BuildCopySource(input.SourceBucket, input.SourceKey)) } req = c.newRequest(opUploadPartCopy, input, output) output = &UploadPartCopyOutput{} @@ -5729,54 +5724,6 @@ type metadataWebsiteConfiguration struct { SDKShapeTraits bool `type:"structure"` } -const AllUsersUri = "http://acs.amazonaws.com/groups/global/AllUsers" - -type CannedAccessControlType int32 - -const ( - PublicReadWrite CannedAccessControlType = 0 - PublicRead CannedAccessControlType = 1 - Private CannedAccessControlType = 2 -) - -func GetAcl(resp GetObjectACLOutput) CannedAccessControlType { - - allUsersPermissions := map[string]*string{} - for _, value := range resp.Grants { - if value.Grantee.URI != nil && *value.Grantee.URI == AllUsersUri { - allUsersPermissions[*value.Permission] = value.Permission - } - } - _, read := allUsersPermissions["READ"] - _, write := allUsersPermissions["WRITE"] - if read && write { - return PublicReadWrite - } else if read { - return PublicRead - } else { - return Private - } -} - -func GetBucketAcl(resp GetBucketACLOutput) CannedAccessControlType { - - allUsersPermissions := map[string]*string{} - for _, value := range resp.Grants { - if value.Grantee.URI != nil && *value.Grantee.URI == AllUsersUri { - allUsersPermissions[*value.Permission] = value.Permission - } - } - _, read := allUsersPermissions["READ"] - _, write := allUsersPermissions["WRITE"] - if read && write { - return PublicReadWrite - } else if read { - return PublicRead - } else { - return Private - } -} - func (c *S3) DeleteObjectTaggingRequest(input *DeleteObjectTaggingInput) (req *aws.Request, output *DeleteObjectTaggingOutput) { oprw.Lock() defer oprw.Unlock() diff --git a/service/s3/const.go b/service/s3/const.go index 03c8200..86f648e 100644 --- a/service/s3/const.go +++ b/service/s3/const.go @@ -17,9 +17,9 @@ const ( HTTPHeaderHost = "Host" HTTPHeaderkssACL = "X-kss-Acl" - ChannelBuf int = 1000 - MinPartSize5MB = 5*1024*1024 + 100 // part size, 5MB - MinPartSize = 100 * 1024 // Min part size, 100KB + ChannelBuf int = 1000 + PartSize5MB = 5 * 1024 * 1024 // part size, 5MB + MinPartSize = 100 * 1024 // Min part size, 100KB ) // ACL @@ -51,3 +51,23 @@ const ( BucketTypeDeepIA string = "DEEP_IA" BucketTypeArchive string = "ARCHIVE" ) + +type HTTPMethod string + +const ( + PUT HTTPMethod = "PUT" + GET HTTPMethod = "GET" + DELETE HTTPMethod = "DELETE" + HEAD HTTPMethod = "HEAD" + POST HTTPMethod = "POST" +) + +const AllUsersUri = "http://acs.amazonaws.com/groups/global/AllUsers" + +type CannedAccessControlType int32 + +const ( + PublicReadWrite CannedAccessControlType = 0 + PublicRead CannedAccessControlType = 1 + Private CannedAccessControlType = 2 +) diff --git a/service/s3/service.go b/service/s3/service.go index baff037..8263edc 100755 --- a/service/s3/service.go +++ b/service/s3/service.go @@ -5,8 +5,8 @@ package s3 import ( "github.com/ks3sdklib/aws-sdk-go/aws" "github.com/ks3sdklib/aws-sdk-go/internal/protocol/body" - v2 "github.com/ks3sdklib/aws-sdk-go/internal/signer/v2" - v4 "github.com/ks3sdklib/aws-sdk-go/internal/signer/v4" + "github.com/ks3sdklib/aws-sdk-go/internal/signer/v2" + "github.com/ks3sdklib/aws-sdk-go/internal/signer/v4" "strings" ) @@ -19,13 +19,7 @@ type S3 struct { var initService func(*aws.Service) // Used for custom request initialization logic -var initRequest = func(r *aws.Request) { - switch r.Operation.Name { - case "GetBucketLocation": - // GetBucketLocation has custom parsing logic - r.Handlers.Unmarshal.PushFront(buildGetBucketLocation) - } -} +var initRequest func(*aws.Request) // New returns a new S3 client. func New(config *aws.Config) *S3 { diff --git a/service/s3/s3util/util.go b/service/s3/util.go similarity index 53% rename from service/s3/s3util/util.go rename to service/s3/util.go index f8d0aca..8dd52be 100644 --- a/service/s3/s3util/util.go +++ b/service/s3/util.go @@ -1,4 +1,4 @@ -package s3util +package s3 import ( "crypto/md5" @@ -56,3 +56,41 @@ func BuildCopySource(bucket *string, key *string) string { } return "/" + *bucket + "/" + url.QueryEscape(*key) } + +// GetAcl 获取对象的访问控制权限 +func GetAcl(resp GetObjectACLOutput) CannedAccessControlType { + allUsersPermissions := map[string]*string{} + for _, value := range resp.Grants { + if value.Grantee.URI != nil && *value.Grantee.URI == AllUsersUri { + allUsersPermissions[*value.Permission] = value.Permission + } + } + _, read := allUsersPermissions["READ"] + _, write := allUsersPermissions["WRITE"] + if read && write { + return PublicReadWrite + } else if read { + return PublicRead + } else { + return Private + } +} + +// GetBucketAcl 获取存储空间的访问控制权限 +func GetBucketAcl(resp GetBucketACLOutput) CannedAccessControlType { + allUsersPermissions := map[string]*string{} + for _, value := range resp.Grants { + if value.Grantee.URI != nil && *value.Grantee.URI == AllUsersUri { + allUsersPermissions[*value.Permission] = value.Permission + } + } + _, read := allUsersPermissions["READ"] + _, write := allUsersPermissions["WRITE"] + if read && write { + return PublicReadWrite + } else if read { + return PublicRead + } else { + return Private + } +} diff --git a/test/object_encryption_test.go b/test/object_encryption_test.go index c883199..4cde7cd 100644 --- a/test/object_encryption_test.go +++ b/test/object_encryption_test.go @@ -5,7 +5,6 @@ import ( "context" "github.com/ks3sdklib/aws-sdk-go/aws" "github.com/ks3sdklib/aws-sdk-go/service/s3" - "github.com/ks3sdklib/aws-sdk-go/service/s3/s3util" . "gopkg.in/check.v1" "io" "os" @@ -63,34 +62,34 @@ func (s *Ks3utilCommandSuite) TestPutObjectWithSSE_C(c *C) { Key: aws.String(object), Body: fd, SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: aws.String(s3util.GetBase64Str(customerKey)), - SSECustomerKeyMD5: aws.String(s3util.GetBase64MD5Str(customerKey)), + SSECustomerKey: aws.String(s3.GetBase64Str(customerKey)), + SSECustomerKeyMD5: aws.String(s3.GetBase64MD5Str(customerKey)), }) c.Assert(err, IsNil) c.Assert(*sseResp.SSECustomerAlgorithm, Equals, "AES256") - c.Assert(*sseResp.SSECustomerKeyMD5, Equals, s3util.GetBase64MD5Str(customerKey)) + c.Assert(*sseResp.SSECustomerKeyMD5, Equals, s3.GetBase64MD5Str(customerKey)) // head headResp, err := client.HeadObjectWithContext(context.Background(), &s3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(object), SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: aws.String(s3util.GetBase64Str(customerKey)), - SSECustomerKeyMD5: aws.String(s3util.GetBase64MD5Str(customerKey)), + SSECustomerKey: aws.String(s3.GetBase64Str(customerKey)), + SSECustomerKeyMD5: aws.String(s3.GetBase64MD5Str(customerKey)), }) c.Assert(err, IsNil) c.Assert(*headResp.SSECustomerAlgorithm, Equals, "AES256") - c.Assert(*sseResp.SSECustomerKeyMD5, Equals, s3util.GetBase64MD5Str(customerKey)) + c.Assert(*sseResp.SSECustomerKeyMD5, Equals, s3.GetBase64MD5Str(customerKey)) // get getResp, err := client.GetObjectWithContext(context.Background(), &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(object), SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: aws.String(s3util.GetBase64Str(customerKey)), - SSECustomerKeyMD5: aws.String(s3util.GetBase64MD5Str(customerKey)), + SSECustomerKey: aws.String(s3.GetBase64Str(customerKey)), + SSECustomerKeyMD5: aws.String(s3.GetBase64MD5Str(customerKey)), }) c.Assert(err, IsNil) c.Assert(*getResp.SSECustomerAlgorithm, Equals, "AES256") - c.Assert(*sseResp.SSECustomerKeyMD5, Equals, s3util.GetBase64MD5Str(customerKey)) + c.Assert(*sseResp.SSECustomerKeyMD5, Equals, s3.GetBase64MD5Str(customerKey)) // delete _, err = client.DeleteObjectWithContext(context.Background(), &s3.DeleteObjectInput{ Bucket: aws.String(bucket), @@ -159,12 +158,12 @@ func (s *Ks3utilCommandSuite) TestCopyObjectWithSSE_C(c *C) { Key: aws.String(dstObject), CopySource: aws.String("/" + bucket + "/" + object), SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: aws.String(s3util.GetBase64Str(customerKey)), - SSECustomerKeyMD5: aws.String(s3util.GetBase64MD5Str(customerKey)), + SSECustomerKey: aws.String(s3.GetBase64Str(customerKey)), + SSECustomerKeyMD5: aws.String(s3.GetBase64MD5Str(customerKey)), }) c.Assert(err, IsNil) c.Assert(*copyResp.SSECustomerAlgorithm, Equals, "AES256") - c.Assert(*copyResp.SSECustomerKeyMD5, Equals, s3util.GetBase64MD5Str(customerKey)) + c.Assert(*copyResp.SSECustomerKeyMD5, Equals, s3.GetBase64MD5Str(customerKey)) // delete _, err = client.DeleteObjectWithContext(context.Background(), &s3.DeleteObjectInput{ Bucket: aws.String(bucket), @@ -256,12 +255,12 @@ func (s *Ks3utilCommandSuite) TestMultipartUploadWithSSE_C(c *C) { Bucket: aws.String(bucket), Key: aws.String(object), SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: aws.String(s3util.GetBase64Str(customerKey)), - SSECustomerKeyMD5: aws.String(s3util.GetBase64MD5Str(customerKey)), + SSECustomerKey: aws.String(s3.GetBase64Str(customerKey)), + SSECustomerKeyMD5: aws.String(s3.GetBase64MD5Str(customerKey)), }) c.Assert(err, IsNil) c.Assert(*initRet.SSECustomerAlgorithm, Equals, "AES256") - c.Assert(*initRet.SSECustomerKeyMD5, Equals, s3util.GetBase64MD5Str(customerKey)) + c.Assert(*initRet.SSECustomerKeyMD5, Equals, s3.GetBase64MD5Str(customerKey)) // 获取分块上传Id uploadId := *initRet.UploadID var i int64 = 1 @@ -285,12 +284,12 @@ func (s *Ks3utilCommandSuite) TestMultipartUploadWithSSE_C(c *C) { UploadID: aws.String(uploadId), Body: bytes.NewReader(buffer[:n]), SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: aws.String(s3util.GetBase64Str(customerKey)), - SSECustomerKeyMD5: aws.String(s3util.GetBase64MD5Str(customerKey)), + SSECustomerKey: aws.String(s3.GetBase64Str(customerKey)), + SSECustomerKeyMD5: aws.String(s3.GetBase64MD5Str(customerKey)), }) c.Assert(err, IsNil) c.Assert(*resp.SSECustomerAlgorithm, Equals, "AES256") - c.Assert(*resp.SSECustomerKeyMD5, Equals, s3util.GetBase64MD5Str(customerKey)) + c.Assert(*resp.SSECustomerKeyMD5, Equals, s3.GetBase64MD5Str(customerKey)) partsNum = append(partsNum, i) compParts = append(compParts, &s3.CompletedPart{PartNumber: &partsNum[i], ETag: resp.ETag}) i++ @@ -307,7 +306,7 @@ func (s *Ks3utilCommandSuite) TestMultipartUploadWithSSE_C(c *C) { }) c.Assert(err, IsNil) c.Assert(*comResp.SSECustomerAlgorithm, Equals, "AES256") - c.Assert(*comResp.SSECustomerKeyMD5, Equals, s3util.GetBase64MD5Str(customerKey)) + c.Assert(*comResp.SSECustomerKeyMD5, Equals, s3.GetBase64MD5Str(customerKey)) // delete _, err = client.DeleteObjectWithContext(context.Background(), &s3.DeleteObjectInput{ Bucket: aws.String(bucket), @@ -369,34 +368,34 @@ func (s *Ks3utilCommandSuite) TestAppendObjectWithSSE_C(c *C) { Position: aws.Long(0), Body: fd, SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: aws.String(s3util.GetBase64Str(customerKey)), - SSECustomerKeyMD5: aws.String(s3util.GetBase64MD5Str(customerKey)), + SSECustomerKey: aws.String(s3.GetBase64Str(customerKey)), + SSECustomerKeyMD5: aws.String(s3.GetBase64MD5Str(customerKey)), }) c.Assert(err, IsNil) c.Assert(*sseResp.SSECustomerAlgorithm, Equals, "AES256") - c.Assert(*sseResp.SSECustomerKeyMD5, Equals, s3util.GetBase64MD5Str(customerKey)) + c.Assert(*sseResp.SSECustomerKeyMD5, Equals, s3.GetBase64MD5Str(customerKey)) // head headResp, err := client.HeadObjectWithContext(context.Background(), &s3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(object), SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: aws.String(s3util.GetBase64Str(customerKey)), - SSECustomerKeyMD5: aws.String(s3util.GetBase64MD5Str(customerKey)), + SSECustomerKey: aws.String(s3.GetBase64Str(customerKey)), + SSECustomerKeyMD5: aws.String(s3.GetBase64MD5Str(customerKey)), }) c.Assert(err, IsNil) c.Assert(*headResp.SSECustomerAlgorithm, Equals, "AES256") - c.Assert(*sseResp.SSECustomerKeyMD5, Equals, s3util.GetBase64MD5Str(customerKey)) + c.Assert(*sseResp.SSECustomerKeyMD5, Equals, s3.GetBase64MD5Str(customerKey)) // get getResp, err := client.GetObjectWithContext(context.Background(), &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(object), SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: aws.String(s3util.GetBase64Str(customerKey)), - SSECustomerKeyMD5: aws.String(s3util.GetBase64MD5Str(customerKey)), + SSECustomerKey: aws.String(s3.GetBase64Str(customerKey)), + SSECustomerKeyMD5: aws.String(s3.GetBase64MD5Str(customerKey)), }) c.Assert(err, IsNil) c.Assert(*getResp.SSECustomerAlgorithm, Equals, "AES256") - c.Assert(*sseResp.SSECustomerKeyMD5, Equals, s3util.GetBase64MD5Str(customerKey)) + c.Assert(*sseResp.SSECustomerKeyMD5, Equals, s3.GetBase64MD5Str(customerKey)) // delete _, err = client.DeleteObjectWithContext(context.Background(), &s3.DeleteObjectInput{ Bucket: aws.String(bucket), diff --git a/test/objectsample_test.go b/test/objectsample_test.go index ec13ba2..c4944da 100644 --- a/test/objectsample_test.go +++ b/test/objectsample_test.go @@ -2,13 +2,14 @@ package lib import ( "bytes" + "encoding/xml" "fmt" "github.com/ks3sdklib/aws-sdk-go/aws" "github.com/ks3sdklib/aws-sdk-go/aws/awserr" "github.com/ks3sdklib/aws-sdk-go/service/s3" "github.com/ks3sdklib/aws-sdk-go/service/s3/s3manager" - "github.com/ks3sdklib/aws-sdk-go/service/s3/s3util" . "gopkg.in/check.v1" + "io" "net/http" "net/url" "os" @@ -40,7 +41,7 @@ func (s *Ks3utilCommandSuite) TestPutObject(c *C) { object := randLowStr(10) createFile(object, 1024*1024*1) fd, _ := os.Open(object) - md5, _ := s3util.GetBase64FileMD5Str(object) + md5, _ := s3.GetBase64FileMD5Str(object) _, err := client.PutObject(&s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(object), @@ -132,7 +133,7 @@ func (s *Ks3utilCommandSuite) TestGeneratePresignedUrl(c *C) { // TestGeneratePUTPresignedUrl 根据外链PUT上传 func (s *Ks3utilCommandSuite) TestGeneratePUTPresignedUrl(c *C) { text := "test content" - md5 := s3util.GetBase64MD5Str(text) + md5 := s3.GetBase64MD5Str(text) url, err := client.GeneratePresignedUrl(&s3.GeneratePresignedUrlInput{ Bucket: aws.String(bucket), // 设置 bucket 名称 Key: aws.String(key), // 设置 object key @@ -334,7 +335,7 @@ func (s *Ks3utilCommandSuite) TestMultipartUpload(c *C) { //块的数量可以是1到10,000中的任意一个(包含1和10,000)。块序号用于标识一个块以及其在对象创建时的位置。如果你上传一个新的块,使用之前已经使用的序列号,那么之前的那个块将会被覆盖。当所有块总大小大于5M时,除了最后一个块没有大小限制外,其余的块的大小均要求在5MB以上。当所有块总大小小于5M时,除了最后一个块没有大小限制外,其余的块的大小均要求在100K以上。如果不符合上述要求,会返回413状态码。 //为了保证数据在传输过程中没有损坏,请使用 Content-MD5 头部。当使用此头部时,KS3会自动计算出MD5,并根据用户提供的MD5进行校验,如果不匹配,将会返回错误信息。 //计算sc[:nr]的md5值 - md5 := s3util.GetBase64MD5Str(string(sc[0:nr])) + md5 := s3.GetBase64MD5Str(string(sc[0:nr])) resp, err := client.UploadPart(&s3.UploadPartInput{ Bucket: aws.String(bucket), Key: aws.String(key), @@ -376,9 +377,9 @@ func (s *Ks3utilCommandSuite) TestPutObjectWithSSEC(c *C) { _, err := client.PutObject(&s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), - SSECustomerAlgorithm: aws.String("AES256"), //加密类型 - SSECustomerKey: aws.String(s3util.GetBase64Str(SSECustomerKey)), // 客户端提供的加密密钥 - SSECustomerKeyMD5: aws.String(s3util.GetBase64MD5Str(SSECustomerKey)), // 客户端提供的通过BASE64编码的通过128位MD5加密的密钥的MD5值 + SSECustomerAlgorithm: aws.String("AES256"), //加密类型 + SSECustomerKey: aws.String(s3.GetBase64Str(SSECustomerKey)), // 客户端提供的加密密钥 + SSECustomerKeyMD5: aws.String(s3.GetBase64MD5Str(SSECustomerKey)), // 客户端提供的通过BASE64编码的通过128位MD5加密的密钥的MD5值 }) c.Assert(err, IsNil) } @@ -752,3 +753,242 @@ func (s *Ks3utilCommandSuite) TestHeadNotExistsObject(c *C) { c.Assert(err, NotNil) c.Assert(strings.Index(err.Error(), "[")+1 != strings.Index(err.Error(), "]"), Equals, true) } + +// TestPresignedMultipartUpload 通过外链分块上传 +func (s *Ks3utilCommandSuite) TestPresignedMultipartUpload(c *C) { + object := randLowStr(10) + createFile(object, 1024*1024*1) + fd, _ := os.Open(object) + // 生成init外链 + initUrl, err := client.GeneratePresignedUrl(&s3.GeneratePresignedUrlInput{ + HTTPMethod: s3.POST, + Bucket: aws.String(bucket), + Key: aws.String(object), + Expires: 3600, + Parameters: map[string]*string{ + "uploads": nil, + }, + }) + + fmt.Println(initUrl) + + initRequest, err := http.NewRequest("POST", initUrl, nil) + c.Assert(err, IsNil) + + initResp, err := http.DefaultClient.Do(initRequest) + c.Assert(err, IsNil) + + body, err := io.ReadAll(initResp.Body) + c.Assert(err, IsNil) + + initXml := struct { + UploadId string `xml:"UploadId"` + }{} + err = xml.Unmarshal(body, &initXml) + c.Assert(err, IsNil) + + fmt.Println(initXml.UploadId) + + // 生成upload part外链 + uploadPartUrl, err := client.GeneratePresignedUrl(&s3.GeneratePresignedUrlInput{ + HTTPMethod: s3.PUT, + Bucket: aws.String(bucket), + Key: aws.String(object), + Expires: 3600, + Parameters: map[string]*string{ + "partNumber": aws.String("1"), + "uploadId": aws.String(initXml.UploadId), + }, + }) + c.Assert(err, IsNil) + fmt.Println(uploadPartUrl) + + uploadPartRequest, err := http.NewRequest("PUT", uploadPartUrl, fd) + c.Assert(err, IsNil) + + uploadPartResp, err := http.DefaultClient.Do(uploadPartRequest) + c.Assert(err, IsNil) + + etag := uploadPartResp.Header.Get("ETag") + fmt.Println(etag) + + // 生成complete外链 + completeUrl, err := client.GeneratePresignedUrl(&s3.GeneratePresignedUrlInput{ + HTTPMethod: s3.POST, + Bucket: aws.String(bucket), + Key: aws.String(object), + Expires: 3600, + Parameters: map[string]*string{ + "uploadId": aws.String(initXml.UploadId), + }, + }) + c.Assert(err, IsNil) + fmt.Println(completeUrl) + + completeParts := `<CompleteMultipartUpload> + <Part> + <PartNumber>1</PartNumber> + <ETag>` + etag + `</ETag> + </Part> + </CompleteMultipartUpload>` + fmt.Println(completeParts) + + completeRequest, err := http.NewRequest("POST", completeUrl, strings.NewReader(completeParts)) + c.Assert(err, IsNil) + + completeResp, err := http.DefaultClient.Do(completeRequest) + c.Assert(err, IsNil) + + body, err = io.ReadAll(completeResp.Body) + c.Assert(err, IsNil) + + fmt.Println(string(body)) + + // 获取head外链 + headUrl, err := client.GeneratePresignedUrl(&s3.GeneratePresignedUrlInput{ + HTTPMethod: s3.HEAD, + Bucket: aws.String(bucket), + Key: aws.String(object), + Expires: 3600, + }) + c.Assert(err, IsNil) + fmt.Println(headUrl) + + headRequest, err := http.NewRequest("HEAD", headUrl, nil) + c.Assert(err, IsNil) + + headResp, err := http.DefaultClient.Do(headRequest) + c.Assert(err, IsNil) + c.Assert(headResp.StatusCode, Equals, 200) + + os.Remove(object) +} + +// TestPresignedMultipartCopy 通过外链分块复制 +func (s *Ks3utilCommandSuite) TestPresignedMultipartCopy(c *C) { + object := randLowStr(10) + createFile(object, 1024*1024*1) + s.PutObject(object, c) + + // 生成init外链 + initUrl, err := client.GeneratePresignedUrl(&s3.GeneratePresignedUrlInput{ + HTTPMethod: s3.POST, + Bucket: aws.String(bucket), + Key: aws.String(object + "copy"), + Expires: 3600, + Parameters: map[string]*string{ + "uploads": nil, + }, + }) + c.Assert(err, IsNil) + fmt.Println(initUrl) + + initRequest, err := http.NewRequest("POST", initUrl, nil) + c.Assert(err, IsNil) + + initResp, err := http.DefaultClient.Do(initRequest) + c.Assert(err, IsNil) + + body, err := io.ReadAll(initResp.Body) + c.Assert(err, IsNil) + + initXml := struct { + UploadId string `xml:"UploadId"` + }{} + + err = xml.Unmarshal(body, &initXml) + c.Assert(err, IsNil) + + fmt.Println(initXml.UploadId) + + // 生成upload part外链 + uploadPartUrl, err := client.GeneratePresignedUrl(&s3.GeneratePresignedUrlInput{ + HTTPMethod: s3.PUT, + Bucket: aws.String(bucket), + Key: aws.String(object + "copy"), + Expires: 3600, + Parameters: map[string]*string{ + "partNumber": aws.String("1"), + "uploadId": aws.String(initXml.UploadId), + }, + Headers: map[string]*string{ + "X-Amz-Copy-Source": aws.String("/" + bucket + "/" + object), + }, + }) + c.Assert(err, IsNil) + fmt.Println(uploadPartUrl) + + uploadPartRequest, err := http.NewRequest("PUT", uploadPartUrl, nil) + c.Assert(err, IsNil) + + // 设置header + uploadPartRequest.Header.Set("X-Amz-Copy-Source", "/"+bucket+"/"+object) + + uploadPartResp, err := http.DefaultClient.Do(uploadPartRequest) + c.Assert(err, IsNil) + + body, err = io.ReadAll(uploadPartResp.Body) + c.Assert(err, IsNil) + fmt.Println(string(body)) + + uploadPartXml := struct { + ETag string `xml:"ETag"` + }{} + + err = xml.Unmarshal(body, &uploadPartXml) + c.Assert(err, IsNil) + + etag := uploadPartXml.ETag + + // 生成complete外链 + completeUrl, err := client.GeneratePresignedUrl(&s3.GeneratePresignedUrlInput{ + HTTPMethod: s3.POST, + Bucket: aws.String(bucket), + Key: aws.String(object + "copy"), + Expires: 3600, + Parameters: map[string]*string{ + "uploadId": aws.String(initXml.UploadId), + }, + }) + c.Assert(err, IsNil) + fmt.Println(completeUrl) + + completeParts := `<CompleteMultipartUpload> + <Part> + <PartNumber>1</PartNumber> + <ETag>` + etag + `</ETag> + </Part> + </CompleteMultipartUpload>` + + fmt.Println(completeParts) + + completeRequest, err := http.NewRequest("POST", completeUrl, strings.NewReader(completeParts)) + c.Assert(err, IsNil) + + completeResp, err := http.DefaultClient.Do(completeRequest) + c.Assert(err, IsNil) + + body, err = io.ReadAll(completeResp.Body) + c.Assert(err, IsNil) + + fmt.Println(string(body)) + + // 获取head外链 + headUrl, err := client.GeneratePresignedUrl(&s3.GeneratePresignedUrlInput{ + HTTPMethod: s3.HEAD, + Bucket: aws.String(bucket), + Key: aws.String(object + "copy"), + Expires: 3600, + }) + c.Assert(err, IsNil) + fmt.Println(headUrl) + + headRequest, err := http.NewRequest("HEAD", headUrl, nil) + c.Assert(err, IsNil) + + headResp, err := http.DefaultClient.Do(headRequest) + c.Assert(err, IsNil) + c.Assert(headResp.StatusCode, Equals, 200) + + os.Remove(object) +} From 580f8a5a209f391ff5bc3fbceee452c1ddf13d53 Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Mon, 23 Sep 2024 10:24:07 +0800 Subject: [PATCH 03/13] =?UTF-8?q?test:=20=E5=8D=95=E5=85=83=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/object_encryption_test.go | 24 ++++----- test/objectsample_test.go | 24 ++++----- test/objectwithcontext_test.go | 94 +++++++++++++++------------------- 3 files changed, 61 insertions(+), 81 deletions(-) diff --git a/test/object_encryption_test.go b/test/object_encryption_test.go index 4cde7cd..772f945 100644 --- a/test/object_encryption_test.go +++ b/test/object_encryption_test.go @@ -195,10 +195,9 @@ func (s *Ks3utilCommandSuite) TestMultipartUploadWithSSE_S3(c *C) { c.Assert(*initRet.ServerSideEncryption, Equals, "AES256") // 获取分块上传Id uploadId := *initRet.UploadID - var i int64 = 1 + var partNum int64 = 1 // 待合并分块 - compParts := []*s3.CompletedPart{} - partsNum := []int64{0} + var compParts []*s3.CompletedPart // 缓冲区,分块大小为5MB buffer := make([]byte, 5*1024*1024) for { @@ -212,15 +211,14 @@ func (s *Ks3utilCommandSuite) TestMultipartUploadWithSSE_S3(c *C) { resp, err := client.UploadPartWithContext(context.Background(), &s3.UploadPartInput{ Bucket: aws.String(bucket), Key: aws.String(object), - PartNumber: aws.Long(i), + PartNumber: aws.Long(partNum), UploadID: aws.String(uploadId), Body: bytes.NewReader(buffer[:n]), }) c.Assert(err, IsNil) c.Assert(*resp.ServerSideEncryption, Equals, "AES256") - partsNum = append(partsNum, i) - compParts = append(compParts, &s3.CompletedPart{PartNumber: &partsNum[i], ETag: resp.ETag}) - i++ + compParts = append(compParts, &s3.CompletedPart{PartNumber: aws.Long(partNum), ETag: resp.ETag}) + partNum++ } } // complete @@ -263,10 +261,9 @@ func (s *Ks3utilCommandSuite) TestMultipartUploadWithSSE_C(c *C) { c.Assert(*initRet.SSECustomerKeyMD5, Equals, s3.GetBase64MD5Str(customerKey)) // 获取分块上传Id uploadId := *initRet.UploadID - var i int64 = 1 + var partNum int64 = 1 // 待合并分块 - compParts := []*s3.CompletedPart{} - partsNum := []int64{0} + var compParts []*s3.CompletedPart // 缓冲区,分块大小为5MB buffer := make([]byte, 5*1024*1024) for { @@ -280,7 +277,7 @@ func (s *Ks3utilCommandSuite) TestMultipartUploadWithSSE_C(c *C) { resp, err := client.UploadPartWithContext(context.Background(), &s3.UploadPartInput{ Bucket: aws.String(bucket), Key: aws.String(object), - PartNumber: aws.Long(i), + PartNumber: aws.Long(partNum), UploadID: aws.String(uploadId), Body: bytes.NewReader(buffer[:n]), SSECustomerAlgorithm: aws.String("AES256"), @@ -290,9 +287,8 @@ func (s *Ks3utilCommandSuite) TestMultipartUploadWithSSE_C(c *C) { c.Assert(err, IsNil) c.Assert(*resp.SSECustomerAlgorithm, Equals, "AES256") c.Assert(*resp.SSECustomerKeyMD5, Equals, s3.GetBase64MD5Str(customerKey)) - partsNum = append(partsNum, i) - compParts = append(compParts, &s3.CompletedPart{PartNumber: &partsNum[i], ETag: resp.ETag}) - i++ + compParts = append(compParts, &s3.CompletedPart{PartNumber: aws.Long(partNum), ETag: resp.ETag}) + partNum++ } } // complete diff --git a/test/objectsample_test.go b/test/objectsample_test.go index c4944da..00099ff 100644 --- a/test/objectsample_test.go +++ b/test/objectsample_test.go @@ -315,14 +315,13 @@ func (s *Ks3utilCommandSuite) TestMultipartUpload(c *C) { c.Assert(err, IsNil) defer f.Close() - var i int64 = 1 - //组装分块参数 + var partNum int64 = 1 + // 待合并分块 var compParts []*s3.CompletedPart - partsNum := []int64{0} - sc := make([]byte, 52428800) - + // 缓冲区,分块大小为5MB + buffer := make([]byte, 5*1024*1024) for { - nr, err := f.Read(sc[:]) + nr, err := f.Read(buffer) if nr < 0 { fmt.Fprintf(os.Stderr, "cat: error reading: %s\n", err.Error()) os.Exit(1) @@ -335,21 +334,20 @@ func (s *Ks3utilCommandSuite) TestMultipartUpload(c *C) { //块的数量可以是1到10,000中的任意一个(包含1和10,000)。块序号用于标识一个块以及其在对象创建时的位置。如果你上传一个新的块,使用之前已经使用的序列号,那么之前的那个块将会被覆盖。当所有块总大小大于5M时,除了最后一个块没有大小限制外,其余的块的大小均要求在5MB以上。当所有块总大小小于5M时,除了最后一个块没有大小限制外,其余的块的大小均要求在100K以上。如果不符合上述要求,会返回413状态码。 //为了保证数据在传输过程中没有损坏,请使用 Content-MD5 头部。当使用此头部时,KS3会自动计算出MD5,并根据用户提供的MD5进行校验,如果不匹配,将会返回错误信息。 //计算sc[:nr]的md5值 - md5 := s3.GetBase64MD5Str(string(sc[0:nr])) + md5 := s3.GetBase64MD5Str(string(buffer[0:nr])) resp, err := client.UploadPart(&s3.UploadPartInput{ Bucket: aws.String(bucket), Key: aws.String(key), - PartNumber: aws.Long(i), + PartNumber: aws.Long(partNum), UploadID: aws.String(uploadId), - Body: bytes.NewReader(sc[0:nr]), - ContentLength: aws.Long(int64(len(sc[0:nr]))), + Body: bytes.NewReader(buffer[0:nr]), + ContentLength: aws.Long(int64(len(buffer[0:nr]))), //TrafficLimit: aws.Long(int64(MIN_BANDWIDTH)), ContentMD5: aws.String(md5), }) c.Assert(err, IsNil) - partsNum = append(partsNum, i) - compParts = append(compParts, &s3.CompletedPart{PartNumber: &partsNum[i], ETag: resp.ETag}) - i++ + compParts = append(compParts, &s3.CompletedPart{PartNumber: aws.Long(partNum), ETag: resp.ETag}) + partNum++ } } diff --git a/test/objectwithcontext_test.go b/test/objectwithcontext_test.go index ca00570..66452c8 100644 --- a/test/objectwithcontext_test.go +++ b/test/objectwithcontext_test.go @@ -667,10 +667,9 @@ func (s *Ks3utilCommandSuite) TestCreateMultipartUploadWithContext(c *C) { c.Assert(err, IsNil) // 获取分块上传Id uploadId := *initRet.UploadID - var i int64 = 1 + var partNum int64 = 1 // 待合并分块 - compParts := []*s3.CompletedPart{} - partsNum := []int64{0} + var compParts []*s3.CompletedPart // 缓冲区,分块大小为5MB buffer := make([]byte, 5*1024*1024) for { @@ -684,14 +683,13 @@ func (s *Ks3utilCommandSuite) TestCreateMultipartUploadWithContext(c *C) { resp, err := client.UploadPartWithContext(context.Background(), &s3.UploadPartInput{ Bucket: aws.String(bucket), Key: aws.String(object), - PartNumber: aws.Long(i), + PartNumber: aws.Long(partNum), UploadID: aws.String(uploadId), Body: bytes.NewReader(buffer[:n]), }) c.Assert(err, IsNil) - partsNum = append(partsNum, i) - compParts = append(compParts, &s3.CompletedPart{PartNumber: &partsNum[i], ETag: resp.ETag}) - i++ + compParts = append(compParts, &s3.CompletedPart{PartNumber: aws.Long(partNum), ETag: resp.ETag}) + partNum++ } } // complete @@ -726,10 +724,9 @@ func (s *Ks3utilCommandSuite) TestUploadPartWithContext(c *C) { c.Assert(err, IsNil) // 获取分块上传Id uploadId := *initRet.UploadID - var i int64 = 1 + var partNum int64 = 1 // 待合并分块 - compParts := []*s3.CompletedPart{} - partsNum := []int64{0} + var compParts []*s3.CompletedPart // 缓冲区,分块大小为5MB buffer := make([]byte, 5*1024*1024) ctx, cancelFunc := context.WithTimeout(context.Background(), time.Millisecond*500) @@ -745,7 +742,7 @@ func (s *Ks3utilCommandSuite) TestUploadPartWithContext(c *C) { _, err := client.UploadPartWithContext(ctx, &s3.UploadPartInput{ Bucket: aws.String(bucket), Key: aws.String(object), - PartNumber: aws.Long(i), + PartNumber: aws.Long(partNum), UploadID: aws.String(uploadId), Body: bytes.NewReader(buffer[:n]), }) @@ -765,14 +762,13 @@ func (s *Ks3utilCommandSuite) TestUploadPartWithContext(c *C) { resp, err := client.UploadPartWithContext(context.Background(), &s3.UploadPartInput{ Bucket: aws.String(bucket), Key: aws.String(object), - PartNumber: aws.Long(i), + PartNumber: aws.Long(partNum), UploadID: aws.String(uploadId), Body: bytes.NewReader(buffer[:n]), }) c.Assert(err, IsNil) - partsNum = append(partsNum, i) - compParts = append(compParts, &s3.CompletedPart{PartNumber: &partsNum[i], ETag: resp.ETag}) - i++ + compParts = append(compParts, &s3.CompletedPart{PartNumber: aws.Long(partNum), ETag: resp.ETag}) + partNum++ } } // complete @@ -807,10 +803,9 @@ func (s *Ks3utilCommandSuite) TestCompleteMultipartUploadWithContext(c *C) { c.Assert(err, IsNil) // 获取分块上传Id uploadId := *initRet.UploadID - var i int64 = 1 + var partNum int64 = 1 // 待合并分块 - compParts := []*s3.CompletedPart{} - partsNum := []int64{0} + var compParts []*s3.CompletedPart // 缓冲区,分块大小为5MB buffer := make([]byte, 5*1024*1024) for { @@ -824,14 +819,13 @@ func (s *Ks3utilCommandSuite) TestCompleteMultipartUploadWithContext(c *C) { resp, err := client.UploadPartWithContext(context.Background(), &s3.UploadPartInput{ Bucket: aws.String(bucket), Key: aws.String(object), - PartNumber: aws.Long(i), + PartNumber: aws.Long(partNum), UploadID: aws.String(uploadId), Body: bytes.NewReader(buffer[:n]), }) c.Assert(err, IsNil) - partsNum = append(partsNum, i) - compParts = append(compParts, &s3.CompletedPart{PartNumber: &partsNum[i], ETag: resp.ETag}) - i++ + compParts = append(compParts, &s3.CompletedPart{PartNumber: aws.Long(partNum), ETag: resp.ETag}) + partNum++ } } // complete,通过context取消 @@ -889,10 +883,9 @@ func (s *Ks3utilCommandSuite) TestAbortMultipartUploadWithContext(c *C) { c.Assert(err, IsNil) // 获取分块上传Id uploadId := *initRet.UploadID - var i int64 = 1 + var partNum int64 = 1 // 待合并分块 - compParts := []*s3.CompletedPart{} - partsNum := []int64{0} + var compParts []*s3.CompletedPart // 缓冲区,分块大小为5MB buffer := make([]byte, 5*1024*1024) for { @@ -906,14 +899,13 @@ func (s *Ks3utilCommandSuite) TestAbortMultipartUploadWithContext(c *C) { resp, err := client.UploadPartWithContext(context.Background(), &s3.UploadPartInput{ Bucket: aws.String(bucket), Key: aws.String(object), - PartNumber: aws.Long(i), + PartNumber: aws.Long(partNum), UploadID: aws.String(uploadId), Body: bytes.NewReader(buffer[:n]), }) c.Assert(err, IsNil) - partsNum = append(partsNum, i) - compParts = append(compParts, &s3.CompletedPart{PartNumber: &partsNum[i], ETag: resp.ETag}) - i++ + compParts = append(compParts, &s3.CompletedPart{PartNumber: aws.Long(partNum), ETag: resp.ETag}) + partNum++ } } // abort,通过context取消 @@ -959,10 +951,9 @@ func (s *Ks3utilCommandSuite) TestListPartsWithContext(c *C) { c.Assert(err, IsNil) // 获取分块上传Id uploadId := *initRet.UploadID - var i int64 = 1 + var partNum int64 = 1 // 待合并分块 - compParts := []*s3.CompletedPart{} - partsNum := []int64{0} + var compParts []*s3.CompletedPart // 缓冲区,分块大小为5MB buffer := make([]byte, 5*1024*1024) for { @@ -976,14 +967,13 @@ func (s *Ks3utilCommandSuite) TestListPartsWithContext(c *C) { resp, err := client.UploadPartWithContext(context.Background(), &s3.UploadPartInput{ Bucket: aws.String(bucket), Key: aws.String(object), - PartNumber: aws.Long(i), + PartNumber: aws.Long(partNum), UploadID: aws.String(uploadId), Body: bytes.NewReader(buffer[:n]), }) c.Assert(err, IsNil) - partsNum = append(partsNum, i) - compParts = append(compParts, &s3.CompletedPart{PartNumber: &partsNum[i], ETag: resp.ETag}) - i++ + compParts = append(compParts, &s3.CompletedPart{PartNumber: aws.Long(partNum), ETag: resp.ETag}) + partNum++ } } // list,通过context取消 @@ -1026,10 +1016,9 @@ func (s *Ks3utilCommandSuite) TestListMultipartUploadsWithContext(c *C) { c.Assert(err, IsNil) // 获取分块上传Id uploadId := *initRet.UploadID - var i int64 = 1 + var partNum int64 = 1 // 待合并分块 - compParts := []*s3.CompletedPart{} - partsNum := []int64{0} + var compParts []*s3.CompletedPart // 缓冲区,分块大小为5MB buffer := make([]byte, 5*1024*1024) for { @@ -1043,14 +1032,13 @@ func (s *Ks3utilCommandSuite) TestListMultipartUploadsWithContext(c *C) { resp, err := client.UploadPartWithContext(context.Background(), &s3.UploadPartInput{ Bucket: aws.String(bucket), Key: aws.String(object), - PartNumber: aws.Long(i), + PartNumber: aws.Long(partNum), UploadID: aws.String(uploadId), Body: bytes.NewReader(buffer[:n]), }) c.Assert(err, IsNil) - partsNum = append(partsNum, i) - compParts = append(compParts, &s3.CompletedPart{PartNumber: &partsNum[i], ETag: resp.ETag}) - i++ + compParts = append(compParts, &s3.CompletedPart{PartNumber: aws.Long(partNum), ETag: resp.ETag}) + partNum++ } } // list mul,通过context取消 @@ -1101,11 +1089,10 @@ func (s *Ks3utilCommandSuite) TestPartWithContext(c *C) { c.Assert(err, IsNil) c.Assert(*headObjectResp.StatusCode, Equals, int64(200)) contentLength := *headObjectResp.ContentLength - partSize := int64(5 * 1024 * 1024) - var i int64 = 1 + var partSize int64 = 5 * 1024 * 1024 + var partNum int64 = 1 // 待合并分块 - compParts := []*s3.CompletedPart{} - partsNum := []int64{0} + var compParts []*s3.CompletedPart var start int64 = 0 var end int64 = 0 dstObject := randLowStr(10) @@ -1134,14 +1121,14 @@ func (s *Ks3utilCommandSuite) TestPartWithContext(c *C) { Key: aws.String(dstObject), CopySource: aws.String("/" + bucket + "/" + srcObject), UploadID: aws.String(uploadId), - PartNumber: aws.Long(i), + PartNumber: aws.Long(partNum), CopySourceRange: aws.String("bytes=" + strconv.FormatInt(start, 10) + "-" + strconv.FormatInt(end-1, 10)), }) c.Assert(err, NotNil) - i++ + partNum++ start = end } - i = 1 + partNum = 1 start = 0 end = 0 for { @@ -1159,13 +1146,12 @@ func (s *Ks3utilCommandSuite) TestPartWithContext(c *C) { Key: aws.String(dstObject), CopySource: aws.String("/" + bucket + "/" + srcObject), UploadID: aws.String(uploadId), - PartNumber: aws.Long(i), + PartNumber: aws.Long(partNum), CopySourceRange: aws.String("bytes=" + strconv.FormatInt(start, 10) + "-" + strconv.FormatInt(end-1, 10)), }) c.Assert(err, IsNil) - partsNum = append(partsNum, i) - compParts = append(compParts, &s3.CompletedPart{PartNumber: &partsNum[i], ETag: resp.CopyPartResult.ETag}) - i++ + compParts = append(compParts, &s3.CompletedPart{PartNumber: aws.Long(partNum), ETag: resp.CopyPartResult.ETag}) + partNum++ start = end } // complete From 73b4adb48a50760ef2f733884306d6fd13deff55 Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Wed, 9 Oct 2024 14:20:15 +0800 Subject: [PATCH 04/13] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E5=A4=8D?= =?UTF-8?q?=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/signer/v2/v2.go | 6 + service/s3/api.go | 234 ------------------------ service/s3/decompresspolicy.go | 12 +- service/s3/inventory.go | 1 + service/s3/replication.go | 193 ++++++++++++++++++++ service/s3/retention.go | 318 +++++++++++++++++++++++++++++++++ test/bucketsample_test.go | 65 +++++++ test/bucketwithcontext_test.go | 127 +++++++++++++ 8 files changed, 716 insertions(+), 240 deletions(-) create mode 100644 service/s3/inventory.go create mode 100644 service/s3/replication.go create mode 100644 service/s3/retention.go diff --git a/internal/signer/v2/v2.go b/internal/signer/v2/v2.go index 5d5d061..822dc9d 100644 --- a/internal/signer/v2/v2.go +++ b/internal/signer/v2/v2.go @@ -59,6 +59,12 @@ var signQuerys = map[string]bool{ "append": true, "position": true, "decompresspolicy": true, + "retention": true, + "crr": true, + "inventory": true, + "recycle": true, + "recover": true, + "clear": true, } type signer struct { diff --git a/service/s3/api.go b/service/s3/api.go index 0d12e04..4c043db 100755 --- a/service/s3/api.go +++ b/service/s3/api.go @@ -314,44 +314,6 @@ func (c *S3) DeleteBucketPolicyWithContext(ctx aws.Context, input *DeleteBucketP var opDeleteBucketPolicy *aws.Operation -// DeleteBucketReplicationRequest generates a request for the DeleteBucketReplication operation. -func (c *S3) DeleteBucketReplicationRequest(input *DeleteBucketReplicationInput) (req *aws.Request, output *DeleteBucketReplicationOutput) { - oprw.Lock() - defer oprw.Unlock() - - if opDeleteBucketReplication == nil { - opDeleteBucketReplication = &aws.Operation{ - Name: "DeleteBucketReplication", - HTTPMethod: "DELETE", - HTTPPath: "/{Bucket}?replication", - } - } - - if input == nil { - input = &DeleteBucketReplicationInput{} - } - - req = c.newRequest(opDeleteBucketReplication, input, output) - output = &DeleteBucketReplicationOutput{} - req.Data = output - return -} - -func (c *S3) DeleteBucketReplication(input *DeleteBucketReplicationInput) (*DeleteBucketReplicationOutput, error) { - req, out := c.DeleteBucketReplicationRequest(input) - err := req.Send() - return out, err -} - -func (c *S3) DeleteBucketReplicationWithContext(ctx aws.Context, input *DeleteBucketReplicationInput) (*DeleteBucketReplicationOutput, error) { - req, out := c.DeleteBucketReplicationRequest(input) - req.SetContext(ctx) - err := req.Send() - return out, err -} - -var opDeleteBucketReplication *aws.Operation - // DeleteBucketTaggingRequest generates a request for the DeleteBucketTagging operation. func (c *S3) DeleteBucketTaggingRequest(input *DeleteBucketTaggingInput) (req *aws.Request, output *DeleteBucketTaggingOutput) { oprw.Lock() @@ -829,44 +791,6 @@ func (c *S3) GetBucketPolicyWithContext(ctx aws.Context, input *GetBucketPolicyI var opGetBucketPolicy *aws.Operation -// GetBucketReplicationRequest generates a request for the GetBucketReplication operation. -func (c *S3) GetBucketReplicationRequest(input *GetBucketReplicationInput) (req *aws.Request, output *GetBucketReplicationOutput) { - oprw.Lock() - defer oprw.Unlock() - - if opGetBucketReplication == nil { - opGetBucketReplication = &aws.Operation{ - Name: "GetBucketReplication", - HTTPMethod: "GET", - HTTPPath: "/{Bucket}?replication", - } - } - - if input == nil { - input = &GetBucketReplicationInput{} - } - - req = c.newRequest(opGetBucketReplication, input, output) - output = &GetBucketReplicationOutput{} - req.Data = output - return -} - -func (c *S3) GetBucketReplication(input *GetBucketReplicationInput) (*GetBucketReplicationOutput, error) { - req, out := c.GetBucketReplicationRequest(input) - err := req.Send() - return out, err -} - -func (c *S3) GetBucketReplicationWithContext(ctx aws.Context, input *GetBucketReplicationInput) (*GetBucketReplicationOutput, error) { - req, out := c.GetBucketReplicationRequest(input) - req.SetContext(ctx) - err := req.Send() - return out, err -} - -var opGetBucketReplication *aws.Operation - // GetBucketRequestPaymentRequest generates a request for the GetBucketRequestPayment operation. func (c *S3) GetBucketRequestPaymentRequest(input *GetBucketRequestPaymentInput) (req *aws.Request, output *GetBucketRequestPaymentOutput) { oprw.Lock() @@ -1750,45 +1674,6 @@ func (c *S3) PutBucketPolicyWithContext(ctx aws.Context, input *PutBucketPolicyI var opPutBucketPolicy *aws.Operation -// PutBucketReplicationRequest generates a request for the PutBucketReplication operation. -func (c *S3) PutBucketReplicationRequest(input *PutBucketReplicationInput) (req *aws.Request, output *PutBucketReplicationOutput) { - oprw.Lock() - defer oprw.Unlock() - - if opPutBucketReplication == nil { - opPutBucketReplication = &aws.Operation{ - Name: "PutBucketReplication", - HTTPMethod: "PUT", - HTTPPath: "/{Bucket}?replication", - } - } - - if input == nil { - input = &PutBucketReplicationInput{} - } - - req = c.newRequest(opPutBucketReplication, input, output) - output = &PutBucketReplicationOutput{} - req.Data = output - return -} - -// PutBucketReplication Creates a new replication configuration (or replaces an existing one, if present). -func (c *S3) PutBucketReplication(input *PutBucketReplicationInput) (*PutBucketReplicationOutput, error) { - req, out := c.PutBucketReplicationRequest(input) - err := req.Send() - return out, err -} - -func (c *S3) PutBucketReplicationWithContext(ctx aws.Context, input *PutBucketReplicationInput) (*PutBucketReplicationOutput, error) { - req, out := c.PutBucketReplicationRequest(input) - req.SetContext(ctx) - err := req.Send() - return out, err -} - -var opPutBucketReplication *aws.Operation - // PutBucketRequestPaymentRequest generates a request for the PutBucketRequestPayment operation. func (c *S3) PutBucketRequestPaymentRequest(input *PutBucketRequestPaymentInput) (req *aws.Request, output *PutBucketRequestPaymentOutput) { oprw.Lock() @@ -3001,30 +2886,6 @@ type metadataDeleteBucketPolicyOutput struct { SDKShapeTraits bool `type:"structure"` } -type DeleteBucketReplicationInput struct { - Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - - ContentType *string `location:"header" locationName:"Content-Type" type:"string"` - - metadataDeleteBucketReplicationInput `json:"-" xml:"-"` -} - -type metadataDeleteBucketReplicationInput struct { - SDKShapeTraits bool `type:"structure"` -} - -type DeleteBucketReplicationOutput struct { - metadataDeleteBucketReplicationOutput `json:"-" xml:"-"` - - Metadata map[string]*string `location:"headers" type:"map"` - - StatusCode *int64 `location:"statusCode" type:"integer"` -} - -type metadataDeleteBucketReplicationOutput struct { - SDKShapeTraits bool `type:"structure"` -} - type DeleteBucketTaggingInput struct { Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -3380,34 +3241,6 @@ type metadataGetBucketPolicyOutput struct { SDKShapeTraits bool `type:"structure" payload:"Policy"` } -type GetBucketReplicationInput struct { - Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - - ContentType *string `location:"header" locationName:"Content-Type" type:"string"` - - metadataGetBucketReplicationInput `json:"-" xml:"-"` -} - -type metadataGetBucketReplicationInput struct { - SDKShapeTraits bool `type:"structure"` -} - -type GetBucketReplicationOutput struct { - // Container for replication rules. You can add as many as 1,000 rules. Total - // replication configuration size can be up to 2 MB. - ReplicationConfiguration *ReplicationConfiguration `type:"structure"` - - metadataGetBucketReplicationOutput `json:"-" xml:"-"` - - Metadata map[string]*string `location:"headers" type:"map"` - - StatusCode *int64 `location:"statusCode" type:"integer"` -} - -type metadataGetBucketReplicationOutput struct { - SDKShapeTraits bool `type:"structure" payload:"ReplicationConfiguration"` -} - type GetBucketRequestPaymentInput struct { Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -4736,34 +4569,6 @@ type metadataPutBucketPolicyOutput struct { SDKShapeTraits bool `type:"structure"` } -type PutBucketReplicationInput struct { - Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - - // Container for replication rules. You can add as many as 1,000 rules. Total - // replication configuration size can be up to 2 MB. - ReplicationConfiguration *ReplicationConfiguration `locationName:"ReplicationConfiguration" type:"structure" required:"true"` - - ContentType *string `location:"header" locationName:"Content-Type" type:"string"` - - metadataPutBucketReplicationInput `json:"-" xml:"-"` -} - -type metadataPutBucketReplicationInput struct { - SDKShapeTraits bool `type:"structure" payload:"ReplicationConfiguration"` -} - -type PutBucketReplicationOutput struct { - metadataPutBucketReplicationOutput `json:"-" xml:"-"` - - Metadata map[string]*string `location:"headers" type:"map"` - - StatusCode *int64 `location:"statusCode" type:"integer"` -} - -type metadataPutBucketReplicationOutput struct { - SDKShapeTraits bool `type:"structure"` -} - type PutBucketRequestPaymentInput struct { Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -5265,45 +5070,6 @@ type metadataRedirectAllRequestsTo struct { SDKShapeTraits bool `type:"structure"` } -// ReplicationConfiguration Container for replication rules. You can add as many as 1,000 rules. Total -// replication configuration size can be up to 2 MB. -type ReplicationConfiguration struct { - // Amazon Resource Name (ARN) of an IAM role for Amazon S3 to assume when replicating - // the objects. - Role *string `type:"string" required:"true"` - - // Container for information about a particular replication rule. Replication - // configuration must have at least one rule and can contain up to 1,000 rules. - Rules []*ReplicationRule `locationName:"Rule" type:"list" flattened:"true" required:"true"` - - metadataReplicationConfiguration `json:"-" xml:"-"` -} - -type metadataReplicationConfiguration struct { - SDKShapeTraits bool `type:"structure"` -} - -type ReplicationRule struct { - Destination *Destination `type:"structure" required:"true"` - - // Unique identifier for the rule. The value cannot be longer than 255 characters. - ID *string `type:"string"` - - // Object keyname prefix identifying one or more objects to which the rule applies. - // Maximum prefix length can be up to 1,024 characters. Overlapping prefixes - // are not supported. - Prefix *string `type:"string" required:"true"` - - // The rule is ignored if status is not Enabled. - Status *string `type:"string" required:"true"` - - metadataReplicationRule `json:"-" xml:"-"` -} - -type metadataReplicationRule struct { - SDKShapeTraits bool `type:"structure"` -} - type RequestPaymentConfiguration struct { // Specifies who pays for the download and request fees. Payer *string `type:"string" required:"true"` diff --git a/service/s3/decompresspolicy.go b/service/s3/decompresspolicy.go index a0f9479..1dd67c2 100644 --- a/service/s3/decompresspolicy.go +++ b/service/s3/decompresspolicy.go @@ -54,7 +54,7 @@ type metadataPutBucketDecompressPolicyInput struct { } type BucketDecompressPolicy struct { - Rules []*DecompressPolicyRule `json:"rules,omitempty" type:"list" locationName:"rules" required:"true"` + Rules []*DecompressPolicyRule `json:"rules,omitempty" type:"list" locationName:"rules" required:"true"` } type DecompressPolicyRule struct { @@ -105,7 +105,7 @@ type DecompressPolicyRule struct { } type PutBucketDecompressPolicyOutput struct { - Metadata map[string]*string `location:"headers" type:"map"` + Metadata map[string]*string `location:"headers" type:"map"` StatusCode *int64 `location:"statusCode" type:"integer"` } @@ -156,14 +156,14 @@ type GetBucketDecompressPolicyInput struct { type GetBucketDecompressPolicyOutput struct { BucketDecompressPolicy *BucketDecompressPolicy `locationName:"BucketDecompressPolicy" type:"structure"` - Metadata map[string]*string `location:"headers" type:"map"` + Metadata map[string]*string `location:"headers" type:"map"` StatusCode *int64 `location:"statusCode" type:"integer"` - metadataGetBucketDecompressPolicyInput `json:"-" xml:"-"` + metadataGetBucketDecompressPolicyOutput `json:"-" xml:"-"` } -type metadataGetBucketDecompressPolicyInput struct { +type metadataGetBucketDecompressPolicyOutput struct { SDKShapeTraits bool `type:"structure" payload:"BucketDecompressPolicy"` } @@ -207,7 +207,7 @@ type DeleteBucketDecompressPolicyInput struct { Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } type DeleteBucketDecompressPolicyOutput struct { - Metadata map[string]*string `location:"headers" type:"map"` + Metadata map[string]*string `location:"headers" type:"map"` StatusCode *int64 `location:"statusCode" type:"integer"` } diff --git a/service/s3/inventory.go b/service/s3/inventory.go new file mode 100644 index 0000000..3ed7f97 --- /dev/null +++ b/service/s3/inventory.go @@ -0,0 +1 @@ +package s3 diff --git a/service/s3/replication.go b/service/s3/replication.go new file mode 100644 index 0000000..40bc018 --- /dev/null +++ b/service/s3/replication.go @@ -0,0 +1,193 @@ +package s3 + +import "github.com/ks3sdklib/aws-sdk-go/aws" + +// PutBucketReplicationRequest generates a request for the PutBucketReplication operation. +func (c *S3) PutBucketReplicationRequest(input *PutBucketReplicationInput) (req *aws.Request, output *PutBucketReplicationOutput) { + oprw.Lock() + defer oprw.Unlock() + + if opPutBucketReplication == nil { + opPutBucketReplication = &aws.Operation{ + Name: "PutBucketReplication", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?crr", + } + } + + if input == nil { + input = &PutBucketReplicationInput{} + } + + input.AutoFillMD5 = true + req = c.newRequest(opPutBucketReplication, input, output) + output = &PutBucketReplicationOutput{} + req.Data = output + return +} + +// PutBucketReplication creates a new replication configuration. +func (c *S3) PutBucketReplication(input *PutBucketReplicationInput) (*PutBucketReplicationOutput, error) { + req, out := c.PutBucketReplicationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketReplicationWithContext(ctx aws.Context, input *PutBucketReplicationInput) (*PutBucketReplicationOutput, error) { + req, out := c.PutBucketReplicationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +var opPutBucketReplication *aws.Operation + +type PutBucketReplicationInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ReplicationConfiguration *ReplicationConfiguration `locationName:"Replication" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + metadataPutBucketReplicationInput `json:"-" xml:"-"` +} + +type metadataPutBucketReplicationInput struct { + SDKShapeTraits bool `type:"structure" payload:"ReplicationConfiguration"` + + AutoFillMD5 bool +} + +type PutBucketReplicationOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type ReplicationConfiguration struct { + // Prefix matching, only objects that match prefix rules will be copied. Each copying rule + // can add up to 10 prefix matching rules, and prefixes cannot overlap with each other. + Prefix []*string `locationName:"prefix" type:"list" flattened:"true"` + + // Indicate whether to enable delete replication. If set to Enabled, it means enabled; if set to + // Disabled or not, it means disabled. If set to delete replication, when the source bucket deletes + // an object, the replica of that object in the target bucket will also be deleted. + DeleteMarkerStatus *string `locationName:"DeleteMarkerStatus" type:"string" required:"true"` + + // Target bucket for copying rules. + TargetBucket *string `locationName:"targetBucket" type:"string" required:"true"` + + // Specify whether to copy historical data. Whether to copy the data from the source bucket + // to the target bucket before enabling data replication. + // Enabled: Copy historical data to the target bucket (default value) + // Disabled: Do not copy historical data, only copy new data after enabling the rule to the target bucket. + HistoricalObjectReplication *string `locationName:"HistoricalObjectReplication" type:"string"` + + // Region of the target bucket. + Region *string `locationName:"region" type:"string"` +} + +// GetBucketReplicationRequest generates a request for the GetBucketReplication operation. +func (c *S3) GetBucketReplicationRequest(input *GetBucketReplicationInput) (req *aws.Request, output *GetBucketReplicationOutput) { + oprw.Lock() + defer oprw.Unlock() + + if opGetBucketReplication == nil { + opGetBucketReplication = &aws.Operation{ + Name: "GetBucketReplication", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?crr", + } + } + + if input == nil { + input = &GetBucketReplicationInput{} + } + + req = c.newRequest(opGetBucketReplication, input, output) + output = &GetBucketReplicationOutput{} + req.Data = output + return +} + +// GetBucketReplication gets the replication configuration for the bucket. +func (c *S3) GetBucketReplication(input *GetBucketReplicationInput) (*GetBucketReplicationOutput, error) { + req, out := c.GetBucketReplicationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketReplicationWithContext(ctx aws.Context, input *GetBucketReplicationInput) (*GetBucketReplicationOutput, error) { + req, out := c.GetBucketReplicationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +var opGetBucketReplication *aws.Operation + +type GetBucketReplicationInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` +} + +type GetBucketReplicationOutput struct { + ReplicationConfiguration *ReplicationConfiguration `locationName:"Replication" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketReplicationOutput `json:"-" xml:"-"` +} + +type metadataGetBucketReplicationOutput struct { + SDKShapeTraits bool `type:"structure" payload:"ReplicationConfiguration"` +} + +// DeleteBucketReplicationRequest generates a request for the DeleteBucketReplication operation. +func (c *S3) DeleteBucketReplicationRequest(input *DeleteBucketReplicationInput) (req *aws.Request, output *DeleteBucketReplicationOutput) { + oprw.Lock() + defer oprw.Unlock() + + if opDeleteBucketReplication == nil { + opDeleteBucketReplication = &aws.Operation{ + Name: "DeleteBucketReplication", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?crr", + } + } + + if input == nil { + input = &DeleteBucketReplicationInput{} + } + + req = c.newRequest(opDeleteBucketReplication, input, output) + output = &DeleteBucketReplicationOutput{} + req.Data = output + return +} + +// DeleteBucketReplication deletes the replication configuration for the bucket. +func (c *S3) DeleteBucketReplication(input *DeleteBucketReplicationInput) (*DeleteBucketReplicationOutput, error) { + req, out := c.DeleteBucketReplicationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketReplicationWithContext(ctx aws.Context, input *DeleteBucketReplicationInput) (*DeleteBucketReplicationOutput, error) { + req, out := c.DeleteBucketReplicationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +var opDeleteBucketReplication *aws.Operation + +type DeleteBucketReplicationInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` +} +type DeleteBucketReplicationOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} diff --git a/service/s3/retention.go b/service/s3/retention.go new file mode 100644 index 0000000..2399f3e --- /dev/null +++ b/service/s3/retention.go @@ -0,0 +1,318 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" + "time" +) + +// PutBucketRetentionRequest generates a request for the PutBucketRetention operation. +func (c *S3) PutBucketRetentionRequest(input *PutBucketRetentionInput) (req *aws.Request, output *PutBucketRetentionOutput) { + oprw.Lock() + defer oprw.Unlock() + if opPutBucketRetention == nil { + opPutBucketRetention = &aws.Operation{ + Name: "PutBucketRetention", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?retention", + } + } + if input == nil { + input = &PutBucketRetentionInput{} + } + input.AutoFillMD5 = true + req = c.newRequest(opPutBucketRetention, input, output) + output = &PutBucketRetentionOutput{} + req.Data = output + return +} + +// PutBucketRetention sets the retention configuration on a bucket. +func (c *S3) PutBucketRetention(input *PutBucketRetentionInput) (*PutBucketRetentionOutput, error) { + req, out := c.PutBucketRetentionRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketRetentionWithContext(ctx aws.Context, input *PutBucketRetentionInput) (*PutBucketRetentionOutput, error) { + req, out := c.PutBucketRetentionRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +var opPutBucketRetention *aws.Operation + +type PutBucketRetentionInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + RetentionConfiguration *BucketRetentionConfiguration `locationName:"RetentionConfiguration" type:"structure"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + metadataPutBucketRetentionInput `json:"-" xml:"-"` +} + +type metadataPutBucketRetentionInput struct { + SDKShapeTraits bool `type:"structure" payload:"RetentionConfiguration"` + + AutoFillMD5 bool +} + +type BucketRetentionConfiguration struct { + // A container that contains a specific rule for the recycle bin. + Rule *RetentionRule `locationName:"Rule" type:"structure" required:"true"` +} + +type RetentionRule struct { + // The open status of the recycle bin is not case-sensitive. + // Valid values: Enabled, Disabled. Enabled indicates enabling the recycle bin, Disabled indicates disabling the recycle bin. + Status *string `locationName:"Status" type:"string" required:"true"` + + // Specify how many days after the object enters the recycle bin to be completely deleted. + // When Days is not set, the object will be permanently retained in the recycle bin after deletion. + // Value range: 1-365 + Days *int64 `locationName:"Days" type:"integer"` +} + +type PutBucketRetentionOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// GetBucketRetentionRequest generates a request for the GetBucketRetention operation. +func (c *S3) GetBucketRetentionRequest(input *GetBucketRetentionInput) (req *aws.Request, output *GetBucketRetentionOutput) { + oprw.Lock() + defer oprw.Unlock() + if opGetBucketRetention == nil { + opGetBucketRetention = &aws.Operation{ + Name: "GetBucketRetention", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?retention", + } + } + if input == nil { + input = &GetBucketRetentionInput{} + } + req = c.newRequest(opGetBucketRetention, input, output) + output = &GetBucketRetentionOutput{ + RetentionConfiguration: &BucketRetentionConfiguration{}, + } + req.Data = output + return +} + +// GetBucketRetention gets the retention configuration for the bucket. +func (c *S3) GetBucketRetention(input *GetBucketRetentionInput) (*GetBucketRetentionOutput, error) { + req, out := c.GetBucketRetentionRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketRetentionWithContext(ctx aws.Context, input *GetBucketRetentionInput) (*GetBucketRetentionOutput, error) { + req, out := c.GetBucketRetentionRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +var opGetBucketRetention *aws.Operation + +type GetBucketRetentionInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` +} + +type GetBucketRetentionOutput struct { + RetentionConfiguration *BucketRetentionConfiguration `locationName:"RetentionConfiguration" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketRetentionInput `json:"-" xml:"-"` +} + +type metadataGetBucketRetentionInput struct { + SDKShapeTraits bool `type:"structure" payload:"RetentionConfiguration"` +} + +// ListRetentionRequest generates a request for the ListRetention operation. +func (c *S3) ListRetentionRequest(input *ListRetentionInput) (req *aws.Request, output *ListRetentionOutput) { + oprw.Lock() + defer oprw.Unlock() + + if opListRetention == nil { + opListRetention = &aws.Operation{ + Name: "ListRetention", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?recycle", + } + } + + if input == nil { + input = &ListRetentionInput{} + } + + req = c.newRequest(opListRetention, input, output) + output = &ListRetentionOutput{} + req.Data = output + return +} + +// ListRetention lists the objects in the recycle bin. +func (c *S3) ListRetention(input *ListRetentionInput) (*ListRetentionOutput, error) { + req, out := c.ListRetentionRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) ListRetentionWithContext(ctx aws.Context, input *ListRetentionInput) (*ListRetentionOutput, error) { + req, out := c.ListRetentionRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +var opListRetention *aws.Operation + +type ListRetentionInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Specifies the key to start with when listing objects in a bucket. + Marker *string `location:"querystring" locationName:"marker" type:"string"` + + // Sets the maximum number of keys returned in the response. The response might + // contain fewer keys but will never contain more. + MaxKeys *int64 `location:"querystring" locationName:"max-keys" type:"integer"` + + // Limits the response to keys that begin with the specified prefix. + Prefix *string `location:"querystring" locationName:"prefix" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + metadataListRetentionInput `json:"-" xml:"-"` +} + +type metadataListRetentionInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type ListRetentionOutput struct { + // A container that lists information about the list of objects in the recycle bin. + ListRetentionResult *ListRetentionResult `locationName:"ListRetentionResult" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataListRetentionOutput `json:"-" xml:"-"` +} + +type metadataListRetentionOutput struct { + SDKShapeTraits bool `type:"structure" payload:"ListRetentionResult"` +} + +type ListRetentionResult struct { + // The name of the bucket. + Name *string `type:"string"` + + // Specify the prefix of the Key when requesting this List. + Prefix *string `type:"string"` + + // The maximum number of objects returned is 1000 by default. + MaxKeys *int64 `type:"integer"` + + // Specify the starting position of the object in the target bucket. + Marker *string `type:"string"` + + // The starting point for the next listed file. Users can use this value as a marker parameter + // for the next List Retention. + NextMarker *string `type:"string"` + + // Whether it has been truncated. If the number of records in the Object list exceeds the set + // maximum value, it will be truncated. + IsTruncated *bool `type:"boolean"` + + // The encoding method for Object names. + EncodingType *string `type:"string"` + + // List of Objects Listed. + Contents []*RetentionObject `type:"list" flattened:"true"` +} + +type RetentionObject struct { + // The key of the object. + Key *string `type:"string"` + + // The size of the object is measured in bytes. + Size *int64 `type:"integer"` + + // The entity label of an object, ETag, is generated when uploading an object to identify its content. + ETag *string `type:"string"` + + // The last time the object was modified. + LastModified *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + // The owner information of this bucket. + Owner *Owner `type:"structure"` + + // The class of storage used to store the object. + StorageClass *string `type:"string"` + + // The version ID of the object. + RetentionId *string `type:"string"` + + // The time when the object was moved to the recycle bin. + RecycleTime *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + // The time when an object is completely deleted from the recycle bin. + EstimatedClearTime *time.Time `type:"timestamp" timestampFormat:"iso8601"` +} + +// RecoverObjectRequest generates a request for the RecoverObject operation.` +func (c *S3) RecoverObjectRequest(input *ListRetentionInput) (req *aws.Request, output *ListRetentionOutput) { + oprw.Lock() + defer oprw.Unlock() + + if opListRetention == nil { + opListRetention = &aws.Operation{ + Name: "ListRetention", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}/{Key+}?recover", + } + } + + if input == nil { + input = &ListRetentionInput{} + } + + req = c.newRequest(opListRetention, input, output) + output = &ListRetentionOutput{} + req.Data = output + return +} + +// RecoverObject recovers the object from the recycle bin. +func (c *S3) RecoverObject(input *ListRetentionInput) (*ListRetentionOutput, error) { + req, out := c.ListRetentionRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) RecoverObjectWithContext(ctx aws.Context, input *ListRetentionInput) (*ListRetentionOutput, error) { + req, out := c.ListRetentionRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +var opRecoverObject *aws.Operation + +type RecoverObjectInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // The key of the object. + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` +} diff --git a/test/bucketsample_test.go b/test/bucketsample_test.go index e21960c..818a8b5 100644 --- a/test/bucketsample_test.go +++ b/test/bucketsample_test.go @@ -1,7 +1,9 @@ package lib import ( + "fmt" "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/aws/awsutil" "github.com/ks3sdklib/aws-sdk-go/service/s3" . "gopkg.in/check.v1" ) @@ -327,3 +329,66 @@ func (s *Ks3utilCommandSuite) TestBucketDecompressPolicy(c *C) { }) c.Assert(err, IsNil) } + +// TestBucketRetention bucket retention +func (s *Ks3utilCommandSuite) TestBucketRetention(c *C) { + _, err := client.PutBucketRetention(&s3.PutBucketRetentionInput{ + Bucket: aws.String(bucket), + RetentionConfiguration: &s3.BucketRetentionConfiguration{ + Rule: &s3.RetentionRule{ + Status: aws.String("Enabled"), + Days: aws.Long(30), + }, + }, + }) + c.Assert(err, IsNil) + + resp, err := client.GetBucketRetention(&s3.GetBucketRetentionInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) + c.Assert(*resp.RetentionConfiguration.Rule.Status, Equals, "Enabled") + c.Assert(*resp.RetentionConfiguration.Rule.Days, Equals, int64(30)) + + _, err = client.ListRetention(&s3.ListRetentionInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) +} + +// TestBucketReplication bucket replication +func (s *Ks3utilCommandSuite) TestBucketReplication(c *C) { + _, err := client.PutBucketReplication(&s3.PutBucketReplicationInput{ + Bucket: aws.String(bucket), + ReplicationConfiguration: &s3.ReplicationConfiguration{ + Prefix: []*string{aws.String("test/")}, + DeleteMarkerStatus: aws.String("Disabled"), + TargetBucket: aws.String(bucket), + HistoricalObjectReplication: aws.String("Enabled"), + }, + }) + c.Assert(err, IsNil) + + resp, err := client.GetBucketReplication(&s3.GetBucketReplicationInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) + c.Assert(len(resp.ReplicationConfiguration.Prefix), Equals, 1) + c.Assert(*resp.ReplicationConfiguration.Prefix[0], Equals, "test/") + c.Assert(*resp.ReplicationConfiguration.DeleteMarkerStatus, Equals, "Disabled") + c.Assert(*resp.ReplicationConfiguration.TargetBucket, Equals, bucket) + c.Assert(*resp.ReplicationConfiguration.HistoricalObjectReplication, Equals, "Enabled") + + _, err = client.DeleteBucketReplication(&s3.DeleteBucketReplicationInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) +} + +func (s *Ks3utilCommandSuite) TestListRetention(c *C) { + resp, err := client.ListRetention(&s3.ListRetentionInput{ + Bucket: aws.String("likui-test2"), + }) + c.Assert(err, IsNil) + fmt.Println(awsutil.StringValue(resp.ListRetentionResult)) +} diff --git a/test/bucketwithcontext_test.go b/test/bucketwithcontext_test.go index d06602a..9c36937 100644 --- a/test/bucketwithcontext_test.go +++ b/test/bucketwithcontext_test.go @@ -910,6 +910,133 @@ func (s *Ks3utilCommandSuite) TestDeleteBucketDecompressPolicyWithContext(c *C) c.Assert(err, IsNil) } +// PUT Bucket Replication +func (s *Ks3utilCommandSuite) TestPutBucketReplicationWithContext(c *C) { + // put,不通过context取消 + _, err := client.PutBucketReplicationWithContext(context.Background(), &s3.PutBucketReplicationInput{ + Bucket: aws.String(bucket), + ReplicationConfiguration: &s3.ReplicationConfiguration{ + Prefix: []*string{aws.String("test/")}, + DeleteMarkerStatus: aws.String("Disabled"), + TargetBucket: aws.String(bucket), + HistoricalObjectReplication: aws.String("Enabled"), + }, + }) + c.Assert(err, IsNil) + // get + resp, err := client.GetBucketReplicationWithContext(context.Background(), &s3.GetBucketReplicationInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) + c.Assert(len(resp.ReplicationConfiguration.Prefix), Equals, 1) + c.Assert(*resp.ReplicationConfiguration.Prefix[0], Equals, "test/") + c.Assert(*resp.ReplicationConfiguration.DeleteMarkerStatus, Equals, "Disabled") + c.Assert(*resp.ReplicationConfiguration.TargetBucket, Equals, bucket) + c.Assert(*resp.ReplicationConfiguration.HistoricalObjectReplication, Equals, "Enabled") + // put,通过context取消 + ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) + defer cancelFunc() + _, err = client.PutBucketReplicationWithContext(ctx, &s3.PutBucketReplicationInput{ + Bucket: aws.String(bucket), + ReplicationConfiguration: &s3.ReplicationConfiguration{ + Prefix: []*string{aws.String("test2/")}, + DeleteMarkerStatus: aws.String("Enabled"), + TargetBucket: aws.String(bucket), + HistoricalObjectReplication: aws.String("Disabled"), + }, + }) + c.Assert(err, NotNil) + // get + resp, err = client.GetBucketReplicationWithContext(context.Background(), &s3.GetBucketReplicationInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) + c.Assert(len(resp.ReplicationConfiguration.Prefix), Equals, 1) + c.Assert(*resp.ReplicationConfiguration.Prefix[0], Equals, "test/") + c.Assert(*resp.ReplicationConfiguration.DeleteMarkerStatus, Equals, "Disabled") + c.Assert(*resp.ReplicationConfiguration.TargetBucket, Equals, bucket) + c.Assert(*resp.ReplicationConfiguration.HistoricalObjectReplication, Equals, "Enabled") + // delete + _, err = client.DeleteBucketReplicationWithContext(context.Background(), &s3.DeleteBucketReplicationInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) +} + +// GET Bucket Replication +func (s *Ks3utilCommandSuite) TestGetBucketReplicationWithContext(c *C) { + // put + _, err := client.PutBucketReplicationWithContext(context.Background(), &s3.PutBucketReplicationInput{ + Bucket: aws.String(bucket), + ReplicationConfiguration: &s3.ReplicationConfiguration{ + Prefix: []*string{aws.String("test/")}, + DeleteMarkerStatus: aws.String("Disabled"), + TargetBucket: aws.String(bucket), + HistoricalObjectReplication: aws.String("Enabled"), + }, + }) + c.Assert(err, IsNil) + // get,不通过context取消 + resp, err := client.GetBucketReplicationWithContext(context.Background(), &s3.GetBucketReplicationInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) + c.Assert(len(resp.ReplicationConfiguration.Prefix), Equals, 1) + c.Assert(*resp.ReplicationConfiguration.Prefix[0], Equals, "test/") + c.Assert(*resp.ReplicationConfiguration.DeleteMarkerStatus, Equals, "Disabled") + c.Assert(*resp.ReplicationConfiguration.TargetBucket, Equals, bucket) + c.Assert(*resp.ReplicationConfiguration.HistoricalObjectReplication, Equals, "Enabled") + // get,通过context取消 + ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) + defer cancelFunc() + resp, err = client.GetBucketReplicationWithContext(ctx, &s3.GetBucketReplicationInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, NotNil) + // delete + _, err = client.DeleteBucketReplicationWithContext(context.Background(), &s3.DeleteBucketReplicationInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) +} + +// DELETE Bucket Replication +func (s *Ks3utilCommandSuite) TestDeleteBucketReplicationWithContext(c *C) { + // put + _, err := client.PutBucketReplicationWithContext(context.Background(), &s3.PutBucketReplicationInput{ + Bucket: aws.String(bucket), + ReplicationConfiguration: &s3.ReplicationConfiguration{ + Prefix: []*string{aws.String("test/")}, + DeleteMarkerStatus: aws.String("Disabled"), + TargetBucket: aws.String(bucket), + HistoricalObjectReplication: aws.String("Enabled"), + }, + }) + c.Assert(err, IsNil) + // get + resp, err := client.GetBucketReplicationWithContext(context.Background(), &s3.GetBucketReplicationInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) + c.Assert(len(resp.ReplicationConfiguration.Prefix), Equals, 1) + c.Assert(*resp.ReplicationConfiguration.Prefix[0], Equals, "test/") + c.Assert(*resp.ReplicationConfiguration.DeleteMarkerStatus, Equals, "Disabled") + c.Assert(*resp.ReplicationConfiguration.TargetBucket, Equals, bucket) + c.Assert(*resp.ReplicationConfiguration.HistoricalObjectReplication, Equals, "Enabled") + // delete,通过context取消 + ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) + defer cancelFunc() + _, err = client.DeleteBucketReplicationWithContext(ctx, &s3.DeleteBucketReplicationInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, NotNil) + // delete,不通过context取消 + _, err = client.DeleteBucketReplicationWithContext(context.Background(), &s3.DeleteBucketReplicationInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) +} + // PUT Bucket ACL func (s *Ks3utilCommandSuite) TestPutBucketACLWithContext(c *C) { // put,不通过context取消 From aecd575b3d028ca3fdea6d6fbebf919898164eca Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Wed, 9 Oct 2024 19:23:35 +0800 Subject: [PATCH 05/13] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E5=9B=9E?= =?UTF-8?q?=E6=94=B6=E7=AB=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- service/s3/retention.go | 111 ++++++++++++++++++++++++++------ service/s3/s3iface/interface.go | 16 +++++ test/bucketsample_test.go | 10 +++ test/bucketwithcontext_test.go | 101 +++++++++++++++++++++++++++++ 4 files changed, 217 insertions(+), 21 deletions(-) diff --git a/service/s3/retention.go b/service/s3/retention.go index 2399f3e..490d980 100644 --- a/service/s3/retention.go +++ b/service/s3/retention.go @@ -9,6 +9,7 @@ import ( func (c *S3) PutBucketRetentionRequest(input *PutBucketRetentionInput) (req *aws.Request, output *PutBucketRetentionOutput) { oprw.Lock() defer oprw.Unlock() + if opPutBucketRetention == nil { opPutBucketRetention = &aws.Operation{ Name: "PutBucketRetention", @@ -16,9 +17,11 @@ func (c *S3) PutBucketRetentionRequest(input *PutBucketRetentionInput) (req *aws HTTPPath: "/{Bucket}?retention", } } + if input == nil { input = &PutBucketRetentionInput{} } + input.AutoFillMD5 = true req = c.newRequest(opPutBucketRetention, input, output) output = &PutBucketRetentionOutput{} @@ -84,6 +87,7 @@ type PutBucketRetentionOutput struct { func (c *S3) GetBucketRetentionRequest(input *GetBucketRetentionInput) (req *aws.Request, output *GetBucketRetentionOutput) { oprw.Lock() defer oprw.Unlock() + if opGetBucketRetention == nil { opGetBucketRetention = &aws.Operation{ Name: "GetBucketRetention", @@ -91,9 +95,11 @@ func (c *S3) GetBucketRetentionRequest(input *GetBucketRetentionInput) (req *aws HTTPPath: "/{Bucket}?retention", } } + if input == nil { input = &GetBucketRetentionInput{} } + req = c.newRequest(opGetBucketRetention, input, output) output = &GetBucketRetentionOutput{ RetentionConfiguration: &BucketRetentionConfiguration{}, @@ -188,14 +194,6 @@ type ListRetentionInput struct { // Limits the response to keys that begin with the specified prefix. Prefix *string `location:"querystring" locationName:"prefix" type:"string"` - - ContentType *string `location:"header" locationName:"Content-Type" type:"string"` - - metadataListRetentionInput `json:"-" xml:"-"` -} - -type metadataListRetentionInput struct { - SDKShapeTraits bool `type:"structure"` } type ListRetentionOutput struct { @@ -270,38 +268,38 @@ type RetentionObject struct { EstimatedClearTime *time.Time `type:"timestamp" timestampFormat:"iso8601"` } -// RecoverObjectRequest generates a request for the RecoverObject operation.` -func (c *S3) RecoverObjectRequest(input *ListRetentionInput) (req *aws.Request, output *ListRetentionOutput) { +// RecoverObjectRequest generates a request for the RecoverObject operation. +func (c *S3) RecoverObjectRequest(input *RecoverObjectInput) (req *aws.Request, output *RecoverObjectOutput) { oprw.Lock() defer oprw.Unlock() - if opListRetention == nil { - opListRetention = &aws.Operation{ - Name: "ListRetention", - HTTPMethod: "GET", + if opRecoverObject == nil { + opRecoverObject = &aws.Operation{ + Name: "RecoverObject", + HTTPMethod: "POST", HTTPPath: "/{Bucket}/{Key+}?recover", } } if input == nil { - input = &ListRetentionInput{} + input = &RecoverObjectInput{} } - req = c.newRequest(opListRetention, input, output) - output = &ListRetentionOutput{} + req = c.newRequest(opRecoverObject, input, output) + output = &RecoverObjectOutput{} req.Data = output return } // RecoverObject recovers the object from the recycle bin. -func (c *S3) RecoverObject(input *ListRetentionInput) (*ListRetentionOutput, error) { - req, out := c.ListRetentionRequest(input) +func (c *S3) RecoverObject(input *RecoverObjectInput) (*RecoverObjectOutput, error) { + req, out := c.RecoverObjectRequest(input) err := req.Send() return out, err } -func (c *S3) RecoverObjectWithContext(ctx aws.Context, input *ListRetentionInput) (*ListRetentionOutput, error) { - req, out := c.ListRetentionRequest(input) +func (c *S3) RecoverObjectWithContext(ctx aws.Context, input *RecoverObjectInput) (*RecoverObjectOutput, error) { + req, out := c.RecoverObjectRequest(input) req.SetContext(ctx) err := req.Send() return out, err @@ -315,4 +313,75 @@ type RecoverObjectInput struct { // The key of the object. Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Does it support overwriting when an object with the same name exists in the bucket after being + // recovered from the recycle bin. When the value is true, it indicates overwriting, and the overwritten + // objects in the bucket will enter the recycle bin. + RetentionOverwrite *bool `location:"header" locationName:"x-kss-retention-overwrite" type:"boolean"` + + // Specify the deletion ID of the recovered object. When the request header is not included, + // only the latest version is restored by default. + RetentionId *string `location:"header" locationName:"x-kss-retention-id" type:"string"` +} + +type RecoverObjectOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// ClearObjectRequest generates a request for the ClearObject operation. +func (c *S3) ClearObjectRequest(input *ClearObjectInput) (req *aws.Request, output *ClearObjectOutput) { + oprw.Lock() + defer oprw.Unlock() + + if opClearObject == nil { + opClearObject = &aws.Operation{ + Name: "ClearObject", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}/{Key+}?clear", + } + } + + if input == nil { + input = &ClearObjectInput{} + } + + req = c.newRequest(opClearObject, input, output) + output = &ClearObjectOutput{} + req.Data = output + return +} + +// ClearObject clears the object from the recycle bin. +func (c *S3) ClearObject(input *ClearObjectInput) (*ClearObjectOutput, error) { + req, out := c.ClearObjectRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) ClearObjectWithContext(ctx aws.Context, input *ClearObjectInput) (*ClearObjectOutput, error) { + req, out := c.ClearObjectRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +var opClearObject *aws.Operation + +type ClearObjectInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // The key of the object. + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Specify the deletion ID of the deleted object. + RetentionId *string `location:"header" locationName:"x-kss-retention-id" type:"string" required:"true"` +} + +type ClearObjectOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` } diff --git a/service/s3/s3iface/interface.go b/service/s3/s3iface/interface.go index d430bd1..8903b63 100755 --- a/service/s3/s3iface/interface.go +++ b/service/s3/s3iface/interface.go @@ -21,6 +21,8 @@ type S3API interface { CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) + ClearObject(*s3.ClearObjectInput) (*s3.ClearObjectOutput, error) + DeleteBucket(*s3.DeleteBucketInput) (*s3.DeleteBucketOutput, error) DeleteBucketCORS(*s3.DeleteBucketCORSInput) (*s3.DeleteBucketCORSOutput, error) @@ -35,6 +37,8 @@ type S3API interface { DeleteBucketWebsite(*s3.DeleteBucketWebsiteInput) (*s3.DeleteBucketWebsiteOutput, error) + DeleteBucketDecompressPolicy(*s3.DeleteBucketDecompressPolicyInput) (*s3.DeleteBucketDecompressPolicyOutput, error) + DeleteObject(*s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) DeleteObjects(*s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) @@ -65,6 +69,10 @@ type S3API interface { GetBucketWebsite(*s3.GetBucketWebsiteInput) (*s3.GetBucketWebsiteOutput, error) + GetBucketDecompressPolicy(*s3.GetBucketDecompressPolicyInput) (*s3.GetBucketDecompressPolicyOutput, error) + + GetBucketRetention(*s3.GetBucketRetentionInput) (*s3.GetBucketRetentionOutput, error) + GetObject(*s3.GetObjectInput) (*s3.GetObjectOutput, error) GetObjectACL(*s3.GetObjectACLInput) (*s3.GetObjectACLOutput, error) @@ -85,6 +93,8 @@ type S3API interface { ListParts(*s3.ListPartsInput) (*s3.ListPartsOutput, error) + ListRetention(*s3.ListRetentionInput) (*s3.ListRetentionOutput, error) + PutBucketACL(*s3.PutBucketACLInput) (*s3.PutBucketACLOutput, error) PutBucketCORS(*s3.PutBucketCORSInput) (*s3.PutBucketCORSOutput, error) @@ -109,12 +119,18 @@ type S3API interface { PutBucketWebsite(*s3.PutBucketWebsiteInput) (*s3.PutBucketWebsiteOutput, error) + PutBucketDecompressPolicy(*s3.PutBucketDecompressPolicyInput) (*s3.PutBucketDecompressPolicyOutput, error) + + PutBucketRetention(*s3.PutBucketRetentionInput) (*s3.PutBucketRetentionOutput, error) + PutObject(*s3.PutObjectInput) (*s3.PutObjectOutput, error) PutObjectACL(*s3.PutObjectACLInput) (*s3.PutObjectACLOutput, error) RestoreObject(*s3.RestoreObjectInput) (*s3.RestoreObjectOutput, error) + RecoverObject(*s3.RecoverObjectInput) (*s3.RecoverObjectOutput, error) + UploadPart(*s3.UploadPartInput) (*s3.UploadPartOutput, error) UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) diff --git a/test/bucketsample_test.go b/test/bucketsample_test.go index 818a8b5..c5423bf 100644 --- a/test/bucketsample_test.go +++ b/test/bucketsample_test.go @@ -392,3 +392,13 @@ func (s *Ks3utilCommandSuite) TestListRetention(c *C) { c.Assert(err, IsNil) fmt.Println(awsutil.StringValue(resp.ListRetentionResult)) } + +func (s *Ks3utilCommandSuite) TestClearObject(c *C) { + resp, err := client.ClearObject(&s3.ClearObjectInput{ + Bucket: aws.String("likui-test2"), + Key: aws.String("upgrade.json"), + RetentionId: aws.String("29807535817133_MA=="), + }) + c.Assert(err, IsNil) + fmt.Println(awsutil.StringValue(resp)) +} diff --git a/test/bucketwithcontext_test.go b/test/bucketwithcontext_test.go index 9c36937..a8e207b 100644 --- a/test/bucketwithcontext_test.go +++ b/test/bucketwithcontext_test.go @@ -1037,6 +1037,107 @@ func (s *Ks3utilCommandSuite) TestDeleteBucketReplicationWithContext(c *C) { c.Assert(err, IsNil) } +// PUT Bucket Retention +func (s *Ks3utilCommandSuite) TestPutBucketRetentionWithContext(c *C) { + // put,不通过context取消 + _, err := client.PutBucketRetentionWithContext(context.Background(), &s3.PutBucketRetentionInput{ + Bucket: aws.String(bucket), + RetentionConfiguration: &s3.BucketRetentionConfiguration{ + Rule: &s3.RetentionRule{ + Status: aws.String("Enabled"), + Days: aws.Long(30), + }, + }, + }) + c.Assert(err, IsNil) + // get + resp, err := client.GetBucketRetentionWithContext(context.Background(), &s3.GetBucketRetentionInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) + c.Assert(*resp.RetentionConfiguration.Rule.Status, Equals, "Enabled") + c.Assert(*resp.RetentionConfiguration.Rule.Days, Equals, int64(30)) + // put,通过context取消 + ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) + defer cancelFunc() + _, err = client.PutBucketRetentionWithContext(ctx, &s3.PutBucketRetentionInput{ + Bucket: aws.String(bucket), + RetentionConfiguration: &s3.BucketRetentionConfiguration{ + Rule: &s3.RetentionRule{ + Status: aws.String("Enabled"), + Days: aws.Long(60), + }, + }, + }) + c.Assert(err, NotNil) + // get + resp, err = client.GetBucketRetentionWithContext(context.Background(), &s3.GetBucketRetentionInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) + c.Assert(*resp.RetentionConfiguration.Rule.Status, Equals, "Enabled") + c.Assert(*resp.RetentionConfiguration.Rule.Days, Equals, int64(30)) +} + +// GET Bucket Retention +func (s *Ks3utilCommandSuite) TestGetBucketRetentionWithContext(c *C) { + // put + _, err := client.PutBucketRetentionWithContext(context.Background(), &s3.PutBucketRetentionInput{ + Bucket: aws.String(bucket), + RetentionConfiguration: &s3.BucketRetentionConfiguration{ + Rule: &s3.RetentionRule{ + Status: aws.String("Enabled"), + Days: aws.Long(30), + }, + }, + }) + c.Assert(err, IsNil) + // get,不通过context取消 + resp, err := client.GetBucketRetentionWithContext(context.Background(), &s3.GetBucketRetentionInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) + c.Assert(*resp.RetentionConfiguration.Rule.Status, Equals, "Enabled") + c.Assert(*resp.RetentionConfiguration.Rule.Days, Equals, int64(30)) + // get,通过context取消 + ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) + defer cancelFunc() + resp, err = client.GetBucketRetentionWithContext(ctx, &s3.GetBucketRetentionInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, NotNil) +} + +// List Retention +func (s *Ks3utilCommandSuite) TestListBucketRetentionWithContext(c *C) { + // put + _, err := client.PutBucketRetentionWithContext(context.Background(), &s3.PutBucketRetentionInput{ + Bucket: aws.String(bucket), + RetentionConfiguration: &s3.BucketRetentionConfiguration{ + Rule: &s3.RetentionRule{ + Status: aws.String("Enabled"), + Days: aws.Long(30), + }, + }, + }) + c.Assert(err, IsNil) + // list,不通过context取消 + resp, err := client.ListRetentionWithContext(context.Background(), &s3.ListRetentionInput{ + Bucket: aws.String(bucket), + Prefix: aws.String("test/"), + }) + c.Assert(err, IsNil) + c.Assert(*resp.ListRetentionResult.Prefix, Equals, "test/") + // list,通过context取消 + ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) + defer cancelFunc() + resp, err = client.ListRetentionWithContext(ctx, &s3.ListRetentionInput{ + Bucket: aws.String(bucket), + Prefix: aws.String("test/"), + }) + c.Assert(err, NotNil) +} + // PUT Bucket ACL func (s *Ks3utilCommandSuite) TestPutBucketACLWithContext(c *C) { // put,不通过context取消 From b3e588676d609f60da4fb5ce23ddc957435e0001 Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Wed, 9 Oct 2024 19:26:14 +0800 Subject: [PATCH 06/13] =?UTF-8?q?test:=20=E5=8D=95=E5=85=83=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/bucketsample_test.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/test/bucketsample_test.go b/test/bucketsample_test.go index c5423bf..81f47fe 100644 --- a/test/bucketsample_test.go +++ b/test/bucketsample_test.go @@ -1,9 +1,7 @@ package lib import ( - "fmt" "github.com/ks3sdklib/aws-sdk-go/aws" - "github.com/ks3sdklib/aws-sdk-go/aws/awsutil" "github.com/ks3sdklib/aws-sdk-go/service/s3" . "gopkg.in/check.v1" ) @@ -384,21 +382,3 @@ func (s *Ks3utilCommandSuite) TestBucketReplication(c *C) { }) c.Assert(err, IsNil) } - -func (s *Ks3utilCommandSuite) TestListRetention(c *C) { - resp, err := client.ListRetention(&s3.ListRetentionInput{ - Bucket: aws.String("likui-test2"), - }) - c.Assert(err, IsNil) - fmt.Println(awsutil.StringValue(resp.ListRetentionResult)) -} - -func (s *Ks3utilCommandSuite) TestClearObject(c *C) { - resp, err := client.ClearObject(&s3.ClearObjectInput{ - Bucket: aws.String("likui-test2"), - Key: aws.String("upgrade.json"), - RetentionId: aws.String("29807535817133_MA=="), - }) - c.Assert(err, IsNil) - fmt.Println(awsutil.StringValue(resp)) -} From 35261945d08a2bb9e6cde0eec3e3ea89753a84fe Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Sat, 12 Oct 2024 11:23:43 +0800 Subject: [PATCH 07/13] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E6=B8=85?= =?UTF-8?q?=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/signer/v2/v2.go | 2 + service/s3/api.go | 12 -- service/s3/inventory.go | 309 ++++++++++++++++++++++++++++++++ service/s3/s3iface/interface.go | 8 + test/bucketsample_test.go | 62 +++++++ test/bucketwithcontext_test.go | 282 +++++++++++++++++++++++++++++ 6 files changed, 663 insertions(+), 12 deletions(-) diff --git a/internal/signer/v2/v2.go b/internal/signer/v2/v2.go index 822dc9d..56daafd 100644 --- a/internal/signer/v2/v2.go +++ b/internal/signer/v2/v2.go @@ -65,6 +65,8 @@ var signQuerys = map[string]bool{ "recycle": true, "recover": true, "clear": true, + "id": true, + "continuation-token": true, } type signer struct { diff --git a/service/s3/api.go b/service/s3/api.go index 4c043db..e71d7c2 100755 --- a/service/s3/api.go +++ b/service/s3/api.go @@ -3083,18 +3083,6 @@ type metadataDeletedObject struct { SDKShapeTraits bool `type:"structure"` } -type Destination struct { - // Amazon resource name (ARN) of the bucket where you want Amazon S3 to store - // replicas of the object identified by the rule. - Bucket *string `type:"string" required:"true"` - - metadataDestination `json:"-" xml:"-"` -} - -type metadataDestination struct { - SDKShapeTraits bool `type:"structure"` -} - type Error struct { Code *string `type:"string"` diff --git a/service/s3/inventory.go b/service/s3/inventory.go index 3ed7f97..24e695f 100644 --- a/service/s3/inventory.go +++ b/service/s3/inventory.go @@ -1 +1,310 @@ package s3 + +import "github.com/ks3sdklib/aws-sdk-go/aws" + +// PutBucketInventoryRequest generates a request for the PutBucketInventory operation. +func (c *S3) PutBucketInventoryRequest(input *PutBucketInventoryInput) (req *aws.Request, output *PutBucketInventoryOutput) { + oprw.Lock() + defer oprw.Unlock() + + if opPutBucketInventory == nil { + opPutBucketInventory = &aws.Operation{ + Name: "PutBucketInventory", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?inventory", + } + } + + if input == nil { + input = &PutBucketInventoryInput{} + } + + input.AutoFillMD5 = true + req = c.newRequest(opPutBucketInventory, input, output) + output = &PutBucketInventoryOutput{} + req.Data = output + return +} + +// PutBucketInventory creates a new inventory configuration. +func (c *S3) PutBucketInventory(input *PutBucketInventoryInput) (*PutBucketInventoryOutput, error) { + req, out := c.PutBucketInventoryRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketInventoryWithContext(ctx aws.Context, input *PutBucketInventoryInput) (*PutBucketInventoryOutput, error) { + req, out := c.PutBucketInventoryRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +var opPutBucketInventory *aws.Operation + +type PutBucketInventoryInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Id *string `location:"querystring" locationName:"id" type:"string" required:"true"` + + InventoryConfiguration *InventoryConfiguration `locationName:"InventoryConfiguration" type:"structure" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + metadataPutBucketInventoryInput `json:"-" xml:"-"` +} + +type metadataPutBucketInventoryInput struct { + SDKShapeTraits bool `type:"structure" payload:"InventoryConfiguration"` + + AutoFillMD5 bool +} + +type PutBucketInventoryOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type InventoryConfiguration struct { + // The list name specified by the user is unique within a single bucket. + Id *string `locationName:"Id" type:"string" required:"true"` + + // Is the inventory function enabled. + IsEnabled *bool `locationName:"IsEnabled" type:"boolean" required:"true"` + + // Specify scanning prefix information. + Filter *InventoryFilter `locationName:"Filter" type:"structure"` + + // Storage inventory results. + Destination *Destination `locationName:"Destination" type:"structure" required:"true"` + + // Container for storing inventory export cycle information. + Schedule *Schedule `locationName:"Schedule" type:"structure" required:"true"` + + // Set the configuration items included in the inventory results. + OptionalFields *OptionalFields `locationName:"OptionalFields" type:"structure" required:"true"` +} + +type InventoryFilter struct { + // The storage path prefix of the inventory file. + Prefix *string `locationName:"Prefix" type:"string" required:"true"` + + // The starting timestamp of the last modification time of the filtered file, in seconds. + LastModifyBeginTimeStamp *string `locationName:"LastModifyBeginTimeStamp" type:"string"` + + // End timestamp of the last modification time of the filtered file, in seconds. + LastModifyEndTimeStamp *string `locationName:"LastModifyEndTimeStamp" type:"string"` +} + +type Destination struct { + // Bucket information stored after exporting the inventory results. + KS3BucketDestination *KS3BucketDestination `locationName:"KS3BucketDestination" type:"structure" required:"true"` +} + +type KS3BucketDestination struct { + // The file format of the inventory file is a CSV file compressed using GZIP after exporting the manifest file. + Format *string `locationName:"Format" type:"string" required:"true"` + + // Bucket owner's account ID. + AccountId *string `locationName:"AccountId" type:"string"` + + // Bucket for storing exported inventory files. + Bucket *string `locationName:"Bucket" type:"string" required:"true"` + + // The storage path prefix of the inventory file. + Prefix *string `locationName:"Prefix" type:"string"` +} + +type Schedule struct { + // Cycle of exporting inventory files. + Frequency *string `locationName:"Frequency" type:"string" required:"true"` +} + +type OptionalFields struct { + // Configuration items included in the inventory results. + // Valid values: + // Size: The size of the object. + // LastModifiedDate: The last modified time of an object. + // ETag: The ETag value of an object, used to identify its contents. + // StorageClass: The storage type of Object. + // IsMultipartUploaded: Is it an object uploaded through shard upload method. + // EncryptionStatus: Whether the object is encrypted. If the object is encrypted, the value of this field is True; otherwise, it is False. + Field []*string `locationName:"Field" type:"list" flattened:"true"` +} + +// GetBucketInventoryRequest generates a request for the GetBucketInventory operation. +func (c *S3) GetBucketInventoryRequest(input *GetBucketInventoryInput) (req *aws.Request, output *GetBucketInventoryOutput) { + oprw.Lock() + defer oprw.Unlock() + + if opGetBucketInventory == nil { + opGetBucketInventory = &aws.Operation{ + Name: "GetBucketInventory", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?inventory", + } + } + + if input == nil { + input = &GetBucketInventoryInput{} + } + + req = c.newRequest(opGetBucketInventory, input, output) + output = &GetBucketInventoryOutput{} + req.Data = output + return +} + +// GetBucketInventory gets the inventory configuration for the bucket. +func (c *S3) GetBucketInventory(input *GetBucketInventoryInput) (*GetBucketInventoryOutput, error) { + req, out := c.GetBucketInventoryRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketInventoryWithContext(ctx aws.Context, input *GetBucketInventoryInput) (*GetBucketInventoryOutput, error) { + req, out := c.GetBucketInventoryRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +var opGetBucketInventory *aws.Operation + +type GetBucketInventoryInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Id *string `location:"querystring" locationName:"id" type:"string" required:"true"` +} + +type GetBucketInventoryOutput struct { + InventoryConfiguration *InventoryConfiguration `locationName:"Inventory" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketInventoryOutput `json:"-" xml:"-"` +} + +type metadataGetBucketInventoryOutput struct { + SDKShapeTraits bool `type:"structure" payload:"InventoryConfiguration"` +} + +// DeleteBucketInventoryRequest generates a request for the DeleteBucketInventory operation. +func (c *S3) DeleteBucketInventoryRequest(input *DeleteBucketInventoryInput) (req *aws.Request, output *DeleteBucketInventoryOutput) { + oprw.Lock() + defer oprw.Unlock() + + if opDeleteBucketInventory == nil { + opDeleteBucketInventory = &aws.Operation{ + Name: "DeleteBucketInventory", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?inventory", + } + } + + if input == nil { + input = &DeleteBucketInventoryInput{} + } + + req = c.newRequest(opDeleteBucketInventory, input, output) + output = &DeleteBucketInventoryOutput{} + req.Data = output + return +} + +// DeleteBucketInventory deletes the inventory configuration for the bucket. +func (c *S3) DeleteBucketInventory(input *DeleteBucketInventoryInput) (*DeleteBucketInventoryOutput, error) { + req, out := c.DeleteBucketInventoryRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketInventoryWithContext(ctx aws.Context, input *DeleteBucketInventoryInput) (*DeleteBucketInventoryOutput, error) { + req, out := c.DeleteBucketInventoryRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +var opDeleteBucketInventory *aws.Operation + +type DeleteBucketInventoryInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Id *string `location:"querystring" locationName:"id" type:"string" required:"true"` +} +type DeleteBucketInventoryOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// ListBucketInventoryRequest generates a request for the ListBucketInventory operation. +func (c *S3) ListBucketInventoryRequest(input *ListBucketInventoryInput) (req *aws.Request, output *ListBucketInventoryOutput) { + oprw.Lock() + defer oprw.Unlock() + + if opListBucketInventory == nil { + opListBucketInventory = &aws.Operation{ + Name: "ListBucketInventory", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?inventory", + } + } + + if input == nil { + input = &ListBucketInventoryInput{} + } + + req = c.newRequest(opListBucketInventory, input, output) + output = &ListBucketInventoryOutput{} + req.Data = output + return +} + +// ListBucketInventory lists the inventory configurations for the bucket. +func (c *S3) ListBucketInventory(input *ListBucketInventoryInput) (*ListBucketInventoryOutput, error) { + req, out := c.ListBucketInventoryRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) ListBucketInventoryWithContext(ctx aws.Context, input *ListBucketInventoryInput) (*ListBucketInventoryOutput, error) { + req, out := c.ListBucketInventoryRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +var opListBucketInventory *aws.Operation + +type ListBucketInventoryInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContinuationToken *string `location:"querystring" locationName:"continuation-token" type:"string"` +} + +type ListInventoryConfigurationsResult struct { + InventoryConfigurations []*InventoryConfiguration `locationName:"InventoryConfiguration" type:"list" flattened:"true"` + + IsTruncated *bool `locationName:"IsTruncated" type:"boolean"` + + NextContinuationToken *string `locationName:"NextContinuationToken" type:"string"` +} + +type ListBucketInventoryOutput struct { + InventoryConfigurationsResult *ListInventoryConfigurationsResult `locationName:"InventoryConfigurationsResult" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataListBucketInventoryOutput `json:"-" xml:"-"` +} + +type metadataListBucketInventoryOutput struct { + SDKShapeTraits bool `type:"structure" payload:"InventoryConfigurationsResult"` +} diff --git a/service/s3/s3iface/interface.go b/service/s3/s3iface/interface.go index 8903b63..ab0247d 100755 --- a/service/s3/s3iface/interface.go +++ b/service/s3/s3iface/interface.go @@ -39,6 +39,8 @@ type S3API interface { DeleteBucketDecompressPolicy(*s3.DeleteBucketDecompressPolicyInput) (*s3.DeleteBucketDecompressPolicyOutput, error) + DeleteBucketInventory(*s3.DeleteBucketInventoryInput) (*s3.DeleteBucketInventoryOutput, error) + DeleteObject(*s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) DeleteObjects(*s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) @@ -73,6 +75,8 @@ type S3API interface { GetBucketRetention(*s3.GetBucketRetentionInput) (*s3.GetBucketRetentionOutput, error) + GetBucketInventory(*s3.GetBucketInventoryInput) (*s3.GetBucketInventoryOutput, error) + GetObject(*s3.GetObjectInput) (*s3.GetObjectOutput, error) GetObjectACL(*s3.GetObjectACLInput) (*s3.GetObjectACLOutput, error) @@ -95,6 +99,8 @@ type S3API interface { ListRetention(*s3.ListRetentionInput) (*s3.ListRetentionOutput, error) + ListBucketInventory(*s3.ListBucketInventoryInput) (*s3.ListBucketInventoryOutput, error) + PutBucketACL(*s3.PutBucketACLInput) (*s3.PutBucketACLOutput, error) PutBucketCORS(*s3.PutBucketCORSInput) (*s3.PutBucketCORSOutput, error) @@ -123,6 +129,8 @@ type S3API interface { PutBucketRetention(*s3.PutBucketRetentionInput) (*s3.PutBucketRetentionOutput, error) + PutBucketInventory(*s3.PutBucketInventoryInput) (*s3.PutBucketInventoryOutput, error) + PutObject(*s3.PutObjectInput) (*s3.PutObjectOutput, error) PutObjectACL(*s3.PutObjectACLInput) (*s3.PutObjectACLOutput, error) diff --git a/test/bucketsample_test.go b/test/bucketsample_test.go index 81f47fe..9a413a5 100644 --- a/test/bucketsample_test.go +++ b/test/bucketsample_test.go @@ -382,3 +382,65 @@ func (s *Ks3utilCommandSuite) TestBucketReplication(c *C) { }) c.Assert(err, IsNil) } + +func (s *Ks3utilCommandSuite) TestBucketInventory(c *C) { + id := randLowStr(8) + _, err := client.PutBucketInventory(&s3.PutBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + InventoryConfiguration: &s3.InventoryConfiguration{ + Id: aws.String(id), + IsEnabled: aws.Boolean(true), + Filter: &s3.InventoryFilter{ + Prefix: aws.String("abc/"), + }, + Destination: &s3.Destination{ + KS3BucketDestination: &s3.KS3BucketDestination{ + Format: aws.String("CSV"), + Bucket: aws.String(bucket), + Prefix: aws.String("prefix/"), + }, + }, + Schedule: &s3.Schedule{ + Frequency: aws.String("Once"), + }, + OptionalFields: &s3.OptionalFields{ + Field: []*string{ + aws.String("Size"), + aws.String("LastModifiedDate"), + aws.String("ETag"), + aws.String("StorageClass"), + aws.String("IsMultipartUploaded"), + aws.String("EncryptionStatus"), + }, + }, + }, + }) + c.Assert(err, IsNil) + + resp, err := client.GetBucketInventory(&s3.GetBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + }) + c.Assert(err, IsNil) + c.Assert(*resp.InventoryConfiguration.Id, Equals, id) + c.Assert(*resp.InventoryConfiguration.IsEnabled, Equals, true) + c.Assert(*resp.InventoryConfiguration.Filter.Prefix, Equals, "abc/") + c.Assert(*resp.InventoryConfiguration.Destination.KS3BucketDestination.Format, Equals, "CSV") + c.Assert(*resp.InventoryConfiguration.Destination.KS3BucketDestination.Bucket, Equals, bucket) + c.Assert(*resp.InventoryConfiguration.Destination.KS3BucketDestination.Prefix, Equals, "prefix/") + c.Assert(*resp.InventoryConfiguration.Schedule.Frequency, Equals, "Once") + c.Assert(len(resp.InventoryConfiguration.OptionalFields.Field), Equals, 6) + + listResp, err := client.ListBucketInventory(&s3.ListBucketInventoryInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) + c.Assert(len(listResp.InventoryConfigurationsResult.InventoryConfigurations), Equals, 1) + + _, err = client.DeleteBucketInventory(&s3.DeleteBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + }) + c.Assert(err, IsNil) +} diff --git a/test/bucketwithcontext_test.go b/test/bucketwithcontext_test.go index a8e207b..be6a29a 100644 --- a/test/bucketwithcontext_test.go +++ b/test/bucketwithcontext_test.go @@ -1138,6 +1138,288 @@ func (s *Ks3utilCommandSuite) TestListBucketRetentionWithContext(c *C) { c.Assert(err, NotNil) } +// PUT Bucket Inventory +func (s *Ks3utilCommandSuite) TestPutBucketInventoryWithContext(c *C) { + id := randLowStr(8) + // put,不通过context取消 + _, err := client.PutBucketInventoryWithContext(context.Background(), &s3.PutBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + InventoryConfiguration: &s3.InventoryConfiguration{ + Id: aws.String(id), + IsEnabled: aws.Boolean(true), + Filter: &s3.InventoryFilter{ + Prefix: aws.String("abc/"), + }, + Destination: &s3.Destination{ + KS3BucketDestination: &s3.KS3BucketDestination{ + Format: aws.String("CSV"), + Bucket: aws.String(bucket), + Prefix: aws.String("prefix/"), + }, + }, + Schedule: &s3.Schedule{ + Frequency: aws.String("Once"), + }, + OptionalFields: &s3.OptionalFields{ + Field: []*string{ + aws.String("Size"), + aws.String("LastModifiedDate"), + aws.String("ETag"), + aws.String("StorageClass"), + aws.String("IsMultipartUploaded"), + aws.String("EncryptionStatus"), + }, + }, + }, + }) + c.Assert(err, IsNil) + // get + resp, err := client.GetBucketInventoryWithContext(context.Background(), &s3.GetBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + }) + c.Assert(err, IsNil) + c.Assert(*resp.InventoryConfiguration.Id, Equals, id) + c.Assert(*resp.InventoryConfiguration.IsEnabled, Equals, true) + c.Assert(*resp.InventoryConfiguration.Filter.Prefix, Equals, "abc/") + c.Assert(*resp.InventoryConfiguration.Destination.KS3BucketDestination.Format, Equals, "CSV") + c.Assert(*resp.InventoryConfiguration.Destination.KS3BucketDestination.Bucket, Equals, bucket) + c.Assert(*resp.InventoryConfiguration.Destination.KS3BucketDestination.Prefix, Equals, "prefix/") + c.Assert(*resp.InventoryConfiguration.Schedule.Frequency, Equals, "Once") + c.Assert(len(resp.InventoryConfiguration.OptionalFields.Field), Equals, 6) + // put,通过context取消 + ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) + defer cancelFunc() + _, err = client.PutBucketInventoryWithContext(ctx, &s3.PutBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + InventoryConfiguration: &s3.InventoryConfiguration{ + Id: aws.String(id), + IsEnabled: aws.Boolean(true), + Filter: &s3.InventoryFilter{ + Prefix: aws.String("abc/"), + }, + Destination: &s3.Destination{ + KS3BucketDestination: &s3.KS3BucketDestination{ + Format: aws.String("CSV"), + Bucket: aws.String(bucket), + Prefix: aws.String("prefix/"), + }, + }, + Schedule: &s3.Schedule{ + Frequency: aws.String("Once"), + }, + OptionalFields: &s3.OptionalFields{ + Field: []*string{ + aws.String("Size"), + aws.String("LastModifiedDate"), + aws.String("ETag"), + aws.String("StorageClass"), + aws.String("IsMultipartUploaded"), + aws.String("EncryptionStatus"), + }, + }, + }, + }) + c.Assert(err, NotNil) + // delete + _, err = client.DeleteBucketInventoryWithContext(context.Background(), &s3.DeleteBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + }) + c.Assert(err, IsNil) +} + +// GET Bucket Inventory +func (s *Ks3utilCommandSuite) TestGetBucketInventoryWithContext(c *C) { + id := randLowStr(8) + // put,不通过context取消 + _, err := client.PutBucketInventoryWithContext(context.Background(), &s3.PutBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + InventoryConfiguration: &s3.InventoryConfiguration{ + Id: aws.String(id), + IsEnabled: aws.Boolean(true), + Filter: &s3.InventoryFilter{ + Prefix: aws.String("abc/"), + }, + Destination: &s3.Destination{ + KS3BucketDestination: &s3.KS3BucketDestination{ + Format: aws.String("CSV"), + Bucket: aws.String(bucket), + Prefix: aws.String("prefix/"), + }, + }, + Schedule: &s3.Schedule{ + Frequency: aws.String("Once"), + }, + OptionalFields: &s3.OptionalFields{ + Field: []*string{ + aws.String("Size"), + aws.String("LastModifiedDate"), + aws.String("ETag"), + aws.String("StorageClass"), + aws.String("IsMultipartUploaded"), + aws.String("EncryptionStatus"), + }, + }, + }, + }) + c.Assert(err, IsNil) + // get + resp, err := client.GetBucketInventoryWithContext(context.Background(), &s3.GetBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + }) + c.Assert(err, IsNil) + c.Assert(*resp.InventoryConfiguration.Id, Equals, id) + c.Assert(*resp.InventoryConfiguration.IsEnabled, Equals, true) + c.Assert(*resp.InventoryConfiguration.Filter.Prefix, Equals, "abc/") + c.Assert(*resp.InventoryConfiguration.Destination.KS3BucketDestination.Format, Equals, "CSV") + c.Assert(*resp.InventoryConfiguration.Destination.KS3BucketDestination.Bucket, Equals, bucket) + c.Assert(*resp.InventoryConfiguration.Destination.KS3BucketDestination.Prefix, Equals, "prefix/") + c.Assert(*resp.InventoryConfiguration.Schedule.Frequency, Equals, "Once") + c.Assert(len(resp.InventoryConfiguration.OptionalFields.Field), Equals, 6) + // put,通过context取消 + ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) + defer cancelFunc() + _, err = client.GetBucketInventoryWithContext(ctx, &s3.GetBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + }) + c.Assert(err, NotNil) + // delete + _, err = client.DeleteBucketInventoryWithContext(context.Background(), &s3.DeleteBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + }) + c.Assert(err, IsNil) +} + +// DELETE Bucket Inventory +func (s *Ks3utilCommandSuite) TestDeleteBucketInventoryWithContext(c *C) { + id := randLowStr(8) + // put,不通过context取消 + _, err := client.PutBucketInventoryWithContext(context.Background(), &s3.PutBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + InventoryConfiguration: &s3.InventoryConfiguration{ + Id: aws.String(id), + IsEnabled: aws.Boolean(true), + Filter: &s3.InventoryFilter{ + Prefix: aws.String("abc/"), + }, + Destination: &s3.Destination{ + KS3BucketDestination: &s3.KS3BucketDestination{ + Format: aws.String("CSV"), + Bucket: aws.String(bucket), + Prefix: aws.String("prefix/"), + }, + }, + Schedule: &s3.Schedule{ + Frequency: aws.String("Once"), + }, + OptionalFields: &s3.OptionalFields{ + Field: []*string{ + aws.String("Size"), + aws.String("LastModifiedDate"), + aws.String("ETag"), + aws.String("StorageClass"), + aws.String("IsMultipartUploaded"), + aws.String("EncryptionStatus"), + }, + }, + }, + }) + c.Assert(err, IsNil) + // get + resp, err := client.GetBucketInventoryWithContext(context.Background(), &s3.GetBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + }) + c.Assert(err, IsNil) + c.Assert(*resp.InventoryConfiguration.Id, Equals, id) + c.Assert(*resp.InventoryConfiguration.IsEnabled, Equals, true) + c.Assert(*resp.InventoryConfiguration.Filter.Prefix, Equals, "abc/") + c.Assert(*resp.InventoryConfiguration.Destination.KS3BucketDestination.Format, Equals, "CSV") + c.Assert(*resp.InventoryConfiguration.Destination.KS3BucketDestination.Bucket, Equals, bucket) + c.Assert(*resp.InventoryConfiguration.Destination.KS3BucketDestination.Prefix, Equals, "prefix/") + c.Assert(*resp.InventoryConfiguration.Schedule.Frequency, Equals, "Once") + c.Assert(len(resp.InventoryConfiguration.OptionalFields.Field), Equals, 6) + // delete,通过context取消 + ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) + defer cancelFunc() + _, err = client.DeleteBucketInventoryWithContext(ctx, &s3.DeleteBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + }) + c.Assert(err, NotNil) + // delete + _, err = client.DeleteBucketInventoryWithContext(context.Background(), &s3.DeleteBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + }) + c.Assert(err, IsNil) +} + +// List Bucket Inventory +func (s *Ks3utilCommandSuite) TestListBucketInventoryWithContext(c *C) { + id := randLowStr(8) + // put,不通过context取消 + _, err := client.PutBucketInventoryWithContext(context.Background(), &s3.PutBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + InventoryConfiguration: &s3.InventoryConfiguration{ + Id: aws.String(id), + IsEnabled: aws.Boolean(true), + Filter: &s3.InventoryFilter{ + Prefix: aws.String("abc/"), + }, + Destination: &s3.Destination{ + KS3BucketDestination: &s3.KS3BucketDestination{ + Format: aws.String("CSV"), + Bucket: aws.String(bucket), + Prefix: aws.String("prefix/"), + }, + }, + Schedule: &s3.Schedule{ + Frequency: aws.String("Once"), + }, + OptionalFields: &s3.OptionalFields{ + Field: []*string{ + aws.String("Size"), + aws.String("LastModifiedDate"), + aws.String("ETag"), + aws.String("StorageClass"), + aws.String("IsMultipartUploaded"), + aws.String("EncryptionStatus"), + }, + }, + }, + }) + c.Assert(err, IsNil) + // list + resp, err := client.ListBucketInventoryWithContext(context.Background(), &s3.ListBucketInventoryInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, IsNil) + c.Assert(len(resp.InventoryConfigurationsResult.InventoryConfigurations), Equals, 1) + // list,通过context取消 + ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) + defer cancelFunc() + _, err = client.ListBucketInventoryWithContext(ctx, &s3.ListBucketInventoryInput{ + Bucket: aws.String(bucket), + }) + c.Assert(err, NotNil) + // delete + _, err = client.DeleteBucketInventoryWithContext(context.Background(), &s3.DeleteBucketInventoryInput{ + Bucket: aws.String(bucket), + Id: aws.String(id), + }) + c.Assert(err, IsNil) +} + // PUT Bucket ACL func (s *Ks3utilCommandSuite) TestPutBucketACLWithContext(c *C) { // put,不通过context取消 From f172fc80b80390a716499717d2b20aa14313dcda Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Sat, 12 Oct 2024 14:40:36 +0800 Subject: [PATCH 08/13] =?UTF-8?q?chore:=20=E6=9B=B4=E6=96=B0=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 2 +- go.sum | 32 +++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 7b8293f..241942f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,6 @@ go 1.16 require ( github.com/stretchr/testify v1.8.1 - golang.org/x/net v0.24.0 + golang.org/x/net v0.30.0 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 ) diff --git a/go.sum b/go.sum index 162c901..67a271c 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -13,21 +14,32 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -35,24 +47,34 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From af6d31a9dfc4320d47590a0a117b8e8fc0a1a719 Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Thu, 17 Oct 2024 11:16:05 +0800 Subject: [PATCH 09/13] =?UTF-8?q?fix:=20=E9=94=99=E8=AF=AF=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- go.mod | 1 - go.sum | 66 ---------------------- internal/protocol/query/unmarshal_error.go | 46 ++++----------- 4 files changed, 14 insertions(+), 102 deletions(-) diff --git a/.gitignore b/.gitignore index d566b8a..0c15d57 100755 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ vendor doc .idea/ -.DS_Store \ No newline at end of file +.DS_Store +go-sdk-test* \ No newline at end of file diff --git a/go.mod b/go.mod index 241942f..03b673a 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,5 @@ go 1.16 require ( github.com/stretchr/testify v1.8.1 - golang.org/x/net v0.30.0 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 ) diff --git a/go.sum b/go.sum index 67a271c..2ec90f7 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -11,71 +10,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/protocol/query/unmarshal_error.go b/internal/protocol/query/unmarshal_error.go index 88fed23..472d148 100755 --- a/internal/protocol/query/unmarshal_error.go +++ b/internal/protocol/query/unmarshal_error.go @@ -1,13 +1,11 @@ package query import ( - "bytes" "encoding/xml" "github.com/ks3sdklib/aws-sdk-go/aws" "github.com/ks3sdklib/aws-sdk-go/internal/apierr" - "golang.org/x/net/html" "io" - "strings" + "regexp" ) type XmlErrorResponse struct { @@ -31,19 +29,17 @@ func UnmarshalError(r *aws.Request) { } // 如果响应类型是html,则解析html文本 - if strings.Contains(r.HTTPResponse.Header.Get("Content-Type"), "html") { - // 解析HTML文本 - doc, err := html.Parse(bytes.NewReader(body)) - if err != nil { - r.Error = apierr.New("Unmarshal", "failed to parse html", err) - return + if r.HTTPResponse.Header.Get("Content-Type") == "text/html" { + // 获取HTML文本中title标签的内容 + re := regexp.MustCompile(`<title>(.*?)</title>`) + matches := re.FindStringSubmatch(string(body)) + + title := "" + if len(matches) > 1 { + title = matches[1] } - title := findTitle(doc) - r.Error = apierr.NewRequestError( - apierr.New(title, "", nil), - r.HTTPResponse.StatusCode, - "", - ) + + r.Error = apierr.NewRequestError(apierr.New(title, "", nil), r.HTTPResponse.StatusCode, "") return } @@ -52,7 +48,7 @@ func UnmarshalError(r *aws.Request) { // head请求无法从body中获取request id,如果是head请求,则从header中获取 if resp.RequestID == "" && r.HTTPRequest.Method == "HEAD" { - resp.RequestID = r.HTTPResponse.Header.Get("X-KSS-Request-Id") + resp.RequestID = r.HTTPResponse.Header.Get("X-Kss-Request-Id") } if err != nil && err != io.EOF { @@ -65,21 +61,3 @@ func UnmarshalError(r *aws.Request) { ) } } - -// findTitle 提取HTML文档中<title>标签的内容 -func findTitle(doc *html.Node) string { - var title string - - var traverse func(*html.Node) - traverse = func(n *html.Node) { - if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil { - title = n.FirstChild.Data - } - for c := n.FirstChild; c != nil; c = c.NextSibling { - traverse(c) - } - } - - traverse(doc) - return title -} From 637f8571bb0718640266c3e39817e64022cebc7e Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Thu, 17 Oct 2024 11:29:55 +0800 Subject: [PATCH 10/13] =?UTF-8?q?test:=20=E5=9B=9E=E6=94=B6=E7=AB=99?= =?UTF-8?q?=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/bucketsample_test.go | 6 +++--- test/bucketwithcontext_test.go | 18 +++++++++--------- test/common_test.go | 1 + 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/test/bucketsample_test.go b/test/bucketsample_test.go index 9a413a5..34f5813 100644 --- a/test/bucketsample_test.go +++ b/test/bucketsample_test.go @@ -331,7 +331,7 @@ func (s *Ks3utilCommandSuite) TestBucketDecompressPolicy(c *C) { // TestBucketRetention bucket retention func (s *Ks3utilCommandSuite) TestBucketRetention(c *C) { _, err := client.PutBucketRetention(&s3.PutBucketRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), RetentionConfiguration: &s3.BucketRetentionConfiguration{ Rule: &s3.RetentionRule{ Status: aws.String("Enabled"), @@ -342,14 +342,14 @@ func (s *Ks3utilCommandSuite) TestBucketRetention(c *C) { c.Assert(err, IsNil) resp, err := client.GetBucketRetention(&s3.GetBucketRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), }) c.Assert(err, IsNil) c.Assert(*resp.RetentionConfiguration.Rule.Status, Equals, "Enabled") c.Assert(*resp.RetentionConfiguration.Rule.Days, Equals, int64(30)) _, err = client.ListRetention(&s3.ListRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), }) c.Assert(err, IsNil) } diff --git a/test/bucketwithcontext_test.go b/test/bucketwithcontext_test.go index be6a29a..56f9a5e 100644 --- a/test/bucketwithcontext_test.go +++ b/test/bucketwithcontext_test.go @@ -1052,7 +1052,7 @@ func (s *Ks3utilCommandSuite) TestPutBucketRetentionWithContext(c *C) { c.Assert(err, IsNil) // get resp, err := client.GetBucketRetentionWithContext(context.Background(), &s3.GetBucketRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), }) c.Assert(err, IsNil) c.Assert(*resp.RetentionConfiguration.Rule.Status, Equals, "Enabled") @@ -1061,7 +1061,7 @@ func (s *Ks3utilCommandSuite) TestPutBucketRetentionWithContext(c *C) { ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) defer cancelFunc() _, err = client.PutBucketRetentionWithContext(ctx, &s3.PutBucketRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), RetentionConfiguration: &s3.BucketRetentionConfiguration{ Rule: &s3.RetentionRule{ Status: aws.String("Enabled"), @@ -1072,7 +1072,7 @@ func (s *Ks3utilCommandSuite) TestPutBucketRetentionWithContext(c *C) { c.Assert(err, NotNil) // get resp, err = client.GetBucketRetentionWithContext(context.Background(), &s3.GetBucketRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), }) c.Assert(err, IsNil) c.Assert(*resp.RetentionConfiguration.Rule.Status, Equals, "Enabled") @@ -1083,7 +1083,7 @@ func (s *Ks3utilCommandSuite) TestPutBucketRetentionWithContext(c *C) { func (s *Ks3utilCommandSuite) TestGetBucketRetentionWithContext(c *C) { // put _, err := client.PutBucketRetentionWithContext(context.Background(), &s3.PutBucketRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), RetentionConfiguration: &s3.BucketRetentionConfiguration{ Rule: &s3.RetentionRule{ Status: aws.String("Enabled"), @@ -1094,7 +1094,7 @@ func (s *Ks3utilCommandSuite) TestGetBucketRetentionWithContext(c *C) { c.Assert(err, IsNil) // get,不通过context取消 resp, err := client.GetBucketRetentionWithContext(context.Background(), &s3.GetBucketRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), }) c.Assert(err, IsNil) c.Assert(*resp.RetentionConfiguration.Rule.Status, Equals, "Enabled") @@ -1103,7 +1103,7 @@ func (s *Ks3utilCommandSuite) TestGetBucketRetentionWithContext(c *C) { ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) defer cancelFunc() resp, err = client.GetBucketRetentionWithContext(ctx, &s3.GetBucketRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), }) c.Assert(err, NotNil) } @@ -1112,7 +1112,7 @@ func (s *Ks3utilCommandSuite) TestGetBucketRetentionWithContext(c *C) { func (s *Ks3utilCommandSuite) TestListBucketRetentionWithContext(c *C) { // put _, err := client.PutBucketRetentionWithContext(context.Background(), &s3.PutBucketRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), RetentionConfiguration: &s3.BucketRetentionConfiguration{ Rule: &s3.RetentionRule{ Status: aws.String("Enabled"), @@ -1123,7 +1123,7 @@ func (s *Ks3utilCommandSuite) TestListBucketRetentionWithContext(c *C) { c.Assert(err, IsNil) // list,不通过context取消 resp, err := client.ListRetentionWithContext(context.Background(), &s3.ListRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), Prefix: aws.String("test/"), }) c.Assert(err, IsNil) @@ -1132,7 +1132,7 @@ func (s *Ks3utilCommandSuite) TestListBucketRetentionWithContext(c *C) { ctx, cancelFunc := context.WithTimeout(context.Background(), bucketTimeout) defer cancelFunc() resp, err = client.ListRetentionWithContext(ctx, &s3.ListRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), Prefix: aws.String("test/"), }) c.Assert(err, NotNil) diff --git a/test/common_test.go b/test/common_test.go index 7008c8a..42314d9 100644 --- a/test/common_test.go +++ b/test/common_test.go @@ -41,6 +41,7 @@ var ( commonNamePrefix = "go-sdk-test-" testFileDir = "go-sdk-test-file/" timeout = 1 * time.Microsecond + retentionBucket = commonNamePrefix + randLowStr(10) ) // SetUpSuite 在测试套件启动前执行一次 From 45c564ff2650e98f42865744d5a1f31049e11bcc Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Thu, 17 Oct 2024 14:15:36 +0800 Subject: [PATCH 11/13] =?UTF-8?q?test:=20=E5=9B=9E=E6=94=B6=E7=AB=99?= =?UTF-8?q?=E5=8D=95=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/bucketsample_test.go | 3 +++ test/bucketwithcontext_test.go | 9 +++++++++ test/common_test.go | 15 ++++++++++++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/test/bucketsample_test.go b/test/bucketsample_test.go index 34f5813..671a34f 100644 --- a/test/bucketsample_test.go +++ b/test/bucketsample_test.go @@ -330,6 +330,8 @@ func (s *Ks3utilCommandSuite) TestBucketDecompressPolicy(c *C) { // TestBucketRetention bucket retention func (s *Ks3utilCommandSuite) TestBucketRetention(c *C) { + retentionBucket := commonNamePrefix + randLowStr(10) + s.CreateBucket(retentionBucket, c) _, err := client.PutBucketRetention(&s3.PutBucketRetentionInput{ Bucket: aws.String(retentionBucket), RetentionConfiguration: &s3.BucketRetentionConfiguration{ @@ -352,6 +354,7 @@ func (s *Ks3utilCommandSuite) TestBucketRetention(c *C) { Bucket: aws.String(retentionBucket), }) c.Assert(err, IsNil) + s.DeleteBucket(retentionBucket, c) } // TestBucketReplication bucket replication diff --git a/test/bucketwithcontext_test.go b/test/bucketwithcontext_test.go index 56f9a5e..7e6ba59 100644 --- a/test/bucketwithcontext_test.go +++ b/test/bucketwithcontext_test.go @@ -1039,6 +1039,8 @@ func (s *Ks3utilCommandSuite) TestDeleteBucketReplicationWithContext(c *C) { // PUT Bucket Retention func (s *Ks3utilCommandSuite) TestPutBucketRetentionWithContext(c *C) { + retentionBucket := commonNamePrefix + randLowStr(10) + s.CreateBucket(retentionBucket, c) // put,不通过context取消 _, err := client.PutBucketRetentionWithContext(context.Background(), &s3.PutBucketRetentionInput{ Bucket: aws.String(bucket), @@ -1077,10 +1079,13 @@ func (s *Ks3utilCommandSuite) TestPutBucketRetentionWithContext(c *C) { c.Assert(err, IsNil) c.Assert(*resp.RetentionConfiguration.Rule.Status, Equals, "Enabled") c.Assert(*resp.RetentionConfiguration.Rule.Days, Equals, int64(30)) + s.DeleteBucket(retentionBucket, c) } // GET Bucket Retention func (s *Ks3utilCommandSuite) TestGetBucketRetentionWithContext(c *C) { + retentionBucket := commonNamePrefix + randLowStr(10) + s.CreateBucket(retentionBucket, c) // put _, err := client.PutBucketRetentionWithContext(context.Background(), &s3.PutBucketRetentionInput{ Bucket: aws.String(retentionBucket), @@ -1106,10 +1111,13 @@ func (s *Ks3utilCommandSuite) TestGetBucketRetentionWithContext(c *C) { Bucket: aws.String(retentionBucket), }) c.Assert(err, NotNil) + s.DeleteBucket(retentionBucket, c) } // List Retention func (s *Ks3utilCommandSuite) TestListBucketRetentionWithContext(c *C) { + retentionBucket := commonNamePrefix + randLowStr(10) + s.CreateBucket(retentionBucket, c) // put _, err := client.PutBucketRetentionWithContext(context.Background(), &s3.PutBucketRetentionInput{ Bucket: aws.String(retentionBucket), @@ -1136,6 +1144,7 @@ func (s *Ks3utilCommandSuite) TestListBucketRetentionWithContext(c *C) { Prefix: aws.String("test/"), }) c.Assert(err, NotNil) + s.DeleteBucket(retentionBucket, c) } // PUT Bucket Inventory diff --git a/test/common_test.go b/test/common_test.go index 42314d9..fa94595 100644 --- a/test/common_test.go +++ b/test/common_test.go @@ -41,7 +41,6 @@ var ( commonNamePrefix = "go-sdk-test-" testFileDir = "go-sdk-test-file/" timeout = 1 * time.Microsecond - retentionBucket = commonNamePrefix + randLowStr(10) ) // SetUpSuite 在测试套件启动前执行一次 @@ -228,3 +227,17 @@ func (s *Ks3utilCommandSuite) DeleteObject(key string, c *C) { }) c.Assert(err, IsNil) } + +func (s *Ks3utilCommandSuite) CreateBucket(bucketName string, c *C) { + _, err := client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + }) + c.Assert(err, IsNil) +} + +func (s *Ks3utilCommandSuite) DeleteBucket(bucketName string, c *C) { + _, err := client.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) + c.Assert(err, IsNil) +} From 9e8af44d8fcc389261fd99b4608800a37061dc31 Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Thu, 17 Oct 2024 14:23:15 +0800 Subject: [PATCH 12/13] =?UTF-8?q?test:=20=E5=9B=9E=E6=94=B6=E7=AB=99?= =?UTF-8?q?=E5=8D=95=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/bucketwithcontext_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bucketwithcontext_test.go b/test/bucketwithcontext_test.go index 7e6ba59..0f1fcb1 100644 --- a/test/bucketwithcontext_test.go +++ b/test/bucketwithcontext_test.go @@ -1043,7 +1043,7 @@ func (s *Ks3utilCommandSuite) TestPutBucketRetentionWithContext(c *C) { s.CreateBucket(retentionBucket, c) // put,不通过context取消 _, err := client.PutBucketRetentionWithContext(context.Background(), &s3.PutBucketRetentionInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(retentionBucket), RetentionConfiguration: &s3.BucketRetentionConfiguration{ Rule: &s3.RetentionRule{ Status: aws.String("Enabled"), From e9333a699fdf7013cea58f54f44370d4bb3d8017 Mon Sep 17 00:00:00 2001 From: likui2 <likui2@kingsoft.com> Date: Thu, 17 Oct 2024 16:07:09 +0800 Subject: [PATCH 13/13] =?UTF-8?q?fix:=20=E9=94=99=E8=AF=AF=E8=A7=A3?= =?UTF-8?q?=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/protocol/query/unmarshal_error.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/protocol/query/unmarshal_error.go b/internal/protocol/query/unmarshal_error.go index 472d148..dc2049c 100755 --- a/internal/protocol/query/unmarshal_error.go +++ b/internal/protocol/query/unmarshal_error.go @@ -6,6 +6,7 @@ import ( "github.com/ks3sdklib/aws-sdk-go/internal/apierr" "io" "regexp" + "strings" ) type XmlErrorResponse struct { @@ -29,7 +30,7 @@ func UnmarshalError(r *aws.Request) { } // 如果响应类型是html,则解析html文本 - if r.HTTPResponse.Header.Get("Content-Type") == "text/html" { + if strings.Contains(r.HTTPResponse.Header.Get("Content-Type"), "text/html") { // 获取HTML文本中title标签的内容 re := regexp.MustCompile(`<title>(.*?)</title>`) matches := re.FindStringSubmatch(string(body))