Merge dev-1.3.0 into master

aws/config.go

@@ -1,6 +1,7 @@
 package aws
 
 import (
+	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -59,6 +60,7 @@ type Config struct {
 	S3ForcePathStyle        bool
 	DomainMode              bool
 	SignerVersion           string
+	CrcCheckEnabled         bool
 }
 
 // Copy will return a shallow copy of the Config object.
@@ -79,6 +81,7 @@ func (c Config) Copy() Config {
 	dst.S3ForcePathStyle = c.S3ForcePathStyle
 	dst.DomainMode = c.DomainMode
 	dst.SignerVersion = c.SignerVersion
+	dst.CrcCheckEnabled = c.CrcCheckEnabled
 	return dst
 }
 
@@ -183,5 +186,22 @@ func (c Config) Merge(newcfg *Config) *Config {
 	} else {
 		cfg.SignerVersion = c.SignerVersion
 	}
+	if newcfg.CrcCheckEnabled {
+		cfg.CrcCheckEnabled = newcfg.CrcCheckEnabled
+	} else {
+		cfg.CrcCheckEnabled = c.CrcCheckEnabled
+	}
 	return &cfg
 }
+
+const (
+	LogOff = iota
+	LogOn
+)
+
+func (c Config) WriteLog(LogLevel uint, format string, a ...interface{}) {
+	if c.LogLevel < LogLevel {
+		return
+	}
+	fmt.Fprintf(c.Logger, format, a...)
+}

aws/context.go

@@ -2,14 +2,13 @@ package aws
 
 import (
 	"context"
-	"github.com/aws/aws-sdk-go/aws"
 	"github.com/ks3sdklib/aws-sdk-go/internal/apierr"
 )
 
 type Context = context.Context
 
 // SetContext adds a Context to the current request that can be used to cancel
-func (r *Request) SetContext(ctx aws.Context) {
+func (r *Request) SetContext(ctx Context) {
 	if ctx == nil {
 		r.Error = apierr.New("InvalidParameter", "context cannot be nil", nil)
 	}

aws/handlers.go

@@ -13,6 +13,7 @@ type Handlers struct {
 	UnmarshalError   HandlerList
 	Retry            HandlerList
 	AfterRetry       HandlerList
+	CheckCrc64       HandlerList
 }
 
 // copy returns of this handler's lists.
@@ -28,6 +29,7 @@ func (h *Handlers) copy() Handlers {
 		UnmarshalMeta:    h.UnmarshalMeta.copy(),
 		Retry:            h.Retry.copy(),
 		AfterRetry:       h.AfterRetry.copy(),
+		CheckCrc64:       h.CheckCrc64.copy(),
 	}
 }
 
@@ -43,6 +45,7 @@ func (h *Handlers) Clear() {
 	h.ValidateResponse.Clear()
 	h.Retry.Clear()
 	h.AfterRetry.Clear()
+	h.CheckCrc64.Clear()
 }
 
 // A HandlerList manages zero or more handlers in a list.

aws/request.go

@@ -2,10 +2,11 @@ package aws
 
 import (
 	"bytes"
-	"github.com/aws/aws-sdk-go/aws"
 	"github.com/ks3sdklib/aws-sdk-go/aws/awsutil"
+	"github.com/ks3sdklib/aws-sdk-go/internal/crc"
+	"github.com/ks3sdklib/aws-sdk-go/internal/util"
+	"hash"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"reflect"
@@ -31,9 +32,9 @@ type Request struct {
 	RetryCount   uint
 	Retryable    SettableBool
 	RetryDelay   time.Duration
-
-	built   bool
-	context aws.Context
+	built        bool
+	context      Context
+	Crc64        hash.Hash64
 }
 
 // An Operation is the service API operation to be made.
@@ -133,7 +134,14 @@ func (r *Request) SetStringBody(s string) {
 
 // SetReaderBody will set the request's body reader.
 func (r *Request) SetReaderBody(reader io.ReadSeeker) {
-	r.HTTPRequest.Body = ioutil.NopCloser(reader)
+	if r.Config.CrcCheckEnabled {
+		crc := crc.NewCRC(crc.CrcTable(), 0)
+		teeReader := util.TeeReader(reader, crc)
+		r.HTTPRequest.Body = teeReader
+		r.Crc64 = crc
+	} else {
+		r.HTTPRequest.Body = io.NopCloser(reader)
+	}
 	r.Body = reader
 }
 
@@ -216,6 +224,7 @@ func (r *Request) Send() error {
 		}
 
 		r.Handlers.Unmarshal.Run(r)
+		r.Handlers.CheckCrc64.Run(r)
 		if r.Error != nil {
 			r.Handlers.Retry.Run(r)
 			r.Handlers.AfterRetry.Run(r)

aws/version.go

@@ -5,4 +5,4 @@ package aws
 const SDKName = "aws-sdk-go"
 
 // SDKVersion is the version of this SDK
-const SDKVersion = "1.2.9"
+const SDKVersion = "1.3.0"

go.mod

@@ -3,7 +3,6 @@ module github.com/ks3sdklib/aws-sdk-go
 go 1.16
 
 require (
-	github.com/aws/aws-sdk-go v1.44.204
 	github.com/stretchr/testify v1.8.1
 	gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405
 )

go.sum

@@ -1,60 +1,17 @@
-github.com/aws/aws-sdk-go v1.44.204 h1:7/tPUXfNOHB390A63t6fJIwmlwVQAkAwcbzKsU2/6OQ=
-github.com/aws/aws-sdk-go v1.44.204/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/panjf2000/ants/v2 v2.7.1 h1:qBy5lfSdbxvrR0yUnZfaEDjf0FlCw4ufsbcsxmE7r+M=
-github.com/panjf2000/ants/v2 v2.7.1/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
-go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/crc/crc64.go

@@ -0,0 +1,127 @@
+package crc
+
+import (
+	"hash"
+	"hash/crc64"
+)
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	crc uint64
+	tab *crc64.Table
+}
+
+// NewCRC creates a new hash.Hash64 computing the CRC64 checksum
+// using the polynomial represented by the Table.
+func NewCRC(tab *crc64.Table, init uint64) hash.Hash64 { return &digest{init, tab} }
+
+var CrcTable = func() *crc64.Table {
+	return crc64.MakeTable(crc64.ECMA)
+}
+
+// Size returns the number of bytes sum will return.
+func (d *digest) Size() int { return crc64.Size }
+
+// BlockSize returns the hash's underlying block size.
+// The Write method must be able to accept any amount
+// of data, but it may operate more efficiently if all writes
+// are a multiple of the block size.
+func (d *digest) BlockSize() int { return 1 }
+
+// Reset resets the hash to its initial state.
+func (d *digest) Reset() { d.crc = 0 }
+
+// Write (via the embedded io.Writer interface) adds more data to the running hash.
+// It never returns an error.
+func (d *digest) Write(p []byte) (n int, err error) {
+	d.crc = crc64.Update(d.crc, d.tab, p)
+	return len(p), nil
+}
+
+// Sum64 returns CRC64 value.
+func (d *digest) Sum64() uint64 { return d.crc }
+
+// Sum returns hash value.
+func (d *digest) Sum(in []byte) []byte {
+	s := d.Sum64()
+	return append(in, byte(s>>56), byte(s>>48), byte(s>>40), byte(s>>32), byte(s>>24), byte(s>>16), byte(s>>8), byte(s))
+}
+
+// gf2Dim dimension of GF(2) vectors (length of CRC)
+const gf2Dim int = 64
+
+func gf2MatrixTimes(mat []uint64, vec uint64) uint64 {
+	var sum uint64
+	for i := 0; vec != 0; i++ {
+		if vec&1 != 0 {
+			sum ^= mat[i]
+		}
+
+		vec >>= 1
+	}
+	return sum
+}
+
+func gf2MatrixSquare(square []uint64, mat []uint64) {
+	for n := 0; n < gf2Dim; n++ {
+		square[n] = gf2MatrixTimes(mat, mat[n])
+	}
+}
+
+// CRC64Combine combines CRC64
+func CRC64Combine(crc1 uint64, crc2 uint64, len2 uint64) uint64 {
+	var even [gf2Dim]uint64 // Even-power-of-two zeros operator
+	var odd [gf2Dim]uint64  // Odd-power-of-two zeros operator
+
+	// Degenerate case
+	if len2 == 0 {
+		return crc1
+	}
+
+	// Put operator for one zero bit in odd
+	odd[0] = crc64.ECMA // CRC64 polynomial
+	var row uint64 = 1
+	for n := 1; n < gf2Dim; n++ {
+		odd[n] = row
+		row <<= 1
+	}
+
+	// Put operator for two zero bits in even
+	gf2MatrixSquare(even[:], odd[:])
+
+	// Put operator for four zero bits in odd
+	gf2MatrixSquare(odd[:], even[:])
+
+	// Apply len2 zeros to crc1, first square will put the operator for one zero byte, eight zero bits, in even
+	for {
+		// Apply zeros operator for this bit of len2
+		gf2MatrixSquare(even[:], odd[:])
+
+		if len2&1 != 0 {
+			crc1 = gf2MatrixTimes(even[:], crc1)
+		}
+
+		len2 >>= 1
+
+		// If no more bits set, then done
+		if len2 == 0 {
+			break
+		}
+
+		// Another iteration of the loop with odd and even swapped
+		gf2MatrixSquare(odd[:], even[:])
+		if len2&1 != 0 {
+			crc1 = gf2MatrixTimes(odd[:], crc1)
+		}
+		len2 >>= 1
+
+		// If no more bits set, then done
+		if len2 == 0 {
+			break
+		}
+	}
+
+	// Return combined CRC
+	crc1 ^= crc2
+	return crc1
+}

internal/protocol/ec2query/build_test.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"encoding/xml"
-	"github.com/aws/aws-sdk-go/private/util"
+	"github.com/ks3sdklib/aws-sdk-go/internal/util"
 	"io"
 	"io/ioutil"
 	"net/http"

internal/protocol/jsonrpc/build_test.go

@@ -15,7 +15,7 @@ import (
 	"github.com/ks3sdklib/aws-sdk-go/internal/protocol/jsonrpc"
 	"github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil"
 	"github.com/ks3sdklib/aws-sdk-go/internal/signer/v4"
-	"github.com/aws/aws-sdk-go/private/util"
+	"github.com/ks3sdklib/aws-sdk-go/internal/util"
 	"github.com/stretchr/testify/assert"
 )
 

internal/protocol/query/build_test.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"encoding/xml"
-	"github.com/aws/aws-sdk-go/private/util"
+	"github.com/ks3sdklib/aws-sdk-go/internal/util"
 	"io"
 	"io/ioutil"
 	"net/http"

internal/protocol/query/unmarshal_test.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"encoding/xml"
-	"github.com/aws/aws-sdk-go/private/util"
+	"github.com/ks3sdklib/aws-sdk-go/internal/util"
 	"io"
 	"io/ioutil"
 	"net/http"

internal/protocol/restjson/unmarshal_test.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"encoding/xml"
-	"github.com/aws/aws-sdk-go/private/util"
+	"github.com/ks3sdklib/aws-sdk-go/internal/util"
 	"io"
 	"io/ioutil"
 	"net/http"

internal/signer/v2/v2.go

@@ -57,6 +57,8 @@ var signQuerys = map[string]bool{
 	"copy":                         true,
 	"mirror":                       true,
 	"restore":                      true,
+	"append":                       true,
+	"position":                     true,
 }
 
 type signer struct {