Merge pull request #734 from tesshuflower/cherry-pick-release-0.7-res…

…tic-v0.15.2

Merge pull request #733 from tesshuflower/restic_v0.15.2

CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [unreleased]
 
+### Added
+
+- Restic - ReplicationSource/ReplicationDestination can now specify a CustomCA
+  that is from a configmap rather than only from a secret.
+- Rclone - ReplicationSource/ReplicationDestination can now specify a CustomCA
+  that is contained in either a configmap or secret.
+- Restic - New option to run a restic unlock before the backup in the next sync.
+
+### Changed
+
+- Syncthing upgraded to v1.23.4
+- Restic upgraded to v0.15.2
+
+## [0.7.1]
+
 ### Changed
 
 - Modified leader election settings (LeaseDuration, RenewDeadline, RetryPeriod)

mover-restic/SOURCE_VERSIONS

@@ -1,2 +1,2 @@
-https://github.com/restic/restic.git v0.15.1 7d4b7ad9cb45d6801df96fcf61f876d580416fa5
-https://github.com/minio/minio-go.git v7.0.47 b37640650d4f94e749bd8542d490580f5251e2a0
+https://github.com/restic/restic.git v0.15.2 be8be3397c2ba4f3ec986998a5af4a5a4c309bc3
+https://github.com/minio/minio-go.git v7.0.50 7c9b5ff1c09425b93bf72ff2e3ed6d4a272bd910

mover-restic/minio-go/.github/workflows/codeql.yml

@@ -27,15 +27,15 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       # Override language selection by uncommenting this and choosing your languages
       # with:
       #   languages: go, javascript, csharp, python, cpp, java
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -49,4 +49,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

mover-restic/minio-go/.github/workflows/go-windows.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.18.x, 1.19.x]
+        go-version: [1.19.x, 1.20.x]
         os: [windows-latest]
     steps:
       - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }}

mover-restic/minio-go/.github/workflows/go.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x, 1.19.x]
+        go-version: [1.19.x, 1.20.x]
         os: [ubuntu-latest]
     steps:
       - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }}

mover-restic/minio-go/.github/workflows/vulncheck.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go-version: [ 1.19 ]
+        go-version: [ 1.20.x ]
     steps:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3

mover-restic/minio-go/.gitignore

@@ -2,4 +2,5 @@
 *.test
 validator
 golangci-lint
-functional_tests
+functional_tests
+.idea

mover-restic/minio-go/api-datatypes.go

@@ -43,7 +43,7 @@ type StringMap map[string]string
 // if m is nil it can be initialized, which is often the case if m is
 // nested in another xml structural. This is also why the first thing done
 // on the first line is initialize it.
-func (m *StringMap) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+func (m *StringMap) UnmarshalXML(d *xml.Decoder, _ xml.StartElement) error {
 	*m = StringMap{}
 	type Item struct {
 		Key   string

mover-restic/minio-go/api-get-object.go

@@ -23,8 +23,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"net/url"
-	"strconv"
 	"sync"
 
 	"github.com/minio/minio-go/v7/pkg/s3utils"
@@ -654,19 +652,11 @@ func (c *Client) getObject(ctx context.Context, bucketName, objectName string, o
 		return nil, ObjectInfo{}, nil, err
 	}
 
-	urlValues := make(url.Values)
-	if opts.VersionID != "" {
-		urlValues.Set("versionId", opts.VersionID)
-	}
-	if opts.PartNumber > 0 {
-		urlValues.Set("partNumber", strconv.Itoa(opts.PartNumber))
-	}
-
 	// Execute GET on objectName.
 	resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
 		bucketName:       bucketName,
 		objectName:       objectName,
-		queryValues:      urlValues,
+		queryValues:      opts.toQueryValues(),
 		customHeader:     opts.Header(),
 		contentSHA256Hex: emptySHA256Hex,
 	})

mover-restic/minio-go/api-get-options.go

@@ -20,6 +20,8 @@ package minio
 import (
 	"fmt"
 	"net/http"
+	"net/url"
+	"strconv"
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/encrypt"
@@ -36,6 +38,7 @@ type AdvancedGetOptions struct {
 // during GET requests.
 type GetObjectOptions struct {
 	headers              map[string]string
+	reqParams            url.Values
 	ServerSideEncryption encrypt.ServerSide
 	VersionID            string
 	PartNumber           int
@@ -83,6 +86,34 @@ func (o *GetObjectOptions) Set(key, value string) {
 	o.headers[http.CanonicalHeaderKey(key)] = value
 }
 
+// SetReqParam - set request query string parameter
+// supported key: see supportedQueryValues.
+// If an unsupported key is passed in, it will be ignored and nothing will be done.
+func (o *GetObjectOptions) SetReqParam(key, value string) {
+	if !isStandardQueryValue(key) {
+		// do nothing
+		return
+	}
+	if o.reqParams == nil {
+		o.reqParams = make(url.Values)
+	}
+	o.reqParams.Set(key, value)
+}
+
+// AddReqParam - add request query string parameter
+// supported key: see supportedQueryValues.
+// If an unsupported key is passed in, it will be ignored and nothing will be done.
+func (o *GetObjectOptions) AddReqParam(key, value string) {
+	if !isStandardQueryValue(key) {
+		// do nothing
+		return
+	}
+	if o.reqParams == nil {
+		o.reqParams = make(url.Values)
+	}
+	o.reqParams.Add(key, value)
+}
+
 // SetMatchETag - set match etag.
 func (o *GetObjectOptions) SetMatchETag(etag string) error {
 	if etag == "" {
@@ -149,3 +180,24 @@ func (o *GetObjectOptions) SetRange(start, end int64) error {
 	}
 	return nil
 }
+
+// toQueryValues - Convert the versionId, partNumber, and reqParams in Options to query string parameters.
+func (o *GetObjectOptions) toQueryValues() url.Values {
+	urlValues := make(url.Values)
+	if o.VersionID != "" {
+		urlValues.Set("versionId", o.VersionID)
+	}
+	if o.PartNumber > 0 {
+		urlValues.Set("partNumber", strconv.Itoa(o.PartNumber))
+	}
+
+	if o.reqParams != nil {
+		for key, values := range o.reqParams {
+			for _, value := range values {
+				urlValues.Add(key, value)
+			}
+		}
+	}
+
+	return urlValues
+}

mover-restic/minio-go/api-put-object-multipart.go

@@ -387,6 +387,12 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
 		return UploadInfo{}, err
 	}
 
+	headers := opts.Header()
+	if s3utils.IsAmazonEndpoint(*c.endpointURL) {
+		headers.Del(encrypt.SseKmsKeyID)      // Remove X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id not supported in CompleteMultipartUpload
+		headers.Del(encrypt.SseGenericHeader) // Remove X-Amz-Server-Side-Encryption not supported in CompleteMultipartUpload
+	}
+
 	// Instantiate all the complete multipart buffer.
 	completeMultipartUploadBuffer := bytes.NewReader(completeMultipartUploadBytes)
 	reqMetadata := requestMetadata{
@@ -396,7 +402,7 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
 		contentBody:      completeMultipartUploadBuffer,
 		contentLength:    int64(len(completeMultipartUploadBytes)),
 		contentSHA256Hex: sum256Hex(completeMultipartUploadBytes),
-		customHeader:     opts.Header(),
+		customHeader:     headers,
 	}
 
 	// Execute POST to complete multipart upload for an objectName.

mover-restic/minio-go/api-put-object-streaming.go

@@ -500,8 +500,6 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
 	// CRC32C is ~50% faster on AMD64 @ 30GB/s
 	var crcBytes []byte
 	crc := crc32.New(crc32.MakeTable(crc32.Castagnoli))
-	md5Hash := c.md5Hasher()
-	defer md5Hash.Close()
 
 	// Total data read and written to server. should be equal to 'size' at the end of the call.
 	var totalUploadedSize int64
@@ -569,9 +567,10 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
 			var md5Base64 string
 
 			if opts.SendContentMd5 {
-				md5Hash.Reset()
+				md5Hash := c.md5Hasher()
 				md5Hash.Write(buf[:length])
 				md5Base64 = base64.StdEncoding.EncodeToString(md5Hash.Sum(nil))
+				md5Hash.Close()
 			}
 
 			defer wg.Done()
@@ -590,6 +589,7 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
 			objPart, uerr := c.uploadPart(ctx, p)
 			if uerr != nil {
 				errCh <- uerr
+				return
 			}
 
 			// Save successfully uploaded part metadata.

mover-restic/minio-go/api-put-object.go

@@ -93,6 +93,28 @@ type PutObjectOptions struct {
 	// This can be used for faster uploads on non-seekable or slow-to-seek input.
 	ConcurrentStreamParts bool
 	Internal              AdvancedPutOptions
+
+	customHeaders http.Header
+}
+
+// SetMatchETag if etag matches while PUT MinIO returns an error
+// this is a MinIO specific extension to support optimistic locking
+// semantics.
+func (opts *PutObjectOptions) SetMatchETag(etag string) {
+	if opts.customHeaders == nil {
+		opts.customHeaders = http.Header{}
+	}
+	opts.customHeaders.Set("If-Match", "\""+etag+"\"")
+}
+
+// SetMatchETagExcept if etag does not match while PUT MinIO returns an
+// error this is a MinIO specific extension to support optimistic locking
+// semantics.
+func (opts *PutObjectOptions) SetMatchETagExcept(etag string) {
+	if opts.customHeaders == nil {
+		opts.customHeaders = http.Header{}
+	}
+	opts.customHeaders.Set("If-None-Match", "\""+etag+"\"")
 }
 
 // getNumThreads - gets the number of threads to be used in the multipart
@@ -187,6 +209,12 @@ func (opts PutObjectOptions) Header() (header http.Header) {
 			header.Set("x-amz-meta-"+k, v)
 		}
 	}
+
+	// set any other additional custom headers.
+	for k, v := range opts.customHeaders {
+		header[k] = v
+	}
+
 	return
 }
 

mover-restic/minio-go/api-putobject-snowball.go

@@ -59,6 +59,7 @@ type SnowballObject struct {
 	Size int64
 
 	// Modtime to apply to the object.
+	// If Modtime is the zero value current time will be used.
 	ModTime time.Time
 
 	// Content of the object.
@@ -172,6 +173,10 @@ objectLoop:
 				ModTime:  obj.ModTime,
 				Format:   tar.FormatPAX,
 			}
+			if header.ModTime.IsZero() {
+				header.ModTime = time.Now().UTC()
+			}
+
 			if err := t.WriteHeader(&header); err != nil {
 				closeObj()
 				return err

mover-restic/minio-go/api-remove.go

@@ -235,7 +235,7 @@ func generateRemoveMultiObjectsRequest(objects []ObjectInfo) []byte {
 
 // processRemoveMultiObjectsResponse - parse the remove multi objects web service
 // and return the success/failure result status for each object
-func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, resultCh chan<- RemoveObjectResult) {
+func processRemoveMultiObjectsResponse(body io.Reader, resultCh chan<- RemoveObjectResult) {
 	// Parse multi delete XML response
 	rmResult := &deleteMultiObjectsResult{}
 	err := xmlDecoder(body, rmResult)
@@ -459,7 +459,7 @@ func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh
 		}
 
 		// Process multiobjects remove xml response
-		processRemoveMultiObjectsResponse(resp.Body, batch, resultCh)
+		processRemoveMultiObjectsResponse(resp.Body, resultCh)
 
 		closeResponse(resp)
 	}

mover-restic/minio-go/api-s3-datatypes.go

@@ -110,7 +110,7 @@ type ListVersionsResult struct {
 // UnmarshalXML is a custom unmarshal code for the response of ListObjectVersions, the custom
 // code will unmarshal <Version> and <DeleteMarker> tags and save them in Versions field to
 // preserve the lexical order of the listing.
-func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) {
+func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, _ xml.StartElement) (err error) {
 	for {
 		// Read tokens from the XML document in a stream.
 		t, err := d.Token()

mover-restic/minio-go/api.go

@@ -106,6 +106,12 @@ type Options struct {
 	Region       string
 	BucketLookup BucketLookupType
 
+	// Allows setting a custom region lookup based on URL pattern
+	// not all URL patterns are covered by this library so if you
+	// have a custom endpoints with many regions you can use this
+	// function to perform region lookups appropriately.
+	CustomRegionViaURL func(u url.URL) string
+
 	// TrailingHeaders indicates server support of trailing headers.
 	// Only supported for v4 signatures.
 	TrailingHeaders bool
@@ -118,7 +124,7 @@ type Options struct {
 // Global constants.
 const (
 	libraryName    = "minio-go"
-	libraryVersion = "v7.0.47"
+	libraryVersion = "v7.0.50"
 )
 
 // User Agent should always following the below style.
@@ -234,7 +240,11 @@ func privateNew(endpoint string, opts *Options) (*Client, error) {
 
 	// Sets custom region, if region is empty bucket location cache is used automatically.
 	if opts.Region == "" {
-		opts.Region = s3utils.GetRegionFromURL(*clnt.endpointURL)
+		if opts.CustomRegionViaURL != nil {
+			opts.Region = opts.CustomRegionViaURL(*clnt.endpointURL)
+		} else {
+			opts.Region = s3utils.GetRegionFromURL(*clnt.endpointURL)
+		}
 	}
 	clnt.region = opts.Region
 

mover-restic/minio-go/bucket-cache.go

@@ -190,12 +190,11 @@ func (c *Client) getBucketLocationRequest(ctx context.Context, bucketName string
 		}
 	}
 
-	isVirtualHost := s3utils.IsVirtualHostSupported(targetURL, bucketName)
+	isVirtualStyle := c.isVirtualHostStyleRequest(targetURL, bucketName)
 
 	var urlStr string
 
-	// only support Aliyun OSS for virtual hosted path,  compatible  Amazon & Google Endpoint
-	if isVirtualHost && s3utils.IsAliyunOSSEndpoint(targetURL) {
+	if isVirtualStyle {
 		urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + targetURL.Host + "/?location"
 	} else {
 		targetURL.Path = path.Join(bucketName, "") + "/"