Merge pull request #53 from tencentyun/dev/feature-retry_download

Dev/feature retry download

README.md

@@ -6,13 +6,13 @@
 
 ## 下载链接
 
-当前版本：v0.19.0-beta
+当前版本：v0.20.0-beta
 
-[Linux](https://github.com/tencentyun/coscli/releases/download/v0.19.0-beta/coscli-linux)
+[Linux](https://github.com/tencentyun/coscli/releases/download/v0.20.0-beta/coscli-linux)
 
-[Mac](https://github.com/tencentyun/coscli/releases/download/v0.19.0-beta/coscli-mac)
+[Mac](https://github.com/tencentyun/coscli/releases/download/v0.20.0-beta/coscli-mac)
 
-[Windows](https://github.com/tencentyun/coscli/releases/download/v0.19.0-beta/coscli-windows.exe)
+[Windows](https://github.com/tencentyun/coscli/releases/download/v0.20.0-beta/coscli-windows.exe)
 
 ## 使用方法
 

cmd/cp.go

@@ -51,9 +51,16 @@ Example:
 		threadNum, _ := cmd.Flags().GetInt("thread-num")
 		metaString, _ := cmd.Flags().GetString("meta")
 		meta, err := util.MetaStringToHeader(metaString)
+		retryNum, _ := cmd.Flags().GetInt("retry-num")
 		if err != nil {
 			logger.Fatalln("Copy invalid meta " + err.Error())
 		}
+
+		if retryNum < 0 || retryNum > 10 {
+			logger.Fatalln("retry-num must be between 0 and 10 (inclusive)")
+			return
+		}
+
 		// args[0]: 源地址
 		// args[1]: 目标地址
 		if !util.IsCosPath(args[0]) && util.IsCosPath(args[1]) {
@@ -73,7 +80,7 @@ Example:
 				PartSize:     partSize,
 				ThreadNum:    threadNum,
 			}
-			download(args, recursive, include, exclude, op)
+			download(args, recursive, include, exclude, retryNum, op)
 		} else if util.IsCosPath(args[0]) && util.IsCosPath(args[1]) {
 			// 拷贝
 			cosCopy(args, recursive, include, exclude, meta, storageClass)
@@ -96,6 +103,7 @@ func init() {
 	cpCmd.Flags().String("meta", "",
 		"Set the meta information of the file, "+
 			"the format is header:value#header:value, the example is Cache-Control:no-cache#Content-Encoding:gzip")
+	cpCmd.Flags().Int("retry-num", 0, "Retry download")
 }
 
 func upload(args []string, recursive bool, include string, exclude string, op *util.UploadOptions) {
@@ -110,13 +118,13 @@ func upload(args []string, recursive bool, include string, exclude string, op *u
 	}
 }
 
-func download(args []string, recursive bool, include string, exclude string, op *util.DownloadOptions) {
+func download(args []string, recursive bool, include string, exclude string, retryNum int, op *util.DownloadOptions) {
 	bucketName, cosPath := util.ParsePath(args[0])
 	_, localPath := util.ParsePath(args[1])
 	c := util.NewClient(&config, &param, bucketName)
 
 	if recursive {
-		util.MultiDownload(c, bucketName, cosPath, localPath, include, exclude, op)
+		util.MultiDownload(c, bucketName, cosPath, localPath, include, exclude, retryNum, op)
 	} else {
 		util.SingleDownload(c, bucketName, cosPath, localPath, op, false)
 	}
@@ -133,7 +141,7 @@ func cosCopy(args []string, recursive bool, include string, exclude string, meta
 		// 记录是否是代码添加的路径分隔符
 		isAddSeparator := false
 		// 源路径若不以路径分隔符结尾，则添加
-		if !strings.HasSuffix(cosPath1, "/")  && cosPath1 != ""{
+		if !strings.HasSuffix(cosPath1, "/") && cosPath1 != "" {
 			isAddSeparator = true
 			cosPath1 += "/"
 		}

cmd/root.go

@@ -27,7 +27,7 @@ var rootCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		_ = cmd.Help()
 	},
-	Version: "v0.19.0-beta",
+	Version: "v0.20.0-beta",
 }
 
 func Execute() {

cmd/sync.go

@@ -53,9 +53,16 @@ Example:
 		metaString, _ := cmd.Flags().GetString("meta")
 		snapshotPath, _ := cmd.Flags().GetString("snapshot-path")
 		meta, err := util.MetaStringToHeader(metaString)
+		retryNum, _ := cmd.Flags().GetInt("retry-num")
 		if err != nil {
 			logger.Fatalln("Sync invalid meta, reason: " + err.Error())
 		}
+
+		if retryNum < 0 || retryNum > 10 {
+			logger.Fatalln("retry-num must be between 0 and 10 (inclusive)")
+			return
+		}
+
 		// args[0]: 源地址
 		// args[1]: 目标地址
 		var snapshotDb *leveldb.DB
@@ -86,7 +93,7 @@ Example:
 				SnapshotPath: snapshotPath,
 				SnapshotDb:   snapshotDb,
 			}
-			syncDownload(args, recursive, include, exclude, op)
+			syncDownload(args, recursive, include, exclude, retryNum, op)
 		} else if util.IsCosPath(args[0]) && util.IsCosPath(args[1]) {
 			// 拷贝
 			syncCopy(args, recursive, include, exclude, meta, storageClass)
@@ -129,6 +136,7 @@ func init() {
 		"In addition, coscli does not automatically delete snapshot-path snapshot information, "+
 		"in order to avoid too much snapshot information, when the snapshot information is useless, "+
 		"please clean up your own snapshot-path on your own immediately.")
+	syncCmd.Flags().Int("retry-num", 0, "Retry download")
 }
 
 func syncUpload(args []string, recursive bool, include string, exclude string, op *util.UploadOptions,
@@ -144,13 +152,13 @@ func syncUpload(args []string, recursive bool, include string, exclude string, o
 	}
 }
 
-func syncDownload(args []string, recursive bool, include string, exclude string, op *util.DownloadOptions) {
+func syncDownload(args []string, recursive bool, include string, exclude string, retryNum int, op *util.DownloadOptions) {
 	bucketName, cosPath := util.ParsePath(args[0])
 	_, localPath := util.ParsePath(args[1])
 	c := util.NewClient(&config, &param, bucketName)
 
 	if recursive {
-		util.SyncMultiDownload(c, bucketName, cosPath, localPath, include, exclude, op)
+		util.SyncMultiDownload(c, bucketName, cosPath, localPath, include, exclude, retryNum, op)
 	} else {
 		util.SyncSingleDownload(c, bucketName, cosPath, localPath, op, "", recursive)
 	}
@@ -167,7 +175,7 @@ func syncCopy(args []string, recursive bool, include string, exclude string, met
 		// 记录是否是代码添加的路径分隔符
 		isAddSeparator := false
 		// 源路径若不以路径分隔符结尾，则添加
-		if !strings.HasSuffix(cosPath1, "/")  && cosPath1 != ""{
+		if !strings.HasSuffix(cosPath1, "/") && cosPath1 != "" {
 			isAddSeparator = true
 			cosPath1 += "/"
 		}

util/download.go

@@ -126,7 +126,7 @@ func SingleDownload(c *cos.Client, bucketName, cosPath, localPath string, op *Do
 	return nil
 }
 
-func MultiDownload(c *cos.Client, bucketName, cosDir, localDir, include, exclude string, op *DownloadOptions) {
+func MultiDownload(c *cos.Client, bucketName, cosDir, localDir, include, exclude string, retryNum int, op *DownloadOptions) {
 	if localDir == "" {
 		logger.Fatalln("localDir is empty")
 		os.Exit(1)
@@ -136,12 +136,12 @@ func MultiDownload(c *cos.Client, bucketName, cosDir, localDir, include, exclude
 	// 记录是否是代码添加的路径分隔符
 	isCosAddSeparator := false
 	// cos路径若不以路径分隔符结尾，则添加
-	if !strings.HasSuffix(cosDir, "/") && cosDir != ""{
+	if !strings.HasSuffix(cosDir, "/") && cosDir != "" {
 		isCosAddSeparator = true
 		cosDir += "/"
 	}
 	// 判断cosDir是否是文件夹
-	isDir := CheckCosPathType(c, cosDir, 0)
+	isDir := CheckCosPathType(c, cosDir, 0, retryNum)
 
 	if isDir {
 		// cosDir是文件夹 且 localDir不以路径分隔符结尾，则添加
@@ -162,13 +162,13 @@ func MultiDownload(c *cos.Client, bucketName, cosDir, localDir, include, exclude
 		}
 	}
 
-	objects, commonPrefixes := GetObjectsListRecursive(c, cosDir, 0, include, exclude)
+	objects, commonPrefixes := GetObjectsListRecursive(c, cosDir, 0, include, exclude, retryNum)
 	listObjects(c, bucketName, objects, cosDir, localDir, op)
 
 	if len(commonPrefixes) > 0 {
 		for i := 0; i < len(commonPrefixes); i++ {
 			localDirTemp := localDir + commonPrefixes[i]
-			MultiDownload(c, bucketName, commonPrefixes[i], localDirTemp, include, exclude, op)
+			MultiDownload(c, bucketName, commonPrefixes[i], localDirTemp, include, exclude, retryNum, op)
 		}
 	}
 

util/list.go

@@ -5,10 +5,12 @@ import (
 	"fmt"
 	"io/fs"
 	"io/ioutil"
+	"math/rand"
 	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
+	"time"
 
 	logger "github.com/sirupsen/logrus"
 	"github.com/tencentyun/cos-go-sdk-v5"
@@ -173,7 +175,7 @@ func GetObjectsList(c *cos.Client, prefix string, limit int, include string, exc
 			isTruncated = false
 		} else {
 			isTruncated = res.IsTruncated
-			marker = res.NextMarker
+			marker, _ = url.QueryUnescape(res.NextMarker)
 		}
 	}
 
@@ -191,7 +193,7 @@ func GetObjectsList(c *cos.Client, prefix string, limit int, include string, exc
 
 func GetObjectsListForLs(c *cos.Client, prefix string, limit int, include string, exclude string,
 	marker string) (dirs []string,
-	objects []cos.Object, isTruncated bool, nextMaker string) {
+	objects []cos.Object, isTruncated bool, nextMarker string) {
 	opt := &cos.BucketGetOptions{
 		Prefix:       prefix,
 		Delimiter:    "/",
@@ -220,7 +222,7 @@ func GetObjectsListForLs(c *cos.Client, prefix string, limit int, include string
 		isTruncated = false
 	} else {
 		isTruncated = res.IsTruncated
-		nextMaker = res.NextMarker
+		nextMarker, _ = url.QueryUnescape(res.NextMarker)
 	}
 
 	if len(include) > 0 {
@@ -232,13 +234,19 @@ func GetObjectsListForLs(c *cos.Client, prefix string, limit int, include string
 		dirs = MatchPattern(dirs, exclude, false)
 	}
 
-	return dirs, objects, isTruncated, nextMaker
+	return dirs, objects, isTruncated, nextMarker
 }
 
-func CheckCosPathType(c *cos.Client, prefix string, limit int) (isDir bool) {
+func CheckCosPathType(c *cos.Client, prefix string, limit int, retryCount ...int) (isDir bool) {
 	if prefix == "" {
 		return true
 	}
+
+	retries := 0
+	if len(retryCount) > 0 {
+		retries = retryCount[0]
+	}
+
 	opt := &cos.BucketGetOptions{
 		Prefix:       prefix,
 		Delimiter:    "",
@@ -247,7 +255,7 @@ func CheckCosPathType(c *cos.Client, prefix string, limit int) (isDir bool) {
 		MaxKeys:      limit,
 	}
 
-	res, _, err := c.Bucket.Get(context.Background(), opt)
+	res, err := tryGetBucket(c, opt, retries)
 	if err != nil {
 		logger.Fatalln(err)
 		os.Exit(1)
@@ -260,8 +268,14 @@ func CheckCosPathType(c *cos.Client, prefix string, limit int) (isDir bool) {
 	return isDir
 }
 
-func GetObjectsListRecursive(c *cos.Client, prefix string, limit int, include string, exclude string) (objects []cos.Object,
+func GetObjectsListRecursive(c *cos.Client, prefix string, limit int, include string, exclude string, retryCount ...int) (objects []cos.Object,
 	commonPrefixes []string) {
+
+	retries := 0
+	if len(retryCount) > 0 {
+		retries = retryCount[0]
+	}
+
 	opt := &cos.BucketGetOptions{
 		Prefix:       prefix,
 		Delimiter:    "",
@@ -275,7 +289,7 @@ func GetObjectsListRecursive(c *cos.Client, prefix string, limit int, include st
 	for isTruncated {
 		opt.Marker = marker
 
-		res, _, err := c.Bucket.Get(context.Background(), opt)
+		res, err := tryGetBucket(c, opt, retries)
 		if err != nil {
 			logger.Fatalln(err)
 			os.Exit(1)
@@ -288,7 +302,7 @@ func GetObjectsListRecursive(c *cos.Client, prefix string, limit int, include st
 			isTruncated = false
 		} else {
 			isTruncated = res.IsTruncated
-			marker = res.NextMarker
+			marker, _ = url.QueryUnescape(res.NextMarker)
 		}
 	}
 
@@ -305,6 +319,29 @@ func GetObjectsListRecursive(c *cos.Client, prefix string, limit int, include st
 	return objects, commonPrefixes
 }
 
+func tryGetBucket(c *cos.Client, opt *cos.BucketGetOptions, retryCount int) (*cos.BucketGetResult, error) {
+	for i := 0; i <= retryCount; i++ {
+		res, resp, err := c.Bucket.Get(context.Background(), opt)
+		if err != nil {
+			if resp.StatusCode == 503 {
+				if i == retryCount {
+					return res, err
+				} else {
+					fmt.Println("Error 503: Service Unavailable. Retrying...")
+					waitTime := time.Duration(rand.Intn(10)+1) * time.Second
+					time.Sleep(waitTime)
+					continue
+				}
+			} else {
+				return res, err
+			}
+		} else {
+			return res, err
+		}
+	}
+	return nil, fmt.Errorf("Retry limit exceeded")
+}
+
 func GetObjectsListRecursiveForLs(c *cos.Client, prefix string, limit int, include string, exclude string,
 	marker string) (objects []cos.Object, isTruncated bool, nextMarker string, commonPrefixes []string) {
 	opt := &cos.BucketGetOptions{
@@ -331,7 +368,7 @@ func GetObjectsListRecursiveForLs(c *cos.Client, prefix string, limit int, inclu
 		isTruncated = false
 	} else {
 		isTruncated = res.IsTruncated
-		nextMarker = res.NextMarker
+		nextMarker, _ = url.QueryUnescape(res.NextMarker)
 	}
 
 	if len(include) > 0 {

util/list_iterator.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"context"
+	"net/url"
 	"os"
 
 	logger "github.com/sirupsen/logrus"
@@ -28,7 +29,7 @@ func GetObjectsListIterator(c *cos.Client, prefix, marker string, include, exclu
 	commonPrefixes = res.CommonPrefixes
 
 	isTruncated = res.IsTruncated
-	nextMarker = res.NextMarker
+	nextMarker, _ = url.QueryUnescape(res.NextMarker)
 
 	if len(include) > 0 {
 		objects = MatchCosPattern(objects, include, true)

util/synchronize.go

@@ -274,20 +274,20 @@ func getCosLastModified(c *cos.Client, cosPath string) (lmt string, err error) {
 	}
 }
 
-func SyncMultiDownload(c *cos.Client, bucketName, cosDir, localDir, include, exclude string, op *DownloadOptions) {
+func SyncMultiDownload(c *cos.Client, bucketName, cosDir, localDir, include, exclude string, retryNum int, op *DownloadOptions) {
 	if localDir == "" {
 		logger.Fatalln("localDir is empty")
 		os.Exit(1)
 	}
 	// 记录是否是代码添加的路径分隔符
 	isCosAddSeparator := false
 	// cos路径若不以路径分隔符结尾，则添加
-	if !strings.HasSuffix(cosDir, "/") && cosDir != ""{
+	if !strings.HasSuffix(cosDir, "/") && cosDir != "" {
 		isCosAddSeparator = true
 		cosDir += "/"
 	}
 	// 判断cosDir是否是文件夹
-	isDir := CheckCosPathType(c, cosDir, 0)
+	isDir := CheckCosPathType(c, cosDir, 0, retryNum)
 
 	if isDir {
 		// cosDir是文件夹
@@ -309,7 +309,7 @@ func SyncMultiDownload(c *cos.Client, bucketName, cosDir, localDir, include, exc
 		}
 	}
 
-	objects, _ := GetObjectsListRecursive(c, cosDir, 0, include, exclude)
+	objects, _ := GetObjectsListRecursive(c, cosDir, 0, include, exclude, retryNum)
 	if len(objects) == 0 {
 		logger.Warningf("cosDir: cos://%s is empty\n", cosDir)
 		return