From eed903f3e93d18a15df7b072e45a1b4924eb0e27 Mon Sep 17 00:00:00 2001 From: willppan <1377853927@qq.com> Date: Thu, 10 Aug 2023 15:57:56 +0800 Subject: [PATCH] Dev/feature cvm (#43) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cvm角色及ls,cp命令bug修复 * cvm角色及ls,cp命令bug修复 * cvm角色及ls,cp命令bug修复 * ls中文兼容 * url解码兼容文件夹名 * cli命令传入秘钥使用传入秘钥进行鉴权 * cli命令传入秘钥使用传入秘钥进行鉴权 * ls命令文件名换行问题处理 * v0.14.0-beta版本发布 --- .gitignore | 2 +- README.md | 8 ++--- cmd/config_init.go | 4 +++ cmd/config_set.go | 24 +++++++++++++- cmd/config_show.go | 2 ++ cmd/ls.go | 1 + cmd/root.go | 2 +- util/cam_auth.go | 81 +++++++++++++++++++++++++++++++++++++++++++++ util/client.go | 49 ++++++++++++++++++++++----- util/data.go | 2 ++ util/list.go | 37 +++++++++++++++++++-- util/synchronize.go | 1 + 12 files changed, 194 insertions(+), 19 deletions(-) create mode 100644 util/cam_auth.go diff --git a/.gitignore b/.gitignore index dc6b50b..c647f19 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,5 @@ vendor coscli coscli.exe - +coscli.log test_clear.sh diff --git a/README.md b/README.md index ad5ae34..2c4c9ec 100755 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ ## 下载链接 -当前版本:v0.13.0-beta +当前版本:v0.14.0-beta -[Linux](https://github.com/tencentyun/coscli/releases/download/v0.13.0-beta/coscli-linux) +[Linux](https://github.com/tencentyun/coscli/releases/download/v0.14.0-beta/coscli-linux) -[Mac](https://github.com/tencentyun/coscli/releases/download/v0.13.0-beta/coscli-mac) +[Mac](https://github.com/tencentyun/coscli/releases/download/v0.14.0-beta/coscli-mac) -[Windows](https://github.com/tencentyun/coscli/releases/download/v0.13.0-beta/coscli-windows.exe) +[Windows](https://github.com/tencentyun/coscli/releases/download/v0.14.0-beta/coscli-windows.exe) ## 使用方法 diff --git a/cmd/config_init.go b/cmd/config_init.go index 5601eca..57c0352 100755 --- a/cmd/config_init.go +++ b/cmd/config_init.go @@ -60,6 +60,10 @@ func initConfigFile(cfgFlag bool) { _, _ = fmt.Scanf("%s\n", &config.Base.SecretKey) fmt.Println("Input Your Session Token:") _, _ = fmt.Scanf("%s\n", &config.Base.SessionToken) + fmt.Println("Input Your Mode:") + _, _ = fmt.Scanf("%s\n", &config.Base.Mode) + fmt.Println("Input Your Cvm Role Name:") + _, _ = fmt.Scanf("%s\n", &config.Base.CvmRoleName) if len(config.Base.SessionToken) < 3 { config.Base.SessionToken = "" } diff --git a/cmd/config_set.go b/cmd/config_set.go index 52e7aed..e70c24d 100755 --- a/cmd/config_set.go +++ b/cmd/config_set.go @@ -31,6 +31,8 @@ func init() { configSetCmd.Flags().StringP("secret_id", "", "", "Set secret id") configSetCmd.Flags().StringP("secret_key", "", "", "Set secret key") configSetCmd.Flags().StringP("session_token", "t", "", "Set session token") + configSetCmd.Flags().StringP("mode", "", "", "Set mode") + configSetCmd.Flags().StringP("cvm_role_name", "", "", "Set cvm role name") } func setConfigItem(cmd *cobra.Command) { @@ -38,7 +40,8 @@ func setConfigItem(cmd *cobra.Command) { secretID, _ := cmd.Flags().GetString("secret_id") secretKey, _ := cmd.Flags().GetString("secret_key") sessionToken, _ := cmd.Flags().GetString("session_token") - + mode, _ := cmd.Flags().GetString("mode") + cvmRoleName, _ := cmd.Flags().GetString("cvm_role_name") if secretID != "" { flag = true if secretID == "@" { @@ -63,6 +66,25 @@ func setConfigItem(cmd *cobra.Command) { config.Base.SessionToken = sessionToken } } + if mode != "" { + flag = true + if mode != "SecretKey" && mode != "CvmRole" { + logger.Fatalln("Please Enter Mode As SecretKey Or CvmRole!") + logger.Infoln(cmd.UsageString()) + os.Exit(1) + } else { + config.Base.Mode = mode + } + } + if cvmRoleName != "" { + flag = true + if cvmRoleName == "@" { + config.Base.CvmRoleName = "" + } else { + config.Base.CvmRoleName = cvmRoleName + } + } + if !flag { logger.Fatalln("Enter at least one configuration item to be modified!") logger.Infoln(cmd.UsageString()) diff --git a/cmd/config_show.go b/cmd/config_show.go index 262fcb9..63ecfe5 100755 --- a/cmd/config_show.go +++ b/cmd/config_show.go @@ -33,6 +33,8 @@ func showConfig() { logger.Infof(" Secret ID: %s\n", config.Base.SecretID) logger.Infof(" Secret Key: %s\n", config.Base.SecretKey) logger.Infof(" Session Token: %s\n", config.Base.SessionToken) + logger.Infof(" Mode: %s\n", config.Base.Mode) + logger.Infof(" CvmRoleName: %s\n", config.Base.CvmRoleName) logger.Infoln("====================") logger.Infoln("Bucket Configuration Information:") diff --git a/cmd/ls.go b/cmd/ls.go index 8ed4d8d..4dae0a9 100644 --- a/cmd/ls.go +++ b/cmd/ls.go @@ -84,6 +84,7 @@ func listObjects(cosPath string, limit int, recursive bool, include string, excl table.SetHeader([]string{"Key", "Type", "Last Modified", "Size"}) table.SetBorder(false) table.SetAlignment(tablewriter.ALIGN_RIGHT) + table.SetAutoWrapText(false) if recursive { for { output_num = 0 diff --git a/cmd/root.go b/cmd/root.go index 464fec1..cee308e 100755 --- a/cmd/root.go +++ b/cmd/root.go @@ -25,7 +25,7 @@ var rootCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { _ = cmd.Help() }, - Version: "v0.13.0-beta", + Version: "v0.14.0-beta", } func Execute() { diff --git a/util/cam_auth.go b/util/cam_auth.go new file mode 100644 index 0000000..a9f85ad --- /dev/null +++ b/util/cam_auth.go @@ -0,0 +1,81 @@ +package util + +import ( + "context" + "encoding/json" + logger "github.com/sirupsen/logrus" + "io/ioutil" + "net/http" + "os" + "time" +) + +const ( + CamUrl = "http://metadata.tencentyun.com/meta-data/cam/security-credentials/" +) + +type Data struct { + TmpSecretId string `json:"TmpSecretId"` + TmpSecretKey string `json:"TmpSecretKey"` + ExpiredTime int `json:"ExpiredTime"` + Expiration string `json:"Expiration"` + Token string `json:"Token"` + Code string `json:"Code"` +} + +var data Data + +func CamAuth(roleName string) Data { + if roleName == "" { + logger.Fatalln("Get cam auth error : roleName not set") + os.Exit(1) + } + + // 创建HTTP客户端 + client := &http.Client{} + + // 创建一个5秒的超时上下文 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // 创建一个HTTP GET请求并将上下文与其关联 + req, err := http.NewRequest("GET", CamUrl+roleName, nil) + if err != nil { + logger.Fatalln("Get cam auth error : create request error", err) + os.Exit(1) + } + req = req.WithContext(ctx) + + // 发起HTTP GET请求 + res, err := client.Do(req) + if err != nil { + // 检查是否超时错误 + if ctx.Err() == context.DeadlineExceeded { + logger.Fatalln("Get cam auth timeout", ctx.Err()) + } else { + logger.Fatalln("Get cam auth error : request error", err) + } + os.Exit(1) + } + + defer res.Body.Close() + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + logger.Fatalln("Get cam auth error : get response error", err) + os.Exit(1) + } + + err = json.Unmarshal(body, &data) + if err != nil { + logger.Fatalln("Get cam auth error : auth error") + os.Exit(1) + } + + if data.Code != "Success" { + logger.Fatalln("Get cam auth error : response error", err) + os.Exit(1) + } + + return data +} diff --git a/util/client.go b/util/client.go index 669e3ea..922ac8b 100755 --- a/util/client.go +++ b/util/client.go @@ -1,21 +1,34 @@ package util import ( - "net/http" - "github.com/tencentyun/cos-go-sdk-v5" + "net/http" ) +var secretID, secretKey, secretToken string + // 根据桶别名,从配置文件中加载信息,创建客户端 func NewClient(config *Config, param *Param, bucketName string) *cos.Client { - secretID := config.Base.SecretID - secretKey := config.Base.SecretKey - secretToken := config.Base.SessionToken + if config.Base.Mode == "CvmRole" { + // 若使用 CvmRole 方式,则需请求请求CAM的服务,获取临时密钥 + data := CamAuth(config.Base.CvmRoleName) + secretID = data.TmpSecretId + secretKey = data.TmpSecretKey + secretToken = data.Token + } else { + // SecretKey 方式则直接获取用户配置文件中设置的密钥 + secretID = config.Base.SecretID + secretKey = config.Base.SecretKey + secretToken = config.Base.SessionToken + } + // 若参数中有传 SecretID 或 SecretKey ,需将之前赋值的SessionToken置为空,否则会出现使用参数的 SecretID 和 SecretKey ,却使用了CvmRole方式返回的token,导致鉴权失败 if param.SecretID != "" { secretID = param.SecretID + secretToken = "" } if param.SecretKey != "" { secretKey = param.SecretKey + secretToken = "" } if param.SessionToken != "" { secretToken = param.SessionToken @@ -41,18 +54,36 @@ func NewClient(config *Config, param *Param, bucketName string) *cos.Client { // 根据函数参数创建客户端 func CreateClient(config *Config, param *Param, bucketIDName string) *cos.Client { - secretID := config.Base.SecretID - secretKey := config.Base.SecretKey + if config.Base.Mode == "CvmRole" { + // 若使用 CvmRole 方式,则需请求请求CAM的服务,获取临时密钥 + data := CamAuth(config.Base.CvmRoleName) + secretID = data.TmpSecretId + secretKey = data.TmpSecretKey + secretToken = data.Token + } else { + // SecretKey 方式则直接获取用户配置文件中设置的密钥 + secretID = config.Base.SecretID + secretKey = config.Base.SecretKey + secretToken = config.Base.SessionToken + } + + // 若参数中有传 SecretID 或 SecretKey ,需将之前赋值的SessionToken置为空,否则会出现使用参数的 SecretID 和 SecretKey ,却使用了CvmRole方式返回的token,导致鉴权失败 if param.SecretID != "" { secretID = param.SecretID + secretToken = "" } if param.SecretKey != "" { secretKey = param.SecretKey + secretToken = "" + } + if param.SessionToken != "" { + secretToken = param.SessionToken } return cos.NewClient(CreateURL(bucketIDName, config.Base.Protocol, param.Endpoint), &http.Client{ Transport: &cos.AuthorizationTransport{ - SecretID: secretID, - SecretKey: secretKey, + SecretID: secretID, + SecretKey: secretKey, + SessionToken: secretToken, }, }) } diff --git a/util/data.go b/util/data.go index 144326d..82f019d 100644 --- a/util/data.go +++ b/util/data.go @@ -10,6 +10,8 @@ type BaseCfg struct { SecretKey string `yaml:"secretkey"` SessionToken string `yaml:"sessiontoken"` Protocol string `yaml:"protocol"` + Mode string `yaml:"mode"` + CvmRoleName string `yaml:"cvmrolename"` } type Bucket struct { diff --git a/util/list.go b/util/list.go index 9f9cc34..6237dd4 100644 --- a/util/list.go +++ b/util/list.go @@ -5,6 +5,7 @@ import ( "fmt" "io/fs" "io/ioutil" + "net/url" "os" "regexp" @@ -26,6 +27,24 @@ func MatchBucketPattern(buckets []cos.Bucket, pattern string, include bool) []co return res } +func UrlDecodeCosPattern(objects []cos.Object) []cos.Object { + res := make([]cos.Object, 0) + for _, o := range objects { + o.Key, _ = url.QueryUnescape(o.Key) + res = append(res, o) + } + return res +} + +func UrlDecodePattern(strs []string) []string { + res := make([]string, 0) + for _, s := range strs { + s, _ = url.QueryUnescape(s) + res = append(res, s) + } + return res +} + func MatchCosPattern(objects []cos.Object, pattern string, include bool) []cos.Object { res := make([]cos.Object, 0) for _, o := range objects { @@ -175,7 +194,7 @@ func GetObjectsListForLs(c *cos.Client, prefix string, limit int, include string opt := &cos.BucketGetOptions{ Prefix: prefix, Delimiter: "/", - EncodingType: "", + EncodingType: "url", Marker: marker, MaxKeys: limit, } @@ -190,6 +209,12 @@ func GetObjectsListForLs(c *cos.Client, prefix string, limit int, include string dirs = append(dirs, res.CommonPrefixes...) objects = append(objects, res.Contents...) + // 对key进行urlDecode解码 + objects = UrlDecodeCosPattern(objects) + + // 对dir进行urlDecode解码 + dirs = UrlDecodePattern(dirs) + if limit > 0 { isTruncated = false } else { @@ -214,7 +239,7 @@ func GetObjectsListRecursive(c *cos.Client, prefix string, limit int, include st opt := &cos.BucketGetOptions{ Prefix: prefix, Delimiter: "", - EncodingType: "", + EncodingType: "url", Marker: "", MaxKeys: limit, } @@ -233,6 +258,9 @@ func GetObjectsListRecursive(c *cos.Client, prefix string, limit int, include st objects = append(objects, res.Contents...) commonPrefixes = res.CommonPrefixes + // 对key进行urlDecode解码 + objects = UrlDecodeCosPattern(objects) + if limit > 0 { isTruncated = false } else { @@ -256,7 +284,7 @@ func GetObjectsListRecursiveForLs(c *cos.Client, prefix string, limit int, inclu opt := &cos.BucketGetOptions{ Prefix: prefix, Delimiter: "", - EncodingType: "", + EncodingType: "url", Marker: marker, MaxKeys: limit, } @@ -270,6 +298,9 @@ func GetObjectsListRecursiveForLs(c *cos.Client, prefix string, limit int, inclu objects = append(objects, res.Contents...) commonPrefixes = res.CommonPrefixes + // 对key进行urlDecode解码 + objects = UrlDecodeCosPattern(objects) + if limit > 0 { isTruncated = false } else { diff --git a/util/synchronize.go b/util/synchronize.go index 3d3ecd6..ef6c41a 100644 --- a/util/synchronize.go +++ b/util/synchronize.go @@ -104,6 +104,7 @@ func SyncSingleDownload(c *cos.Client, bucketName, cosPath, localPath string, op cosLastModified string) error { localPath, cosPath, err := DownloadPathFixed(localPath, cosPath) if err != nil { + logger.Fatalln(err) return err } _, err = os.Stat(localPath)