Skip to content

Commit 8943644

Browse files
Mei-ZhaoMei-Zhao
authored
Merge pull request #349 from YangSen-qn/refactor3
Refactor3
2 parents ca10d67 + 77640c3 commit 8943644

File tree

122 files changed

+2330
-1306
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

122 files changed

+2330
-1306
lines changed

‎CHANGELOG.md

+10
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,13 @@
1+
# 2.9.0
2+
1. 支持批量修改 status，命令：batchforbidden
3+
2. qdownload 支持自定义文件本地存储路径
4+
3. 增加 func 命令用于验证 qshell 指定的回调函数，当前有 qdownload 处理文件本地存储路径
5+
4. 增加 stat 接口展示信息
6+
5. 增加 bucket 接口展示信息
7+
6. 优化 qupload 上传流程中检测文件是否存在机制，上传更可靠
8+
7. 优化 qdownload 下载流程中检测文件是否存在机制，下载更可靠
9+
8. qdownload 下载忽略以路径结(/)尾并且大小为 0 的文件
10+
111
# 2.8.0
212
1. 新增创建存储空间命令 mkbucket 和查看存储空间信息命令 bucket
313
2. 存储相关所有 batch 操作命令，支持保存当前执行状态，方便重试，详细使用方式可通过 qshell <子命令> 查看

‎README.md

+60-58
Large diffs are not rendered by default.

‎cmd/aws.go

-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ func awsFetchCmdBuilder(cfg *iqshell.Config) *cobra.Command {
4646

4747
// NewCmdAwsList 返回一个cobra.Command指针
4848
// 该命令列举亚马逊存储空间中的文件, 会忽略目录
49-
5049
func awsListCmdBuilder(cfg *iqshell.Config) *cobra.Command {
5150
info := aws.ListBucketInfo{}
5251
cmd := &cobra.Command{

‎cmd/bucket.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package cmd
33
import (
44
"github.com/qiniu/qshell/v2/docs"
55
"github.com/qiniu/qshell/v2/iqshell"
6+
"github.com/qiniu/qshell/v2/iqshell/common/data"
67
"github.com/qiniu/qshell/v2/iqshell/storage/bucket/operations"
78
"github.com/spf13/cobra"
89
)
@@ -94,7 +95,7 @@ var listBucketCmd2Builder = func(cfg *iqshell.Config) *cobra.Command {
9495
var cmd = &cobra.Command{
9596
Use: "listbucket2 <Bucket>",
9697
Short: "List all the files in the bucket using v2/list interface",
97-
Long: "List all the files in the bucket using v2/list interface to stdout if output file not specified. Each row of data information is displayed in the following order:\n Key\tSize\tHash\tPutTime\tMimeType\tFileType\tEndUser",
98+
Long: "List all the files in the bucket using v2/list interface to stdout if output file not specified. Each row of data information is displayed in the following order by default:\n Key\tFileSize\tHash\tPutTime\tMimeType\tStorageType\tEndUser",
9899
Run: func(cmd *cobra.Command, args []string) {
99100
cfg.CmdCfg.CmdId = docs.ListBucket2Type
100101
if len(args) > 0 {
@@ -121,6 +122,9 @@ var listBucketCmd2Builder = func(cfg *iqshell.Config) *cobra.Command {
121122
cmd.Flags().BoolVarP(&info.Readable, "readable", "r", false, "present file size with human readable format")
122123
cmd.Flags().IntVarP(&info.Limit, "limit", "", -1, "max count of items to output")
123124

125+
cmd.Flags().StringVarP(&info.ShowFields, "show-fields", "", "", "The file attributes to be displayed on each line, separated by commas. Optional range: Key, Hash, FileSize, PutTime, MimeType, StorageType, EndUser.")
126+
cmd.Flags().StringVarP(&info.OutputFieldsSep, "output-fields-sep", "", data.DefaultLineSeparate, "Each line needs to display the delimiter of the file information.")
127+
124128
return cmd
125129
}
126130

‎cmd/rs.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,10 @@ var forbiddenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
3434
Long: "Forbidden object in qiniu bucket, when used with -r option, unforbidden the object",
3535
Run: func(cmd *cobra.Command, args []string) {
3636
cfg.CmdCfg.CmdId = docs.ForbiddenType
37-
if len(args) > 1 {
37+
if len(args) > 0 {
3838
info.Bucket = args[0]
39+
}
40+
if len(args) > 1 {
3941
info.Key = args[1]
4042
}
4143
operations.ForbiddenObject(cfg, info)
@@ -202,7 +204,7 @@ var changeTypeCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
202204
var cmd = &cobra.Command{
203205
Use: "chtype <Bucket> <Key> <FileType>",
204206
Short: "Change the file type of a file",
205-
Long: `Change the file type of a file, file type must be one of 0, 1, 2.
207+
Long: `Change the file type of a file, file type must be one of 0, 1, 2, 3.
206208
And 0 means STANDARD storage,
207209
while 1 means IA storage,
208210
while 2 means ARCHIVE storage.

‎cmd/rsbatch.go

+26
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,31 @@ var batchStatCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
3232
return cmd
3333
}
3434

35+
var batchForbiddenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
36+
var info = operations.BatchChangeStatusInfo{}
37+
var cmd = &cobra.Command{
38+
Use: "batchforbidden <Bucket> [-i <KeyListFile>] [-r]",
39+
Short: "Batch forbidden files in bucket",
40+
Run: func(cmd *cobra.Command, args []string) {
41+
cfg.CmdCfg.CmdId = docs.BatchForbiddenType
42+
info.BatchInfo.EnableStdin = true
43+
if len(args) > 0 {
44+
info.Bucket = args[0]
45+
}
46+
operations.BatchChangeStatus(cfg, info)
47+
},
48+
}
49+
setBatchCmdInputFileFlags(cmd, &info.BatchInfo)
50+
setBatchCmdWorkCountFlags(cmd, &info.BatchInfo)
51+
setBatchCmdSuccessExportFileFlags(cmd, &info.BatchInfo)
52+
setBatchCmdFailExportFileFlags(cmd, &info.BatchInfo)
53+
setBatchCmdForceFlags(cmd, &info.BatchInfo)
54+
setBatchCmdEnableRecordFlags(cmd, &info.BatchInfo)
55+
setBatchCmdRecordRedoWhileErrorFlags(cmd, &info.BatchInfo)
56+
cmd.Flags().BoolVarP(&info.UnForbidden, "reverse", "r", false, "unforbidden object in qiniu bucket")
57+
return cmd
58+
}
59+
3560
var batchDeleteCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
3661
var info = operations.BatchDeleteInfo{}
3762
var cmd = &cobra.Command{
@@ -283,6 +308,7 @@ func init() {
283308
func rsBatchCmdLoader(superCmd *cobra.Command, cfg *iqshell.Config) {
284309
superCmd.AddCommand(
285310
batchStatCmdBuilder(cfg),
311+
batchForbiddenCmdBuilder(cfg),
286312
batchCopyCmdBuilder(cfg),
287313
batchMoveCmdBuilder(cfg),
288314
batchRenameCmdBuilder(cfg),

‎cmd/tools.go

+20
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,25 @@ var dirCacheCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
342342
return cmd
343343
}
344344

345+
var funcCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
346+
var info = operations.FuncCallInfo{}
347+
var cmd = &cobra.Command{
348+
Use: "func <ParamsJson> <FuncTemplate>",
349+
Short: "qshell custom function call",
350+
Run: func(cmd *cobra.Command, args []string) {
351+
cfg.CmdCfg.CmdId = docs.FuncType
352+
if len(args) > 0 {
353+
info.ParamsJson = args[0]
354+
}
355+
if len(args) > 1 {
356+
info.FuncTemplate = args[1]
357+
}
358+
operations.FuncCall(cfg, info)
359+
},
360+
}
361+
return cmd
362+
}
363+
345364
func init() {
346365
registerLoader(toolsCmdLoader)
347366
}
@@ -364,5 +383,6 @@ func toolsCmdLoader(superCmd *cobra.Command, cfg *iqshell.Config) {
364383
IpCmdBuilder(cfg),
365384
TokenCmdBuilder(cfg),
366385
dirCacheCmdBuilder(cfg),
386+
funcCmdBuilder(cfg),
367387
)
368388
}

‎cmd_test/async_fetch_test.go

+6-4
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,17 @@ func TestAsyncFetch(t *testing.T) {
3838
"-e", failLogPath,
3939
"-g", "1",
4040
"-c", "2")
41+
defer func() {
42+
test.RemoveFile(failLogPath)
43+
test.RemoveFile(successLogPath)
44+
}()
4145
if !test.IsFileHasContent(successLogPath) {
42-
t.Fail()
46+
t.Fatal("success log can't empty")
4347
}
44-
test.RemoveFile(successLogPath)
4548

4649
if !test.IsFileHasContent(failLogPath) {
47-
t.Fail()
50+
t.Fatal("fail log can't empty")
4851
}
49-
test.RemoveFile(failLogPath)
5052
}
5153

5254
func TestAsyncFetchNoBucket(t *testing.T) {

‎cmd_test/bucket_list_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func TestBucketList2(t *testing.T) {
6767
result, errs := test.RunCmdWithError("listbucket2", test.Bucket,
6868
"--prefix", "hello",
6969
"--readable",
70-
"--end", "2022-06-15-00-00-00")
70+
"--end", "2022-07-12-00-00-00")
7171
if len(errs) > 0 {
7272
t.Fatal("error:", errs)
7373
}

‎cmd_test/change_minetype_test.go

+13
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
package cmd
44

55
import (
6+
"fmt"
67
"github.com/qiniu/qshell/v2/cmd_test/test"
78
"path/filepath"
89
"strings"
@@ -132,9 +133,15 @@ func TestBatchChangeMimeTypeRecord(t *testing.T) {
132133
"-y",
133134
"-d")
134135
if !strings.Contains(result, "because have done and success") {
136+
fmt.Println("=========================== result start ===========================")
137+
fmt.Println(result)
138+
fmt.Println("=========================== result end ===========================")
135139
t.Fatal("batch result: should skip success work")
136140
}
137141
if strings.Contains(result, "work redo") {
142+
fmt.Println("=========================== result start ===========================")
143+
fmt.Println(result)
144+
fmt.Println("=========================== result end ===========================")
138145
t.Fatal("batch result: shouldn't redo because not set --record-redo-while-error")
139146
}
140147

@@ -146,9 +153,15 @@ func TestBatchChangeMimeTypeRecord(t *testing.T) {
146153
"-y",
147154
"-d")
148155
if !strings.Contains(result, "because have done and success") {
156+
fmt.Println("=========================== result start ===========================")
157+
fmt.Println(result)
158+
fmt.Println("=========================== result end ===========================")
149159
t.Fatal("batch result: should skip success work")
150160
}
151161
if !strings.Contains(result, "work redo") {
162+
fmt.Println("=========================== result start ===========================")
163+
fmt.Println(result)
164+
fmt.Println("=========================== result end ===========================")
152165
t.Fatal("batch result: shouldn redo because set --record-redo-while-error")
153166
}
154167
}

‎cmd_test/change_type_test.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
package cmd
44

55
import (
6+
"fmt"
67
"github.com/qiniu/qshell/v2/cmd_test/test"
78
"path/filepath"
89
"strings"
@@ -151,10 +152,16 @@ func TestBatchChangeTypeRecord(t *testing.T) {
151152
"--worker", "4",
152153
"-y",
153154
"-d")
154-
if !strings.Contains(result, "because have done and success") {
155+
if !strings.Contains(result, "because have done and") {
156+
fmt.Println("=========================== result start ===========================")
157+
fmt.Println(result)
158+
fmt.Println("=========================== result end ===========================")
155159
t.Fatal("batch result: should skip success work")
156160
}
157161
if strings.Contains(result, "work redo") {
162+
fmt.Println("=========================== result start ===========================")
163+
fmt.Println(result)
164+
fmt.Println("=========================== result end ===========================")
158165
t.Fatal("batch result: shouldn't redo because not set --record-redo-while-error")
159166
}
160167

@@ -169,6 +176,9 @@ func TestBatchChangeTypeRecord(t *testing.T) {
169176
t.Fatal("batch result: should skip success work")
170177
}
171178
if !strings.Contains(result, "work redo") {
179+
fmt.Println("=========================== result start ===========================")
180+
fmt.Println(result)
181+
fmt.Println("=========================== result end ===========================")
172182
t.Fatal("batch result: should redo because set --record-redo-while-error")
173183
}
174184

‎cmd_test/copy_test.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
package cmd
44

55
import (
6+
"fmt"
67
"github.com/qiniu/qshell/v2/cmd_test/test"
78
"path/filepath"
89
"strings"
@@ -137,9 +138,15 @@ func TestBatchCopyRecord(t *testing.T) {
137138
"-w",
138139
"-d")
139140
if !strings.Contains(result, "because have done and success") {
141+
fmt.Println("=========================== result start ===========================")
142+
fmt.Println(result)
143+
fmt.Println("=========================== result end ===========================")
140144
t.Fatal("batch result: should skip success work")
141145
}
142146
if strings.Contains(result, "work redo") {
147+
fmt.Println("=========================== result start ===========================")
148+
fmt.Println(result)
149+
fmt.Println("=========================== result end ===========================")
143150
t.Fatal("batch result: shouldn't redo because not set --record-redo-while-error")
144151
}
145152

@@ -155,7 +162,10 @@ func TestBatchCopyRecord(t *testing.T) {
155162
t.Fatal("batch result: should skip success work")
156163
}
157164
if !strings.Contains(result, "work redo") {
158-
t.Fatal("batch result: shouldn redo because set --record-redo-while-error")
165+
fmt.Println("=========================== result start ===========================")
166+
fmt.Println(result)
167+
fmt.Println("=========================== result end ===========================")
168+
t.Fatal("batch result: should redo because set --record-redo-while-error")
159169
}
160170
}
161171

‎cmd_test/delete_test.go

+7
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
package cmd
44

55
import (
6+
"fmt"
67
"github.com/qiniu/qshell/v2/cmd_test/test"
78
"path/filepath"
89
"strings"
@@ -139,9 +140,15 @@ func TestBatchDeleteRecord(t *testing.T) {
139140
"-y",
140141
"-d")
141142
if !strings.Contains(result, "because have done and success") {
143+
fmt.Println("=========================== result start ===========================")
144+
fmt.Println(result)
145+
fmt.Println("=========================== result end ===========================")
142146
t.Fatal("batch result: should skip success work")
143147
}
144148
if !strings.Contains(result, "work redo") {
149+
fmt.Println("=========================== result start ===========================")
150+
fmt.Println(result)
151+
fmt.Println("=========================== result end ===========================")
145152
t.Fatal("batch result: shouldn redo because set --record-redo-while-error")
146153
}
147154
}

‎cmd_test/download_test.go

+12-11
Original file line numberDiff line numberDiff line change
@@ -62,17 +62,18 @@ func TestDownloadWithKeyFile(t *testing.T) {
6262
func TestDownloadFromBucket(t *testing.T) {
6363
cfg := &DownloadCfg{
6464
DownloadCfg: operations.DownloadCfg{
65-
ThreadCount: 4,
66-
KeyFile: "",
67-
Bucket: test.Bucket,
68-
Prefix: "hello3,hello5,hello7",
69-
Suffixes: "",
70-
IoHost: test.BucketDomain,
71-
Public: true,
72-
CheckHash: true,
73-
Referer: "",
74-
CdnDomain: "",
75-
RecordRoot: "",
65+
ThreadCount: 4,
66+
KeyFile: "",
67+
SavePathHandler: "{{pathJoin .DestDir (replace \"hello\" \"lala\" .Key)}}",
68+
Bucket: test.Bucket,
69+
Prefix: "hello3,hello5,hello7",
70+
Suffixes: "",
71+
IoHost: test.BucketDomain,
72+
Public: true,
73+
CheckHash: true,
74+
Referer: "",
75+
CdnDomain: "",
76+
RecordRoot: "",
7677
},
7778
}
7879
path, err := createDownloadConfigFile(cfg)

‎cmd_test/fop_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ func TestFop(t *testing.T) {
1919

2020
result = strings.ReplaceAll(result, "\n", "")
2121
result, errs = test.RunCmdWithError("prefop", result)
22-
if len(errs) > 0 {
22+
if len(errs) > 0 && !strings.Contains(errs, "not_found") {
2323
t.Fail()
2424
}
2525
}

0 commit comments

Comments
 (0)