Skip to content

storagev2包中的uptoken.NewSigner构造出的uptoken.Provider存在线程不安全的问题 #169

@zhangzqs

Description

@zhangzqs
[test] ==================
[test] WARNING: DATA RACE
[test] Read at 0x00c0002098a8 by goroutine 14:
[test]   github.com/qiniu/go-sdk/v7/storagev2/uptoken.(*signer).onceGetUpToken()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uptoken/uploadtoken.go:89 +0x44
[test]   github.com/qiniu/go-sdk/v7/storagev2/uptoken.(*signer).GetUpToken()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uptoken/uploadtoken.go:72 +0x3a
[test]   github.com/qiniu/go-sdk/v7/storagev2/apis.(*innerPostObjectRequest).build()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/apis/api_post_object.go:40 +0x11b
[test]   github.com/qiniu/go-sdk/v7/storagev2/apis.(*Storage).PostObject()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/apis/api_post_object.go:89 +0x197
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.uploadToRegion()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:160 +0x3ba
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.upload.func1()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:132 +0x10b
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.forEachRegion()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:453 +0x1ed
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.upload()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:131 +0x1ba
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.UploadFile()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:87 +0x3fd
[test]   io.Copy()
[test]       io/io.go:388 +0x1b3
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.crc32FromReadSeeker()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:339 +0x102
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.UploadFile()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:82 +0x34d
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*formUploader).UploadFile()
[test]       <autogenerated>:1 +0xbb
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadFile()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/upload_manager.go:215 +0x35a
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory.func3.1()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/upload_manager.go:162 +0x4fa
[test]   golang.org/x/sync/errgroup.(*Group).Go.func1()
[test]       golang.org/x/sync@v0.17.0/errgroup/errgroup.go:93 +0x86
[test] 
[test] Previous write at 0x00c0002098a8 by goroutine 13:
[test]   github.com/qiniu/go-sdk/v7/storagev2/uptoken.(*signer).onceGetUpToken()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uptoken/uploadtoken.go:100 +0xc4
[test]   github.com/qiniu/go-sdk/v7/storagev2/uptoken.(*signer).GetUpToken()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uptoken/uploadtoken.go:72 +0x3a
[test]   github.com/qiniu/go-sdk/v7/storagev2/apis.(*innerPostObjectRequest).build()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/apis/api_post_object.go:40 +0x11b
[test]   github.com/qiniu/go-sdk/v7/storagev2/apis.(*Storage).PostObject()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/apis/api_post_object.go:89 +0x197
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.uploadToRegion()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:160 +0x3ba
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.upload.func1()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:132 +0x10b
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.forEachRegion()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:453 +0x1ed
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.upload()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:131 +0x1ba
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.UploadReader()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:127 +0x4af
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*formUploader).UploadReader()
[test]       <autogenerated>:1 +0xbb
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadReader()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/upload_manager.go:248 +0x654
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory.func3.1()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/upload_manager.go:180 +0x988
[test]   golang.org/x/sync/errgroup.(*Group).Go.func1()
[test]       golang.org/x/sync@v0.17.0/errgroup/errgroup.go:93 +0x86
[test] 
[test] Goroutine 14 (running) created at:
[test]   golang.org/x/sync/errgroup.(*Group).Go()
[test]       golang.org/x/sync@v0.17.0/errgroup/errgroup.go:78 +0x11c
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory.func3()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/upload_manager.go:141 +0x32d
[test]   path/filepath.walk()
[test]       path/filepath/path.go:345 +0x14d
[test]   path/filepath.walk()
[test]       path/filepath/path.go:369 +0x379
[test]   path/filepath.Walk()
[test]       path/filepath/path.go:427 +0x7b
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/upload_manager.go:137 +0x5b0
[test]   github.com/qbox/las/internal/rfsjob/service.(*KodoUploader).Upload()
[test]       github.com/qbox/las/internal/rfsjob/service/kodo_uploader.go:151 +0x987
[test]   github.com/qbox/las/internal/rfsjob/service.TestKodoUploader_Retry()
[test]       github.com/qbox/las/internal/rfsjob/service/kodo_uploader_test.go:205 +0xaaa
[test]   testing.tRunner()
[test]       testing/testing.go:1934 +0x21c
[test]   testing.(*T).Run.gowrap1()
[test]       testing/testing.go:1997 +0x44
[test] 
[test] Goroutine 13 (running) created at:
[test]   golang.org/x/sync/errgroup.(*Group).Go()
[test]       golang.org/x/sync@v0.17.0/errgroup/errgroup.go:78 +0x11c
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory.func3()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/upload_manager.go:141 +0x32d
[test]   path/filepath.walk()
[test]       path/filepath/path.go:349 +0xcd
[test]   path/filepath.Walk()
[test]       path/filepath/path.go:427 +0x7b
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/upload_manager.go:137 +0x5b0
[test]   github.com/qbox/las/internal/rfsjob/service.(*KodoUploader).Upload()
[test]       github.com/qbox/las/internal/rfsjob/service/kodo_uploader.go:151 +0x987
[test]   github.com/qbox/las/internal/rfsjob/service.TestKodoUploader_Retry()
[test]       github.com/qbox/las/internal/rfsjob/service/kodo_uploader_test.go:205 +0xaaa
[test]   testing.tRunner()
[test]       testing/testing.go:1934 +0x21c
[test]   testing.(*T).Run.gowrap1()
[test]       testing/testing.go:1997 +0x44
[test] ==================
[test] ==================
[test] WARNING: DATA RACE
[test] Write at 0x00c0001487a0 by goroutine 20:
[test]   github.com/qiniu/go-sdk/v7/internal/uplog.(*RequestUplog).Intercept.func7()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/uplog/request_uplog.go:154 +0x104
[test]   net.(*sysDialer).dialSingle.func1()
[test]       net/dial.go:711 +0xda
[test]   runtime.deferreturn()
[test]       runtime/panic.go:589 +0x5d
[test]   net.(*sysDialer).dialSerial()
[test]       net/dial.go:686 +0x288
[test]   net.(*sysDialer).dialParallel()
[test]       net/dial.go:587 +0x525
[test]   net.(*Dialer).DialContext()
[test]       net/dial.go:578 +0xbbb
[test]   github.com/qiniu/go-sdk/v7/internal/dialer.dialContextSync()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/dialer/dialer.go:84 +0x1a8
[test]   github.com/qiniu/go-sdk/v7/internal/dialer.dialContextAsync.func1()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/dialer/dialer.go:90 +0x16c
[test] 
[test] Previous read at 0x00c0001487a0 by goroutine 13:
[test]   reflect.Value.Uint()
[test]       reflect/value.go:2528 +0x333
[test]   reflect.Value.IsZero()
[test]       reflect/value.go:1643 +0x260
[test]   encoding/json.isEmptyValue()
[test]       encoding/json/encode.go:361 +0x85
[test]   encoding/json.structEncoder.encode()
[test]       encoding/json/encode.go:746 +0x175
[test]   encoding/json.structEncoder.encode-fm()
[test]       <autogenerated>:1 +0xe4
[test]   encoding/json.ptrEncoder.encode()
[test]       encoding/json/encode.go:930 +0x3c2
[test]   encoding/json.ptrEncoder.encode-fm()
[test]       <autogenerated>:1 +0x84
[test]   encoding/json.(*encodeState).reflectValue()
[test]       encoding/json/encode.go:367 +0x83
[test]   encoding/json.(*encodeState).marshal()
[test]       encoding/json/encode.go:343 +0xdb
[test]   encoding/json.Marshal()
[test]       encoding/json/encode.go:209 +0x11e
[test]   github.com/qiniu/go-sdk/v7/internal/uplog.(*RequestUplog).Intercept()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/uplog/request_uplog.go:218 +0x1c24
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*client).Do.func2()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/clientv2/client.go:92 +0x54
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*simpleRetryInterceptor).callHandler()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/clientv2/interceptor_retry_simple.go:171 +0x87
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*simpleRetryInterceptor).Intercept()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/clientv2/interceptor_retry_simple.go:111 +0x516
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*client).Do.func2()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/clientv2/client.go:92 +0x54
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*hostsRetryInterceptor).Intercept()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/clientv2/interceptor_retry_hosts.go:84 +0x22d
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*client).Do.func2()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/clientv2/client.go:92 +0x54
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*client).Do()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/clientv2/client.go:96 +0x1c1
[test]   github.com/qiniu/go-sdk/v7/storagev2/http_client.(*Client).Do()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/http_client/http_client.go:275 +0x4c1
[test]   github.com/qiniu/go-sdk/v7/storagev2/http_client.(*Client).DoAndAcceptJSON()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/http_client/http_client.go:280 +0x55
[test]   github.com/qiniu/go-sdk/v7/storagev2/apis.(*Storage).PostObject()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/apis/api_post_object.go:148 +0x11ca
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.uploadToRegion()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:160 +0x3ba
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.upload.func1()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:132 +0x10b
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.forEachRegion()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:453 +0x1ed
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.upload()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:131 +0x1ba
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.UploadReader()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/uploaders.go:127 +0x4af
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*formUploader).UploadReader()
[test]       <autogenerated>:1 +0xbb
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadReader()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/upload_manager.go:248 +0x654
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory.func3.1()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/upload_manager.go:180 +0x988
[test]   golang.org/x/sync/errgroup.(*Group).Go.func1()
[test]       golang.org/x/sync@v0.17.0/errgroup/errgroup.go:93 +0x86
[test] 
[test] Goroutine 20 (running) created at:
[test]   github.com/qiniu/go-sdk/v7/internal/dialer.dialContextAsync()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/dialer/dialer.go:88 +0x22d
[test]   github.com/qiniu/go-sdk/v7/internal/dialer.DialContext()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/internal/dialer/dialer.go:48 +0x476
[test]   github.com/qiniu/go-sdk/v7/client.defaultDialFunc()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/client/dialer.go:36 +0x309
[test]   net/http.(*Transport).dial()
[test]       net/http/transport.go:1278 +0xe6
[test]   net/http.(*Transport).dialConn()
[test]       net/http/transport.go:1783 +0xd64
[test]   net/http.(*Transport).dialConnFor()
[test]       net/http/transport.go:1618 +0x10f
[test]   net/http.(*Transport).startDialConnForLocked.func1()
[test]       net/http/transport.go:1600 +0x3c
[test] 
[test] Goroutine 13 (running) created at:
[test]   golang.org/x/sync/errgroup.(*Group).Go()
[test]       golang.org/x/sync@v0.17.0/errgroup/errgroup.go:78 +0x11c
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory.func3()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/upload_manager.go:141 +0x32d
[test]   path/filepath.walk()
[test]       path/filepath/path.go:349 +0xcd
[test]   path/filepath.Walk()
[test]       path/filepath/path.go:427 +0x7b
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory()
[test]       github.com/qiniu/go-sdk/v7@v7.25.4/storagev2/uploader/upload_manager.go:137 +0x5b0
[test]   github.com/qbox/las/internal/rfsjob/service.(*KodoUploader).Upload()
[test]       github.com/qbox/las/internal/rfsjob/service/kodo_uploader.go:151 +0x987
[test]   github.com/qbox/las/internal/rfsjob/service.TestKodoUploader_Retry()
[test]       github.com/qbox/las/internal/rfsjob/service/kodo_uploader_test.go:205 +0xaaa
[test]   testing.tRunner()
[test]       testing/testing.go:1934 +0x21c
[test]   testing.(*T).Run.gowrap1()
[test]       testing/testing.go:1997 +0x44
[test] ==================
[test] --- FAIL: TestKodoUploader_Retry (0.18s)
****************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
[test]     writer.go:27: {"level":"debug","time":1762399299494844515,"message":"Using accessKey/secretKey for Kodo upload"}
[test]     writer.go:27: {"level":"warn","url":"http://127.0.0.1:44741/","attempts":0,"backoff_delay":"67ms","time":1762399299508193116,"message":"kodo upload retry backoff"}
[test]     writer.go:27: {"level":"warn","url":"http://127.0.0.1:44741/","attempts":1,"backoff_delay":"89ms","time":1762399299577824496,"message":"kodo upload retry backoff"}
[test]     testing.go:1617: race detected during execution of test
[test] FAIL
[test] coverage: 8.2% of statements
[test] FAIL	github.com/qbox/las/internal/rfsjob/service	0.290s

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions