Skip to content

Commit 86d92a4

Browse files
vadimartynovvadimartynov
authored
Added tls for http clients (seaweedfs#5766)
* Added global http client * Added Do func for global http client * Changed the code to use the global http client * Fix http client in volume uploader * Fixed pkg name * Fixed http util funcs * Fixed http client for bench_filer_upload * Fixed http client for stress_filer_upload * Fixed http client for filer_server_handlers_proxy * Fixed http client for command_fs_merge_volumes * Fixed http client for command_fs_merge_volumes and command_volume_fsck * Fixed http client for s3api_server * Added init global client for main funcs * Rename global_client to client * Changed: - fixed NewHttpClient; - added CheckIsHttpsClientEnabled func - updated security.toml in scaffold * Reduce the visibility of some functions in the util/http/client pkg * Added the loadSecurityConfig function * Use util.LoadSecurityConfiguration() in NewHttpClient func
1 parent c6dec11 commit 86d92a4

File tree

66 files changed

+646
-198
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+646
-198
lines changed

‎unmaintained/change_superblock/change_superblock.go

+3
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
1212
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
1313
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
14+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1415
)
1516

1617
var (
@@ -40,6 +41,8 @@ that has those volumes.
4041
*/
4142
func main() {
4243
flag.Parse()
44+
util_http.NewGlobalHttpClient()
45+
4346
fileName := strconv.Itoa(*fixVolumeId)
4447
if *fixVolumeCollection != "" {
4548
fileName = *fixVolumeCollection + "_" + fileName

‎unmaintained/diff_volume_servers/diff_volume_servers.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/seaweedfs/seaweedfs/weed/storage/types"
2121
"github.com/seaweedfs/seaweedfs/weed/util"
2222
"google.golang.org/grpc"
23+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
2324
)
2425

2526
var (
@@ -40,7 +41,8 @@ var (
4041
*/
4142
func main() {
4243
flag.Parse()
43-
44+
util_http.InitGlobalHttpClient()
45+
4446
util.LoadSecurityConfiguration()
4547
grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
4648

‎unmaintained/fix_dat/fix_dat.go

+3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
1515
"github.com/seaweedfs/seaweedfs/weed/storage/types"
1616
"github.com/seaweedfs/seaweedfs/weed/util"
17+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1718
)
1819

1920
var (
@@ -36,6 +37,8 @@ The .idx has all correct offsets.
3637
*/
3738
func main() {
3839
flag.Parse()
40+
util_http.InitGlobalHttpClient()
41+
3942
fileName := strconv.Itoa(*fixVolumeId)
4043
if *fixVolumeCollection != "" {
4144
fileName = *fixVolumeCollection + "_" + fileName

‎unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"strconv"
1313
"strings"
1414
"time"
15+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1516
)
1617

1718
var (
@@ -23,8 +24,8 @@ var (
2324
)
2425

2526
func main() {
26-
2727
flag.Parse()
28+
util_http.InitGlobalHttpClient()
2829

2930
if *isWrite {
3031
startGenerateMetadata()

‎unmaintained/remove_duplicate_fids/remove_duplicate_fids.go

+2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
1212
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
1313
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
14+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1415
)
1516

1617
var (
@@ -71,6 +72,7 @@ func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset in
7172

7273
func main() {
7374
flag.Parse()
75+
util_http.InitGlobalHttpClient()
7476

7577
vid := needle.VolumeId(*volumeId)
7678

‎unmaintained/repeated_vacuum/repeated_vacuum.go

+11-3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/seaweedfs/seaweedfs/weed/operation"
1515
"github.com/seaweedfs/seaweedfs/weed/security"
1616
"github.com/seaweedfs/seaweedfs/weed/util"
17+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1718
)
1819

1920
var (
@@ -25,6 +26,7 @@ var (
2526

2627
func main() {
2728
flag.Parse()
29+
util_http.InitGlobalHttpClient()
2830

2931
util.LoadSecurityConfiguration()
3032
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
@@ -34,7 +36,7 @@ func main() {
3436
go func() {
3537
for {
3638
println("vacuum threshold", *garbageThreshold)
37-
_, _, err := util.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=%f", pb.ServerAddress(*master).ToHttpAddress(), *garbageThreshold))
39+
_, _, err := util_http.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=%f", pb.ServerAddress(*master).ToHttpAddress(), *garbageThreshold))
3840
if err != nil {
3941
log.Fatalf("vacuum: %v", err)
4042
}
@@ -47,7 +49,7 @@ func main() {
4749

4850
assignResult, targetUrl := genFile(grpcDialOption, i)
4951

50-
util.Delete(targetUrl, string(assignResult.Auth))
52+
util_http.Delete(targetUrl, string(assignResult.Auth))
5153

5254
}
5355

@@ -76,7 +78,13 @@ func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, st
7678
PairMap: nil,
7779
Jwt: assignResult.Auth,
7880
}
79-
_, err = operation.UploadData(data, uploadOption)
81+
82+
uploader, err := operation.NewUploader()
83+
if err != nil {
84+
log.Fatalf("upload: %v", err)
85+
}
86+
87+
_, err = uploader.UploadData(data, uploadOption)
8088
if err != nil {
8189
log.Fatalf("upload: %v", err)
8290
}

‎unmaintained/s3/presigned_put/presigned_put.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@ import (
77
"github.com/aws/aws-sdk-go/aws"
88
"github.com/aws/aws-sdk-go/aws/session"
99
"github.com/aws/aws-sdk-go/service/s3"
10-
"github.com/seaweedfs/seaweedfs/weed/util"
1110
"net/http"
1211
"strings"
1312
"time"
13+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1414
)
1515

1616
// Downloads an item from an S3 Bucket in the region configured in the shared config
@@ -21,6 +21,8 @@ import (
2121
// For this exampl to work, the domainName is needd
2222
// weed s3 -domainName=localhost
2323
func main() {
24+
util_http.InitGlobalHttpClient()
25+
2426
h := md5.New()
2527
content := strings.NewReader(stringContent)
2628
content.WriteTo(h)
@@ -64,7 +66,7 @@ func main() {
6466
fmt.Printf("error put request: %v\n", err)
6567
return
6668
}
67-
defer util.CloseResponse(resp)
69+
defer util_http.CloseResponse(resp)
6870
fmt.Printf("response: %+v\n", resp)
6971
}
7072

‎unmaintained/see_dat/see_dat.go

+2
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/seaweedfs/seaweedfs/weed/storage"
1111
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
1212
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
13+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1314
)
1415

1516
var (
@@ -40,6 +41,7 @@ func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset in
4041

4142
func main() {
4243
flag.Parse()
44+
util_http.InitGlobalHttpClient()
4345

4446
vid := needle.VolumeId(*volumeId)
4547

‎unmaintained/see_idx/see_idx.go

+3
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/seaweedfs/seaweedfs/weed/glog"
1313
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
1414
"github.com/seaweedfs/seaweedfs/weed/storage/types"
15+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1516
)
1617

1718
var (
@@ -27,6 +28,8 @@ This is to see content in .idx files.
2728
*/
2829
func main() {
2930
flag.Parse()
31+
util_http.InitGlobalHttpClient()
32+
3033
fileName := strconv.Itoa(*fixVolumeId)
3134
if *fixVolumeCollection != "" {
3235
fileName = *fixVolumeCollection + "_" + fileName

‎unmaintained/see_log_entry/see_log_entry.go

+2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/seaweedfs/seaweedfs/weed/filer"
1313
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
1414
"github.com/seaweedfs/seaweedfs/weed/util"
15+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1516
)
1617

1718
var (
@@ -20,6 +21,7 @@ var (
2021

2122
func main() {
2223
flag.Parse()
24+
util_http.InitGlobalHttpClient()
2325

2426
dst, err := os.OpenFile(*logdataFile, os.O_RDONLY, 0644)
2527
if err != nil {

‎unmaintained/see_meta/see_meta.go

+2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
1313
"github.com/seaweedfs/seaweedfs/weed/util"
14+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1415
)
1516

1617
var (
@@ -19,6 +20,7 @@ var (
1920

2021
func main() {
2122
flag.Parse()
23+
util_http.InitGlobalHttpClient()
2224

2325
dst, err := os.OpenFile(*metaFile, os.O_RDONLY, 0644)
2426
if err != nil {

‎unmaintained/stream_read_volume/stream_read_volume.go

+2
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/seaweedfs/seaweedfs/weed/security"
1414
"github.com/seaweedfs/seaweedfs/weed/util"
1515
"google.golang.org/grpc"
16+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1617
)
1718

1819
var (
@@ -23,6 +24,7 @@ var (
2324

2425
func main() {
2526
flag.Parse()
27+
util_http.InitGlobalHttpClient()
2628

2729
util.LoadSecurityConfiguration()
2830
grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")

‎unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go

+8-8
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"strings"
1414
"sync"
1515
"time"
16+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1617
)
1718

1819
var (
@@ -30,8 +31,8 @@ type stat struct {
3031
}
3132

3233
func main() {
33-
3434
flag.Parse()
35+
util_http.InitGlobalHttpClient()
3536

3637
data := make([]byte, *size)
3738
println("data len", len(data))
@@ -43,16 +44,12 @@ func main() {
4344
go func(x int) {
4445
defer wg.Done()
4546

46-
client := &http.Client{Transport: &http.Transport{
47-
MaxIdleConns: 1024,
48-
MaxIdleConnsPerHost: 1024,
49-
}}
5047
r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(x)))
5148

5249
for t := 0; t < *times; t++ {
5350
for f := 0; f < *fileCount; f++ {
5451
fn := r.Intn(*fileCount)
55-
if size, err := uploadFileToFiler(client, data, fmt.Sprintf("file%04d", fn), *destination); err == nil {
52+
if size, err := uploadFileToFiler(data, fmt.Sprintf("file%04d", fn), *destination); err == nil {
5653
statsChan <- stat{
5754
size: size,
5855
}
@@ -93,7 +90,7 @@ func main() {
9390

9491
}
9592

96-
func uploadFileToFiler(client *http.Client, data []byte, filename, destination string) (size int64, err error) {
93+
func uploadFileToFiler(data []byte, filename, destination string) (size int64, err error) {
9794

9895
if !strings.HasSuffix(destination, "/") {
9996
destination = destination + "/"
@@ -116,10 +113,13 @@ func uploadFileToFiler(client *http.Client, data []byte, filename, destination s
116113
uri := destination + filename
117114

118115
request, err := http.NewRequest(http.MethodPost, uri, body)
116+
if err != nil {
117+
return 0, fmt.Errorf("http POST %s: %v", uri, err)
118+
}
119119
request.Header.Set("Content-Type", writer.FormDataContentType())
120120
// request.Close = true // can not use this, which do not reuse http connection, impacting filer->volume also.
121121

122-
resp, err := client.Do(request)
122+
resp, err := util_http.GetGlobalHttpClient().Do(request)
123123
if err != nil {
124124
return 0, fmt.Errorf("http POST %s: %v", uri, err)
125125
} else {

‎unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go

+9-6
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"strings"
1515
"sync"
1616
"time"
17+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1718
)
1819

1920
var (
@@ -30,8 +31,8 @@ type stat struct {
3031
}
3132

3233
func main() {
33-
3434
flag.Parse()
35+
util_http.InitGlobalHttpClient()
3536

3637
var fileNames []string
3738

@@ -51,16 +52,14 @@ func main() {
5152
for x := 0; x < *concurrency; x++ {
5253
wg.Add(1)
5354

54-
client := &http.Client{}
55-
5655
go func() {
5756
defer wg.Done()
5857
rand.Shuffle(len(fileNames), func(i, j int) {
5958
fileNames[i], fileNames[j] = fileNames[j], fileNames[i]
6059
})
6160
for t := 0; t < *times; t++ {
6261
for _, filename := range fileNames {
63-
if size, err := uploadFileToFiler(client, filename, *destination); err == nil {
62+
if size, err := uploadFileToFiler(filename, *destination); err == nil {
6463
statsChan <- stat{
6564
size: size,
6665
}
@@ -99,7 +98,7 @@ func main() {
9998

10099
}
101100

102-
func uploadFileToFiler(client *http.Client, filename, destination string) (size int64, err error) {
101+
func uploadFileToFiler(filename, destination string) (size int64, err error) {
103102
file, err := os.Open(filename)
104103
if err != nil {
105104
panic(err)
@@ -131,9 +130,13 @@ func uploadFileToFiler(client *http.Client, filename, destination string) (size
131130
uri := destination + file.Name()
132131

133132
request, err := http.NewRequest(http.MethodPost, uri, body)
133+
if err != nil {
134+
return 0, fmt.Errorf("http POST %s: %v", uri, err)
135+
}
136+
134137
request.Header.Set("Content-Type", writer.FormDataContentType())
135138

136-
resp, err := client.Do(request)
139+
resp, err := util_http.GetGlobalHttpClient().Do(request)
137140
if err != nil {
138141
return 0, fmt.Errorf("http POST %s: %v", uri, err)
139142
} else {

‎unmaintained/volume_tailer/volume_tailer.go

+2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
1313
util2 "github.com/seaweedfs/seaweedfs/weed/util"
1414
"golang.org/x/tools/godoc/util"
15+
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
1516
)
1617

1718
var (
@@ -24,6 +25,7 @@ var (
2425

2526
func main() {
2627
flag.Parse()
28+
util_http.InitGlobalHttpClient()
2729

2830
util2.LoadSecurityConfiguration()
2931
grpcDialOption := security.LoadClientTLS(util2.GetViper(), "grpc.client")

0 commit comments

Comments
 (0)