diff --git a/api/types/metric.go b/api/types/metric.go index 69aab68..d6eb2c1 100644 --- a/api/types/metric.go +++ b/api/types/metric.go @@ -10,10 +10,10 @@ type ResponseStats struct { Failures int // Duration means the time of benchmark. Duration time.Duration - // Latencies represents the latency distribution in seconds. + // PercentileLatencies represents the latency distribution in seconds. // // NOTE: The key represents quantile. - Latencies map[float64]float64 + PercentileLatencies map[float64]float64 // TODO: // 1. Support total read/upload bytes // 2. Support failures partitioned by http code and verb diff --git a/cmd/kperf/commands/root.go b/cmd/kperf/commands/root.go index 6bac055..144f08d 100644 --- a/cmd/kperf/commands/root.go +++ b/cmd/kperf/commands/root.go @@ -3,6 +3,7 @@ package commands import ( "github.com/Azure/kperf/cmd/kperf/commands/multirunners" "github.com/Azure/kperf/cmd/kperf/commands/runner" + "github.com/Azure/kperf/cmd/kperf/commands/virtualcluster" "github.com/urfave/cli" ) @@ -15,6 +16,7 @@ func App() *cli.App { Commands: []cli.Command{ runner.Command, multirunners.Command, + virtualcluster.Command, }, } } diff --git a/cmd/kperf/commands/runner/runner.go b/cmd/kperf/commands/runner/runner.go index 496fbbd..00f741d 100644 --- a/cmd/kperf/commands/runner/runner.go +++ b/cmd/kperf/commands/runner/runner.go @@ -30,6 +30,11 @@ var runCommand = cli.Command{ Name: "kubeconfig", Usage: "Path to the kubeconfig file", }, + cli.IntFlag{ + Name: "client", + Usage: "Total number of HTTP clients", + Value: 1, + }, cli.StringFlag{ Name: "config", Usage: "Path to the configuration file", @@ -40,6 +45,11 @@ var runCommand = cli.Command{ Usage: "Total number of connections. It can override corresponding value defined by --config", Value: 1, }, + cli.StringFlag{ + Name: "content-type", + Usage: "Content type (json or protobuf)", + Value: "json", + }, cli.IntFlag{ Name: "rate", Usage: "Maximum requests per second (Zero means no limitation). It can override corresponding value defined by --config", @@ -60,17 +70,20 @@ var runCommand = cli.Command{ return err } + // Get the content type from the command-line flag + contentType := cliCtx.String("content-type") kubeCfgPath := cliCtx.String("kubeconfig") userAgent := cliCtx.String("user-agent") conns := profileCfg.Spec.Conns + client := profileCfg.Spec.Conns rate := profileCfg.Spec.Rate - restClis, err := request.NewClients(kubeCfgPath, conns, userAgent, rate) + restClis, err := request.NewClients(kubeCfgPath, conns, userAgent, rate, contentType) if err != nil { return err } + stats, err := request.Schedule(context.TODO(), client, &profileCfg.Spec, restClis) - stats, err := request.Schedule(context.TODO(), &profileCfg.Spec, restClis) if err != nil { return err } @@ -95,11 +108,15 @@ func loadConfig(cliCtx *cli.Context) (*types.LoadProfile, error) { } // override value by flags - // - // TODO(weifu): do not override if flag is not set - profileCfg.Spec.Rate = cliCtx.Int("rate") - profileCfg.Spec.Conns = cliCtx.Int("conns") - profileCfg.Spec.Total = cliCtx.Int("total") + if v := "rate"; cliCtx.IsSet(v) { + profileCfg.Spec.Rate = cliCtx.Int(v) + } + if v := "conns"; cliCtx.IsSet(v) || profileCfg.Spec.Conns == 0 { + profileCfg.Spec.Conns = cliCtx.Int(v) + } + if v := "total"; cliCtx.IsSet(v) || profileCfg.Spec.Total == 0 { + profileCfg.Spec.Total = cliCtx.Int(v) + } if err := profileCfg.Validate(); err != nil { return nil, err @@ -116,13 +133,13 @@ func printResponseStats(stats *types.ResponseStats) { fmt.Printf(" Requests/sec: %.2f\n", float64(stats.Total)/stats.Duration.Seconds()) fmt.Println(" Latency Distribution:") - keys := make([]float64, 0, len(stats.Latencies)) - for q := range stats.Latencies { + keys := make([]float64, 0, len(stats.PercentileLatencies)) + for q := range stats.PercentileLatencies { keys = append(keys, q) } sort.Float64s(keys) for _, q := range keys { - fmt.Printf(" [%.2f] %.3fs\n", q, stats.Latencies[q]) + fmt.Printf(" [%.2f] %.3fs\n", q/100.0, stats.PercentileLatencies[q]) } } diff --git a/go.mod b/go.mod index 0326c7d..d414d58 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/Azure/kperf go 1.20 require ( - github.com/prometheus/client_golang v1.17.0 github.com/stretchr/testify v1.8.4 github.com/urfave/cli v1.22.14 golang.org/x/time v0.3.0 @@ -14,8 +13,6 @@ require ( ) require ( - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.2.4 // indirect @@ -24,13 +21,9 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/net v0.17.0 // indirect diff --git a/go.sum b/go.sum index 2ff18d9..e21baab 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,4 @@ github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -17,7 +13,6 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -39,8 +34,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -49,14 +42,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -88,7 +73,6 @@ golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/metrics/request.go b/metrics/request.go index cd8840a..374c929 100644 --- a/metrics/request.go +++ b/metrics/request.go @@ -1,10 +1,11 @@ package metrics import ( - "fmt" + "container/list" + "math" + "sort" + "sync" "sync/atomic" - - "github.com/prometheus/client_golang/prometheus" ) // ResponseMetric is a measurement related to http response. @@ -14,30 +15,26 @@ type ResponseMetric interface { // ObserveFailure observes failure response. ObserveFailure() // Gather returns the summary. - Gather() (latencies map[float64]float64, failure int, _ error) + Gather() (latencies []float64, percentileLatencies map[float64]float64, failure int) } type responseMetricImpl struct { - latencySeconds *prometheus.SummaryVec - failureCount int64 + mu sync.Mutex + failureCount int64 + latencies *list.List } func NewResponseMetric() ResponseMetric { return &responseMetricImpl{ - latencySeconds: prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "request", - Name: "request_latency_seconds", - Objectives: map[float64]float64{0: 0, 0.5: 0, 0.9: 0, 0.95: 0, 0.99: 0, 1: 0}, - }, - []string{}, - ), + latencies: list.New(), } } // ObserveLatency implements ResponseMetric. func (m *responseMetricImpl) ObserveLatency(seconds float64) { - m.latencySeconds.WithLabelValues().Observe(seconds) + m.mu.Lock() + defer m.mu.Unlock() + m.latencies.PushBack(seconds) } // ObserveFailure implements ResponseMetric. @@ -46,19 +43,39 @@ func (m *responseMetricImpl) ObserveFailure() { } // Gather implements ResponseMetric. -func (m *responseMetricImpl) Gather() (map[float64]float64, int, error) { - reg := prometheus.NewRegistry() - reg.MustRegister(m.latencySeconds) +func (m *responseMetricImpl) Gather() ([]float64, map[float64]float64, int) { + latencies := m.dumpLatencies() + + return latencies, buildPercentileLatencies(latencies), int(atomic.LoadInt64(&m.failureCount)) +} - metricFamilies, err := reg.Gather() - if err != nil { - return nil, 0, fmt.Errorf("failed to gather from local registry: %w", err) +func (m *responseMetricImpl) dumpLatencies() []float64 { + m.mu.Lock() + defer m.mu.Unlock() + res := make([]float64, 0, m.latencies.Len()) + for e := m.latencies.Front(); e != nil; e = e.Next() { + res = append(res, e.Value.(float64)) } + return res +} - latencies := map[float64]float64{} - for _, q := range metricFamilies[0].GetMetric()[0].GetSummary().GetQuantile() { - latencies[q.GetQuantile()] = q.GetValue() +var percentiles = []float64{0, 50, 90, 95, 99, 100} + +func buildPercentileLatencies(latencies []float64) map[float64]float64 { + if len(latencies) == 0 { + return nil } - return latencies, int(atomic.LoadInt64(&m.failureCount)), nil + res := make(map[float64]float64, len(percentiles)) + + n := len(latencies) + sort.Float64s(latencies) + for _, p := range percentiles { + idx := int(math.Ceil(float64(n) * p / 100)) + if idx > 0 { + idx-- + } + res[p] = latencies[idx] + } + return res } diff --git a/metrics/request_test.go b/metrics/request_test.go new file mode 100644 index 0000000..2cd8e15 --- /dev/null +++ b/metrics/request_test.go @@ -0,0 +1,48 @@ +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildPercentileLatencies(t *testing.T) { + ls := make([]float64, 100) + ls[0] = 50 + ls[1] = 49 + ls[2] = 1 + res := buildPercentileLatencies(ls) + assert.Equal(t, float64(0), res[0]) + assert.Equal(t, float64(0), res[50]) + assert.Equal(t, float64(0), res[90]) + assert.Equal(t, float64(0), res[95]) + assert.Equal(t, float64(49), res[99]) + assert.Equal(t, float64(50), res[100]) + + ls = make([]float64, 1000) + ls[0] = 50 + ls[1] = 49 + ls[2] = -1 + res = buildPercentileLatencies(ls) + assert.Equal(t, float64(-1), res[0]) + assert.Equal(t, float64(0), res[50]) + assert.Equal(t, float64(0), res[90]) + assert.Equal(t, float64(0), res[95]) + assert.Equal(t, float64(0), res[99]) + assert.Equal(t, float64(50), res[100]) +} + +func TestResponseMetric(t *testing.T) { + c := NewResponseMetric() + for i := 100; i > 0; i-- { + c.ObserveLatency(float64(i)) + } + + _, res, _ := c.Gather() + assert.Equal(t, float64(1), res[0]) + assert.Equal(t, float64(50), res[50]) + assert.Equal(t, float64(90), res[90]) + assert.Equal(t, float64(95), res[95]) + assert.Equal(t, float64(99), res[99]) + assert.Equal(t, float64(100), res[100]) +} diff --git a/request/client.go b/request/client.go index 45b1c73..cab0618 100644 --- a/request/client.go +++ b/request/client.go @@ -1,6 +1,7 @@ package request import ( + "fmt" "math" "k8s.io/client-go/rest" @@ -15,8 +16,9 @@ import ( // 1. Is it possible to build one http2 client with multiple connections? // 2. How to monitor HTTP2 GOAWAY frame? // 3. Support Protobuf as accepted content -func NewClients(kubeCfgPath string, num int, userAgent string, qps int) ([]rest.Interface, error) { +func NewClients(kubeCfgPath string, ConnsNum int, userAgent string, qps int, contentType string) ([]rest.Interface, error) { restCfg, err := clientcmd.BuildConfigFromFlags("", kubeCfgPath) + if err != nil { return nil, err } @@ -24,20 +26,33 @@ func NewClients(kubeCfgPath string, num int, userAgent string, qps int) ([]rest. if qps == 0 { qps = math.MaxInt32 } + restCfg.QPS = float32(qps) restCfg.NegotiatedSerializer = scheme.Codecs.WithoutConversion() restCfg.UserAgent = userAgent + if restCfg.UserAgent == "" { restCfg.UserAgent = rest.DefaultKubernetesUserAgent() } - restClients := make([]rest.Interface, 0, num) - for i := 0; i < num; i++ { + // Set the content type + switch contentType { + case "json": + restCfg.ContentType = "application/json" + case "protobuf": + restCfg.ContentType = "application/vnd.kubernetes.protobuf" + default: + return nil, fmt.Errorf("invalid content type: %s", contentType) + } + + restClients := make([]rest.Interface, 0, ConnsNum) + for i := 0; i < ConnsNum; i++ { cfgShallowCopy := *restCfg restCli, err := rest.UnversionedRESTClientFor(&cfgShallowCopy) if err != nil { + fmt.Printf("Failed to create rest client: %v\n", err) return nil, err } restClients = append(restClients, restCli) diff --git a/request/schedule.go b/request/schedule.go index f2a216e..c94e307 100644 --- a/request/schedule.go +++ b/request/schedule.go @@ -17,7 +17,7 @@ import ( const defaultTimeout = 60 * time.Second // Schedule files requests to apiserver based on LoadProfileSpec. -func Schedule(ctx context.Context, spec *types.LoadProfileSpec, restCli []rest.Interface) (*types.ResponseStats, error) { +func Schedule(ctx context.Context, clientNum int, spec *types.LoadProfileSpec, restCli []rest.Interface) (*types.ResponseStats, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -36,10 +36,11 @@ func Schedule(ctx context.Context, spec *types.LoadProfileSpec, restCli []rest.I var wg sync.WaitGroup respMetric := metrics.NewResponseMetric() - for _, cli := range restCli { - cli := cli + for i := 0; i < clientNum; i++ { + //reuse connection if client > conns + cli := restCli[i%len(restCli)] wg.Add(1) - go func() { + go func(cli rest.Interface) { defer wg.Done() for builder := range reqBuilderCh { @@ -69,7 +70,7 @@ func Schedule(ctx context.Context, spec *types.LoadProfileSpec, restCli []rest.I } }() } - }() + }(cli) } start := time.Now() @@ -80,14 +81,11 @@ func Schedule(ctx context.Context, spec *types.LoadProfileSpec, restCli []rest.I totalDuration := time.Since(start) - latencies, failures, err := respMetric.Gather() - if err != nil { - return nil, err - } + _, percentileLatencies, failures := respMetric.Gather() return &types.ResponseStats{ - Total: spec.Total, - Failures: failures, - Duration: totalDuration, - Latencies: latencies, + Total: spec.Total, + Failures: failures, + Duration: totalDuration, + PercentileLatencies: percentileLatencies, }, nil }