Update code to improve connection handling and support for protobuf as content type

api/types/metric.go

@@ -10,10 +10,10 @@ type ResponseStats struct {
 	Failures int
 	// Duration means the time of benchmark.
 	Duration time.Duration
-	// Latencies represents the latency distribution in seconds.
+	// PercentileLatencies represents the latency distribution in seconds.
 	//
 	// NOTE: The key represents quantile.
-	Latencies map[float64]float64
+	PercentileLatencies map[float64]float64
 	// TODO:
 	// 1. Support total read/upload bytes
 	// 2. Support failures partitioned by http code and verb

cmd/kperf/commands/root.go

@@ -3,6 +3,7 @@
 import (
 	"github.com/Azure/kperf/cmd/kperf/commands/multirunners"
 	"github.com/Azure/kperf/cmd/kperf/commands/runner"
+	"github.com/Azure/kperf/cmd/kperf/commands/virtualcluster"
 
 	"github.com/urfave/cli"
 )
@@ -15,6 +16,7 @@
 		Commands: []cli.Command{
 			runner.Command,
 			multirunners.Command,
+			virtualcluster.Command,
 		},
 	}
 }

cmd/kperf/commands/runner/runner.go

@@ -30,6 +30,11 @@ var runCommand = cli.Command{
 			Name:  "kubeconfig",
 			Usage: "Path to the kubeconfig file",
 		},
+		cli.IntFlag{
+			Name:  "client",
+			Usage: "Total number of HTTP clients",
+			Value: 1,
+		},
 		cli.StringFlag{
 			Name:     "config",
 			Usage:    "Path to the configuration file",
@@ -40,6 +45,11 @@ var runCommand = cli.Command{
 			Usage: "Total number of connections. It can override corresponding value defined by --config",
 			Value: 1,
 		},
+		cli.StringFlag{
+			Name:  "content-type",
+			Usage: "Content type (json or protobuf)",
+			Value: "json",
+		},
 		cli.IntFlag{
 			Name:  "rate",
 			Usage: "Maximum requests per second (Zero means no limitation). It can override corresponding value defined by --config",
@@ -60,17 +70,20 @@ var runCommand = cli.Command{
 			return err
 		}
 
+		// Get the content type from the command-line flag
+		contentType := cliCtx.String("content-type")
 		kubeCfgPath := cliCtx.String("kubeconfig")
 		userAgent := cliCtx.String("user-agent")
 
 		conns := profileCfg.Spec.Conns
+		client := profileCfg.Spec.Conns
 		rate := profileCfg.Spec.Rate
-		restClis, err := request.NewClients(kubeCfgPath, conns, userAgent, rate)
+		restClis, err := request.NewClients(kubeCfgPath, conns, userAgent, rate, contentType)
 		if err != nil {
 			return err
 		}
+		stats, err := request.Schedule(context.TODO(), client, &profileCfg.Spec, restClis)
 
-		stats, err := request.Schedule(context.TODO(), &profileCfg.Spec, restClis)
 		if err != nil {
 			return err
 		}
@@ -95,11 +108,15 @@ func loadConfig(cliCtx *cli.Context) (*types.LoadProfile, error) {
 	}
 
 	// override value by flags
-	//
-	// TODO(weifu): do not override if flag is not set
-	profileCfg.Spec.Rate = cliCtx.Int("rate")
-	profileCfg.Spec.Conns = cliCtx.Int("conns")
-	profileCfg.Spec.Total = cliCtx.Int("total")
+	if v := "rate"; cliCtx.IsSet(v) {
+		profileCfg.Spec.Rate = cliCtx.Int(v)
+	}
+	if v := "conns"; cliCtx.IsSet(v) || profileCfg.Spec.Conns == 0 {
+		profileCfg.Spec.Conns = cliCtx.Int(v)
+	}
+	if v := "total"; cliCtx.IsSet(v) || profileCfg.Spec.Total == 0 {
+		profileCfg.Spec.Total = cliCtx.Int(v)
+	}
 
 	if err := profileCfg.Validate(); err != nil {
 		return nil, err
@@ -116,13 +133,13 @@ func printResponseStats(stats *types.ResponseStats) {
 	fmt.Printf("  Requests/sec: %.2f\n", float64(stats.Total)/stats.Duration.Seconds())
 
 	fmt.Println("  Latency Distribution:")
-	keys := make([]float64, 0, len(stats.Latencies))
-	for q := range stats.Latencies {
+	keys := make([]float64, 0, len(stats.PercentileLatencies))
+	for q := range stats.PercentileLatencies {
 		keys = append(keys, q)
 	}
 	sort.Float64s(keys)
 
 	for _, q := range keys {
-		fmt.Printf("    [%.2f] %.3fs\n", q, stats.Latencies[q])
+		fmt.Printf("    [%.2f] %.3fs\n", q/100.0, stats.PercentileLatencies[q])
 	}
 }

go.mod

@@ -3,7 +3,6 @@ module github.com/Azure/kperf
 go 1.20
 
 require (
-	github.com/prometheus/client_golang v1.17.0
 	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli v1.22.14
 	golang.org/x/time v0.3.0
@@ -14,8 +13,6 @@ require (
 )
 
 require (
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
@@ -24,13 +21,9 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/net v0.17.0 // indirect

go.sum

@@ -1,8 +1,4 @@
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
-github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -17,7 +13,6 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv
 github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
@@ -39,8 +34,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -49,14 +42,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
-github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
-github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
-github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
-github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
-github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
-github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
-github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -88,7 +73,6 @@ golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
 golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

metrics/request.go

@@ -1,10 +1,11 @@
 package metrics
 
 import (
-	"fmt"
+	"container/list"
+	"math"
+	"sort"
+	"sync"
 	"sync/atomic"
-
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 // ResponseMetric is a measurement related to http response.
@@ -14,30 +15,26 @@ type ResponseMetric interface {
 	// ObserveFailure observes failure response.
 	ObserveFailure()
 	// Gather returns the summary.
-	Gather() (latencies map[float64]float64, failure int, _ error)
+	Gather() (latencies []float64, percentileLatencies map[float64]float64, failure int)
 }
 
 type responseMetricImpl struct {
-	latencySeconds *prometheus.SummaryVec
-	failureCount   int64
+	mu           sync.Mutex
+	failureCount int64
+	latencies    *list.List
 }
 
 func NewResponseMetric() ResponseMetric {
 	return &responseMetricImpl{
-		latencySeconds: prometheus.NewSummaryVec(
-			prometheus.SummaryOpts{
-				Namespace:  "request",
-				Name:       "request_latency_seconds",
-				Objectives: map[float64]float64{0: 0, 0.5: 0, 0.9: 0, 0.95: 0, 0.99: 0, 1: 0},
-			},
-			[]string{},
-		),
+		latencies: list.New(),
 	}
 }
 
 // ObserveLatency implements ResponseMetric.
 func (m *responseMetricImpl) ObserveLatency(seconds float64) {
-	m.latencySeconds.WithLabelValues().Observe(seconds)
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.latencies.PushBack(seconds)
 }
 
 // ObserveFailure implements ResponseMetric.
@@ -46,19 +43,39 @@ func (m *responseMetricImpl) ObserveFailure() {
 }
 
 // Gather implements ResponseMetric.
-func (m *responseMetricImpl) Gather() (map[float64]float64, int, error) {
-	reg := prometheus.NewRegistry()
-	reg.MustRegister(m.latencySeconds)
+func (m *responseMetricImpl) Gather() ([]float64, map[float64]float64, int) {
+	latencies := m.dumpLatencies()
+
+	return latencies, buildPercentileLatencies(latencies), int(atomic.LoadInt64(&m.failureCount))
+}
 
-	metricFamilies, err := reg.Gather()
-	if err != nil {
-		return nil, 0, fmt.Errorf("failed to gather from local registry: %w", err)
+func (m *responseMetricImpl) dumpLatencies() []float64 {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	res := make([]float64, 0, m.latencies.Len())
+	for e := m.latencies.Front(); e != nil; e = e.Next() {
+		res = append(res, e.Value.(float64))
 	}
+	return res
+}
 
-	latencies := map[float64]float64{}
-	for _, q := range metricFamilies[0].GetMetric()[0].GetSummary().GetQuantile() {
-		latencies[q.GetQuantile()] = q.GetValue()
+var percentiles = []float64{0, 50, 90, 95, 99, 100}
+
+func buildPercentileLatencies(latencies []float64) map[float64]float64 {
+	if len(latencies) == 0 {
+		return nil
 	}
 
-	return latencies, int(atomic.LoadInt64(&m.failureCount)), nil
+	res := make(map[float64]float64, len(percentiles))
+
+	n := len(latencies)
+	sort.Float64s(latencies)
+	for _, p := range percentiles {
+		idx := int(math.Ceil(float64(n) * p / 100))
+		if idx > 0 {
+			idx--
+		}
+		res[p] = latencies[idx]
+	}
+	return res
 }

metrics/request_test.go

@@ -0,0 +1,48 @@
+package metrics
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuildPercentileLatencies(t *testing.T) {
+	ls := make([]float64, 100)
+	ls[0] = 50
+	ls[1] = 49
+	ls[2] = 1
+	res := buildPercentileLatencies(ls)
+	assert.Equal(t, float64(0), res[0])
+	assert.Equal(t, float64(0), res[50])
+	assert.Equal(t, float64(0), res[90])
+	assert.Equal(t, float64(0), res[95])
+	assert.Equal(t, float64(49), res[99])
+	assert.Equal(t, float64(50), res[100])
+
+	ls = make([]float64, 1000)
+	ls[0] = 50
+	ls[1] = 49
+	ls[2] = -1
+	res = buildPercentileLatencies(ls)
+	assert.Equal(t, float64(-1), res[0])
+	assert.Equal(t, float64(0), res[50])
+	assert.Equal(t, float64(0), res[90])
+	assert.Equal(t, float64(0), res[95])
+	assert.Equal(t, float64(0), res[99])
+	assert.Equal(t, float64(50), res[100])
+}
+
+func TestResponseMetric(t *testing.T) {
+	c := NewResponseMetric()
+	for i := 100; i > 0; i-- {
+		c.ObserveLatency(float64(i))
+	}
+
+	_, res, _ := c.Gather()
+	assert.Equal(t, float64(1), res[0])
+	assert.Equal(t, float64(50), res[50])
+	assert.Equal(t, float64(90), res[90])
+	assert.Equal(t, float64(95), res[95])
+	assert.Equal(t, float64(99), res[99])
+	assert.Equal(t, float64(100), res[100])
+}