diff --git a/api/types/metric.go b/api/types/metric.go index 69aab68..d6eb2c1 100644 --- a/api/types/metric.go +++ b/api/types/metric.go @@ -10,10 +10,10 @@ type ResponseStats struct { Failures int // Duration means the time of benchmark. Duration time.Duration - // Latencies represents the latency distribution in seconds. + // PercentileLatencies represents the latency distribution in seconds. // // NOTE: The key represents quantile. - Latencies map[float64]float64 + PercentileLatencies map[float64]float64 // TODO: // 1. Support total read/upload bytes // 2. Support failures partitioned by http code and verb diff --git a/cmd/kperf/commands/runner/runner.go b/cmd/kperf/commands/runner/runner.go index 496fbbd..c2ab3f9 100644 --- a/cmd/kperf/commands/runner/runner.go +++ b/cmd/kperf/commands/runner/runner.go @@ -116,13 +116,13 @@ func printResponseStats(stats *types.ResponseStats) { fmt.Printf(" Requests/sec: %.2f\n", float64(stats.Total)/stats.Duration.Seconds()) fmt.Println(" Latency Distribution:") - keys := make([]float64, 0, len(stats.Latencies)) - for q := range stats.Latencies { + keys := make([]float64, 0, len(stats.PercentileLatencies)) + for q := range stats.PercentileLatencies { keys = append(keys, q) } sort.Float64s(keys) for _, q := range keys { - fmt.Printf(" [%.2f] %.3fs\n", q, stats.Latencies[q]) + fmt.Printf(" [%.2f] %.3fs\n", q/100.0, stats.PercentileLatencies[q]) } } diff --git a/go.mod b/go.mod index 0326c7d..d414d58 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/Azure/kperf go 1.20 require ( - github.com/prometheus/client_golang v1.17.0 github.com/stretchr/testify v1.8.4 github.com/urfave/cli v1.22.14 golang.org/x/time v0.3.0 @@ -14,8 +13,6 @@ require ( ) require ( - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.2.4 // indirect @@ -24,13 +21,9 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/net v0.17.0 // indirect diff --git a/go.sum b/go.sum index 2ff18d9..e21baab 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,4 @@ github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -17,7 +13,6 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -39,8 +34,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -49,14 +42,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -88,7 +73,6 @@ golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/metrics/request.go b/metrics/request.go index cd8840a..374c929 100644 --- a/metrics/request.go +++ b/metrics/request.go @@ -1,10 +1,11 @@ package metrics import ( - "fmt" + "container/list" + "math" + "sort" + "sync" "sync/atomic" - - "github.com/prometheus/client_golang/prometheus" ) // ResponseMetric is a measurement related to http response. @@ -14,30 +15,26 @@ type ResponseMetric interface { // ObserveFailure observes failure response. ObserveFailure() // Gather returns the summary. - Gather() (latencies map[float64]float64, failure int, _ error) + Gather() (latencies []float64, percentileLatencies map[float64]float64, failure int) } type responseMetricImpl struct { - latencySeconds *prometheus.SummaryVec - failureCount int64 + mu sync.Mutex + failureCount int64 + latencies *list.List } func NewResponseMetric() ResponseMetric { return &responseMetricImpl{ - latencySeconds: prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "request", - Name: "request_latency_seconds", - Objectives: map[float64]float64{0: 0, 0.5: 0, 0.9: 0, 0.95: 0, 0.99: 0, 1: 0}, - }, - []string{}, - ), + latencies: list.New(), } } // ObserveLatency implements ResponseMetric. func (m *responseMetricImpl) ObserveLatency(seconds float64) { - m.latencySeconds.WithLabelValues().Observe(seconds) + m.mu.Lock() + defer m.mu.Unlock() + m.latencies.PushBack(seconds) } // ObserveFailure implements ResponseMetric. @@ -46,19 +43,39 @@ func (m *responseMetricImpl) ObserveFailure() { } // Gather implements ResponseMetric. -func (m *responseMetricImpl) Gather() (map[float64]float64, int, error) { - reg := prometheus.NewRegistry() - reg.MustRegister(m.latencySeconds) +func (m *responseMetricImpl) Gather() ([]float64, map[float64]float64, int) { + latencies := m.dumpLatencies() + + return latencies, buildPercentileLatencies(latencies), int(atomic.LoadInt64(&m.failureCount)) +} - metricFamilies, err := reg.Gather() - if err != nil { - return nil, 0, fmt.Errorf("failed to gather from local registry: %w", err) +func (m *responseMetricImpl) dumpLatencies() []float64 { + m.mu.Lock() + defer m.mu.Unlock() + res := make([]float64, 0, m.latencies.Len()) + for e := m.latencies.Front(); e != nil; e = e.Next() { + res = append(res, e.Value.(float64)) } + return res +} - latencies := map[float64]float64{} - for _, q := range metricFamilies[0].GetMetric()[0].GetSummary().GetQuantile() { - latencies[q.GetQuantile()] = q.GetValue() +var percentiles = []float64{0, 50, 90, 95, 99, 100} + +func buildPercentileLatencies(latencies []float64) map[float64]float64 { + if len(latencies) == 0 { + return nil } - return latencies, int(atomic.LoadInt64(&m.failureCount)), nil + res := make(map[float64]float64, len(percentiles)) + + n := len(latencies) + sort.Float64s(latencies) + for _, p := range percentiles { + idx := int(math.Ceil(float64(n) * p / 100)) + if idx > 0 { + idx-- + } + res[p] = latencies[idx] + } + return res } diff --git a/metrics/request_test.go b/metrics/request_test.go new file mode 100644 index 0000000..2cd8e15 --- /dev/null +++ b/metrics/request_test.go @@ -0,0 +1,48 @@ +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildPercentileLatencies(t *testing.T) { + ls := make([]float64, 100) + ls[0] = 50 + ls[1] = 49 + ls[2] = 1 + res := buildPercentileLatencies(ls) + assert.Equal(t, float64(0), res[0]) + assert.Equal(t, float64(0), res[50]) + assert.Equal(t, float64(0), res[90]) + assert.Equal(t, float64(0), res[95]) + assert.Equal(t, float64(49), res[99]) + assert.Equal(t, float64(50), res[100]) + + ls = make([]float64, 1000) + ls[0] = 50 + ls[1] = 49 + ls[2] = -1 + res = buildPercentileLatencies(ls) + assert.Equal(t, float64(-1), res[0]) + assert.Equal(t, float64(0), res[50]) + assert.Equal(t, float64(0), res[90]) + assert.Equal(t, float64(0), res[95]) + assert.Equal(t, float64(0), res[99]) + assert.Equal(t, float64(50), res[100]) +} + +func TestResponseMetric(t *testing.T) { + c := NewResponseMetric() + for i := 100; i > 0; i-- { + c.ObserveLatency(float64(i)) + } + + _, res, _ := c.Gather() + assert.Equal(t, float64(1), res[0]) + assert.Equal(t, float64(50), res[50]) + assert.Equal(t, float64(90), res[90]) + assert.Equal(t, float64(95), res[95]) + assert.Equal(t, float64(99), res[99]) + assert.Equal(t, float64(100), res[100]) +} diff --git a/request/schedule.go b/request/schedule.go index f2a216e..246fcc1 100644 --- a/request/schedule.go +++ b/request/schedule.go @@ -80,14 +80,11 @@ func Schedule(ctx context.Context, spec *types.LoadProfileSpec, restCli []rest.I totalDuration := time.Since(start) - latencies, failures, err := respMetric.Gather() - if err != nil { - return nil, err - } + _, percentileLatencies, failures := respMetric.Gather() return &types.ResponseStats{ - Total: spec.Total, - Failures: failures, - Duration: totalDuration, - Latencies: latencies, + Total: spec.Total, + Failures: failures, + Duration: totalDuration, + PercentileLatencies: percentileLatencies, }, nil }