Skip to content

Commit

Permalink
*: use list to track latencies
Browse files Browse the repository at this point in the history
Prometheus client doesn't allow us to export observed data. kperf needs
to support export raw data for aggregated result. In order to avoid
append, this patch uses container/list to maintain latencies.

Signed-off-by: Wei Fu <weifu@microsoft.com>
  • Loading branch information
fuweid committed Dec 14, 2023
1 parent 5084b23 commit 288acb6
Show file tree
Hide file tree
Showing 7 changed files with 100 additions and 61 deletions.
4 changes: 2 additions & 2 deletions api/types/metric.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@ type ResponseStats struct {
Failures int
// Duration means the time of benchmark.
Duration time.Duration
// Latencies represents the latency distribution in seconds.
// PercentileLatencies represents the latency distribution in seconds.
//
// NOTE: The key represents quantile.
Latencies map[float64]float64
PercentileLatencies map[float64]float64
// TODO:
// 1. Support total read/upload bytes
// 2. Support failures partitioned by http code and verb
Expand Down
6 changes: 3 additions & 3 deletions cmd/kperf/commands/runner/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,13 +116,13 @@ func printResponseStats(stats *types.ResponseStats) {
fmt.Printf(" Requests/sec: %.2f\n", float64(stats.Total)/stats.Duration.Seconds())

fmt.Println(" Latency Distribution:")
keys := make([]float64, 0, len(stats.Latencies))
for q := range stats.Latencies {
keys := make([]float64, 0, len(stats.PercentileLatencies))
for q := range stats.PercentileLatencies {
keys = append(keys, q)
}
sort.Float64s(keys)

for _, q := range keys {
fmt.Printf(" [%.2f] %.3fs\n", q, stats.Latencies[q])
fmt.Printf(" [%.2f] %.3fs\n", q/100.0, stats.PercentileLatencies[q])
}
}
7 changes: 0 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ module github.com/Azure/kperf
go 1.20

require (
github.com/prometheus/client_golang v1.17.0
github.com/stretchr/testify v1.8.4
github.com/urfave/cli v1.22.14
golang.org/x/time v0.3.0
Expand All @@ -14,8 +13,6 @@ require (
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-logr/logr v1.2.4 // indirect
Expand All @@ -24,13 +21,9 @@ require (
github.com/google/gofuzz v1.2.0 // indirect
github.com/imdario/mergo v0.3.6 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/net v0.17.0 // indirect
Expand Down
16 changes: 0 additions & 16 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand All @@ -17,7 +13,6 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv
github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
Expand All @@ -39,8 +34,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
Expand All @@ -49,14 +42,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
Expand Down Expand Up @@ -88,7 +73,6 @@ golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down
67 changes: 42 additions & 25 deletions metrics/request.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package metrics

import (
"fmt"
"container/list"
"math"
"sort"
"sync"
"sync/atomic"

"github.com/prometheus/client_golang/prometheus"
)

// ResponseMetric is a measurement related to http response.
Expand All @@ -14,30 +15,26 @@ type ResponseMetric interface {
// ObserveFailure observes failure response.
ObserveFailure()
// Gather returns the summary.
Gather() (latencies map[float64]float64, failure int, _ error)
Gather() (latencies []float64, percentileLatencies map[float64]float64, failure int)
}

type responseMetricImpl struct {
latencySeconds *prometheus.SummaryVec
failureCount int64
mu sync.Mutex
failureCount int64
latencies *list.List
}

func NewResponseMetric() ResponseMetric {
return &responseMetricImpl{
latencySeconds: prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "request",
Name: "request_latency_seconds",
Objectives: map[float64]float64{0: 0, 0.5: 0, 0.9: 0, 0.95: 0, 0.99: 0, 1: 0},
},
[]string{},
),
latencies: list.New(),
}
}

// ObserveLatency implements ResponseMetric.
func (m *responseMetricImpl) ObserveLatency(seconds float64) {
m.latencySeconds.WithLabelValues().Observe(seconds)
m.mu.Lock()
defer m.mu.Unlock()
m.latencies.PushBack(seconds)
}

// ObserveFailure implements ResponseMetric.
Expand All @@ -46,19 +43,39 @@ func (m *responseMetricImpl) ObserveFailure() {
}

// Gather implements ResponseMetric.
func (m *responseMetricImpl) Gather() (map[float64]float64, int, error) {
reg := prometheus.NewRegistry()
reg.MustRegister(m.latencySeconds)
func (m *responseMetricImpl) Gather() ([]float64, map[float64]float64, int) {
latencies := m.dumpLatencies()

return latencies, buildPercentileLatencies(latencies), int(atomic.LoadInt64(&m.failureCount))
}

metricFamilies, err := reg.Gather()
if err != nil {
return nil, 0, fmt.Errorf("failed to gather from local registry: %w", err)
func (m *responseMetricImpl) dumpLatencies() []float64 {
m.mu.Lock()
defer m.mu.Unlock()
res := make([]float64, 0, m.latencies.Len())
for e := m.latencies.Front(); e != nil; e = e.Next() {
res = append(res, e.Value.(float64))
}
return res
}

latencies := map[float64]float64{}
for _, q := range metricFamilies[0].GetMetric()[0].GetSummary().GetQuantile() {
latencies[q.GetQuantile()] = q.GetValue()
var percentiles = []float64{0, 50, 90, 95, 99, 100}

func buildPercentileLatencies(latencies []float64) map[float64]float64 {
if len(latencies) == 0 {
return nil
}

return latencies, int(atomic.LoadInt64(&m.failureCount)), nil
res := make(map[float64]float64, len(percentiles))

n := len(latencies)
sort.Float64s(latencies)
for _, p := range percentiles {
idx := int(math.Ceil(float64(n) * p / 100))
if idx > 0 {
idx--
}
res[p] = latencies[idx]
}
return res
}
48 changes: 48 additions & 0 deletions metrics/request_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package metrics

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestBuildPercentileLatencies(t *testing.T) {
ls := make([]float64, 100)
ls[0] = 50
ls[1] = 49
ls[2] = 1
res := buildPercentileLatencies(ls)
assert.Equal(t, float64(0), res[0])
assert.Equal(t, float64(0), res[50])
assert.Equal(t, float64(0), res[90])
assert.Equal(t, float64(0), res[95])
assert.Equal(t, float64(49), res[99])
assert.Equal(t, float64(50), res[100])

ls = make([]float64, 1000)
ls[0] = 50
ls[1] = 49
ls[2] = -1
res = buildPercentileLatencies(ls)
assert.Equal(t, float64(-1), res[0])
assert.Equal(t, float64(0), res[50])
assert.Equal(t, float64(0), res[90])
assert.Equal(t, float64(0), res[95])
assert.Equal(t, float64(0), res[99])
assert.Equal(t, float64(50), res[100])
}

func TestResponseMetric(t *testing.T) {
c := NewResponseMetric()
for i := 100; i > 0; i-- {
c.ObserveLatency(float64(i))
}

_, res, _ := c.Gather()
assert.Equal(t, float64(1), res[0])
assert.Equal(t, float64(50), res[50])
assert.Equal(t, float64(90), res[90])
assert.Equal(t, float64(95), res[95])
assert.Equal(t, float64(99), res[99])
assert.Equal(t, float64(100), res[100])
}
13 changes: 5 additions & 8 deletions request/schedule.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,14 +80,11 @@ func Schedule(ctx context.Context, spec *types.LoadProfileSpec, restCli []rest.I

totalDuration := time.Since(start)

latencies, failures, err := respMetric.Gather()
if err != nil {
return nil, err
}
_, percentileLatencies, failures := respMetric.Gather()
return &types.ResponseStats{
Total: spec.Total,
Failures: failures,
Duration: totalDuration,
Latencies: latencies,
Total: spec.Total,
Failures: failures,
Duration: totalDuration,
PercentileLatencies: percentileLatencies,
}, nil
}

0 comments on commit 288acb6

Please sign in to comment.