From 288acb690b9e8b4f6e0816871fdc4d82c28a3944 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Thu, 14 Dec 2023 08:06:03 +0000 Subject: [PATCH] *: use list to track latencies Prometheus client doesn't allow us to export observed data. kperf needs to support export raw data for aggregated result. In order to avoid append, this patch uses container/list to maintain latencies. Signed-off-by: Wei Fu --- api/types/metric.go | 4 +- cmd/kperf/commands/runner/runner.go | 6 +-- go.mod | 7 --- go.sum | 16 ------- metrics/request.go | 67 ++++++++++++++++++----------- metrics/request_test.go | 48 +++++++++++++++++++++ request/schedule.go | 13 +++--- 7 files changed, 100 insertions(+), 61 deletions(-) create mode 100644 metrics/request_test.go diff --git a/api/types/metric.go b/api/types/metric.go index 69aab68..d6eb2c1 100644 --- a/api/types/metric.go +++ b/api/types/metric.go @@ -10,10 +10,10 @@ type ResponseStats struct { Failures int // Duration means the time of benchmark. Duration time.Duration - // Latencies represents the latency distribution in seconds. + // PercentileLatencies represents the latency distribution in seconds. // // NOTE: The key represents quantile. - Latencies map[float64]float64 + PercentileLatencies map[float64]float64 // TODO: // 1. Support total read/upload bytes // 2. Support failures partitioned by http code and verb diff --git a/cmd/kperf/commands/runner/runner.go b/cmd/kperf/commands/runner/runner.go index 496fbbd..c2ab3f9 100644 --- a/cmd/kperf/commands/runner/runner.go +++ b/cmd/kperf/commands/runner/runner.go @@ -116,13 +116,13 @@ func printResponseStats(stats *types.ResponseStats) { fmt.Printf(" Requests/sec: %.2f\n", float64(stats.Total)/stats.Duration.Seconds()) fmt.Println(" Latency Distribution:") - keys := make([]float64, 0, len(stats.Latencies)) - for q := range stats.Latencies { + keys := make([]float64, 0, len(stats.PercentileLatencies)) + for q := range stats.PercentileLatencies { keys = append(keys, q) } sort.Float64s(keys) for _, q := range keys { - fmt.Printf(" [%.2f] %.3fs\n", q, stats.Latencies[q]) + fmt.Printf(" [%.2f] %.3fs\n", q/100.0, stats.PercentileLatencies[q]) } } diff --git a/go.mod b/go.mod index 0326c7d..d414d58 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/Azure/kperf go 1.20 require ( - github.com/prometheus/client_golang v1.17.0 github.com/stretchr/testify v1.8.4 github.com/urfave/cli v1.22.14 golang.org/x/time v0.3.0 @@ -14,8 +13,6 @@ require ( ) require ( - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.2.4 // indirect @@ -24,13 +21,9 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/net v0.17.0 // indirect diff --git a/go.sum b/go.sum index 2ff18d9..e21baab 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,4 @@ github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -17,7 +13,6 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -39,8 +34,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -49,14 +42,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -88,7 +73,6 @@ golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/metrics/request.go b/metrics/request.go index cd8840a..374c929 100644 --- a/metrics/request.go +++ b/metrics/request.go @@ -1,10 +1,11 @@ package metrics import ( - "fmt" + "container/list" + "math" + "sort" + "sync" "sync/atomic" - - "github.com/prometheus/client_golang/prometheus" ) // ResponseMetric is a measurement related to http response. @@ -14,30 +15,26 @@ type ResponseMetric interface { // ObserveFailure observes failure response. ObserveFailure() // Gather returns the summary. - Gather() (latencies map[float64]float64, failure int, _ error) + Gather() (latencies []float64, percentileLatencies map[float64]float64, failure int) } type responseMetricImpl struct { - latencySeconds *prometheus.SummaryVec - failureCount int64 + mu sync.Mutex + failureCount int64 + latencies *list.List } func NewResponseMetric() ResponseMetric { return &responseMetricImpl{ - latencySeconds: prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "request", - Name: "request_latency_seconds", - Objectives: map[float64]float64{0: 0, 0.5: 0, 0.9: 0, 0.95: 0, 0.99: 0, 1: 0}, - }, - []string{}, - ), + latencies: list.New(), } } // ObserveLatency implements ResponseMetric. func (m *responseMetricImpl) ObserveLatency(seconds float64) { - m.latencySeconds.WithLabelValues().Observe(seconds) + m.mu.Lock() + defer m.mu.Unlock() + m.latencies.PushBack(seconds) } // ObserveFailure implements ResponseMetric. @@ -46,19 +43,39 @@ func (m *responseMetricImpl) ObserveFailure() { } // Gather implements ResponseMetric. -func (m *responseMetricImpl) Gather() (map[float64]float64, int, error) { - reg := prometheus.NewRegistry() - reg.MustRegister(m.latencySeconds) +func (m *responseMetricImpl) Gather() ([]float64, map[float64]float64, int) { + latencies := m.dumpLatencies() + + return latencies, buildPercentileLatencies(latencies), int(atomic.LoadInt64(&m.failureCount)) +} - metricFamilies, err := reg.Gather() - if err != nil { - return nil, 0, fmt.Errorf("failed to gather from local registry: %w", err) +func (m *responseMetricImpl) dumpLatencies() []float64 { + m.mu.Lock() + defer m.mu.Unlock() + res := make([]float64, 0, m.latencies.Len()) + for e := m.latencies.Front(); e != nil; e = e.Next() { + res = append(res, e.Value.(float64)) } + return res +} - latencies := map[float64]float64{} - for _, q := range metricFamilies[0].GetMetric()[0].GetSummary().GetQuantile() { - latencies[q.GetQuantile()] = q.GetValue() +var percentiles = []float64{0, 50, 90, 95, 99, 100} + +func buildPercentileLatencies(latencies []float64) map[float64]float64 { + if len(latencies) == 0 { + return nil } - return latencies, int(atomic.LoadInt64(&m.failureCount)), nil + res := make(map[float64]float64, len(percentiles)) + + n := len(latencies) + sort.Float64s(latencies) + for _, p := range percentiles { + idx := int(math.Ceil(float64(n) * p / 100)) + if idx > 0 { + idx-- + } + res[p] = latencies[idx] + } + return res } diff --git a/metrics/request_test.go b/metrics/request_test.go new file mode 100644 index 0000000..2cd8e15 --- /dev/null +++ b/metrics/request_test.go @@ -0,0 +1,48 @@ +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildPercentileLatencies(t *testing.T) { + ls := make([]float64, 100) + ls[0] = 50 + ls[1] = 49 + ls[2] = 1 + res := buildPercentileLatencies(ls) + assert.Equal(t, float64(0), res[0]) + assert.Equal(t, float64(0), res[50]) + assert.Equal(t, float64(0), res[90]) + assert.Equal(t, float64(0), res[95]) + assert.Equal(t, float64(49), res[99]) + assert.Equal(t, float64(50), res[100]) + + ls = make([]float64, 1000) + ls[0] = 50 + ls[1] = 49 + ls[2] = -1 + res = buildPercentileLatencies(ls) + assert.Equal(t, float64(-1), res[0]) + assert.Equal(t, float64(0), res[50]) + assert.Equal(t, float64(0), res[90]) + assert.Equal(t, float64(0), res[95]) + assert.Equal(t, float64(0), res[99]) + assert.Equal(t, float64(50), res[100]) +} + +func TestResponseMetric(t *testing.T) { + c := NewResponseMetric() + for i := 100; i > 0; i-- { + c.ObserveLatency(float64(i)) + } + + _, res, _ := c.Gather() + assert.Equal(t, float64(1), res[0]) + assert.Equal(t, float64(50), res[50]) + assert.Equal(t, float64(90), res[90]) + assert.Equal(t, float64(95), res[95]) + assert.Equal(t, float64(99), res[99]) + assert.Equal(t, float64(100), res[100]) +} diff --git a/request/schedule.go b/request/schedule.go index f2a216e..246fcc1 100644 --- a/request/schedule.go +++ b/request/schedule.go @@ -80,14 +80,11 @@ func Schedule(ctx context.Context, spec *types.LoadProfileSpec, restCli []rest.I totalDuration := time.Since(start) - latencies, failures, err := respMetric.Gather() - if err != nil { - return nil, err - } + _, percentileLatencies, failures := respMetric.Gather() return &types.ResponseStats{ - Total: spec.Total, - Failures: failures, - Duration: totalDuration, - Latencies: latencies, + Total: spec.Total, + Failures: failures, + Duration: totalDuration, + PercentileLatencies: percentileLatencies, }, nil }