diff --git a/exporter/exporter.go b/exporter/exporter.go index 12d7a1dd..4862ec21 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -180,6 +180,7 @@ func NewKvrocksExporter(kvrocksURI string, opts Options) (*Exporter, error) { txt string lbls []string }{ + "commands_duration_seconds_bucket": {txt: `Histogram of the amount of time in seconds spent per command`, lbls: []string{"cmd"}}, "commands_duration_seconds_total": {txt: `Total amount of time in seconds spent per command`, lbls: []string{"cmd"}}, "commands_total": {txt: `Total number of calls per command`, lbls: []string{"cmd"}}, "connected_slave_lag_seconds": {txt: "Lag of connected slave", lbls: []string{"slave_ip", "slave_port", "slave_state"}}, diff --git a/exporter/info.go b/exporter/info.go index 865c9caa..3bb5c15d 100644 --- a/exporter/info.go +++ b/exporter/info.go @@ -3,7 +3,9 @@ package exporter import ( "errors" "fmt" + "math" "regexp" + "sort" "strconv" "strings" "time" @@ -341,9 +343,80 @@ func parseMetricsCommandStats(fieldKey string, fieldValue string) (string, float return cmd, calls, usecTotal, nil } +func histSplit(r rune) bool { + return r == '=' || r == ',' +} + +func parseMetricsCommandStatsHist(fieldKey string, fieldValue string) (string, uint64, uint64, map[float64]uint64, error) { + /* + Format: + cmdstathist_get:10=1191,20=1,50=0,70=0,100=0,150=0,inf=0,sum=12388,count=1192 + + broken up like this: + fieldKey = cmdstathist_get + fieldValue = 10=1191,20=1,50=0,70=0,100=0,150=0,inf=0,sum=12388,count=1192 + */ + + const cmdPrefix = "cmdstathist_" + + if !strings.HasPrefix(fieldKey, cmdPrefix) { + return "", 0, 0, nil, errors.New("invalid fieldKey") + } + cmd := strings.TrimPrefix(fieldKey, cmdPrefix) + + splitValues := strings.FieldsFunc(fieldValue, histSplit) + var histogram = map[float64]uint64{} + var keys = make([]float64, 0, len(histogram)) + + if len(splitValues)%2 != 0 { + return "", 0, 0, nil, errors.New("uneven number of keys for bucket") + } + + var sum, count uint64 + var err error + // NB: splitValues slice is a list of tuples so iterating by 2 + for i := 0; i < len(splitValues); i = i + 2 { + if splitValues[i] == "sum" { + sum, err = strconv.ParseUint(splitValues[i+1], 10, 64) + if err != nil { + return "", 0, 0, nil, fmt.Errorf("invalid value for sum: %w", err) + } + continue + } + if splitValues[i] == "count" { + count, err = strconv.ParseUint(splitValues[i+1], 10, 64) + if err != nil { + return "", 0, 0, nil, fmt.Errorf("invalid value for count: %w", err) + } + continue + } + + bucketCount, err := strconv.ParseUint(splitValues[i+1], 10, 64) + if err != nil { + return "", 0, 0, nil, fmt.Errorf("invalid splitValue for bucket: %w", err) + } + bucketValue := math.Inf(1) + if val, err := strconv.ParseFloat(strings.TrimSpace(splitValues[i]), 64); err == nil { + bucketValue = val / 1e6 + } + histogram[bucketValue] = bucketCount + keys = append(keys, bucketValue) + } + + sort.Float64s(keys) + + for i := 1; i < len(keys); i++ { + histogram[keys[i]] += histogram[keys[i-1]] + } + return cmd, count, sum, histogram, nil +} + func (e *Exporter) handleMetricsCommandStats(ch chan<- prometheus.Metric, fieldKey string, fieldValue string) { if cmd, calls, usecTotal, err := parseMetricsCommandStats(fieldKey, fieldValue); err == nil { e.registerConstMetric(ch, "commands_total", calls, prometheus.CounterValue, cmd) e.registerConstMetric(ch, "commands_duration_seconds_total", usecTotal/1e6, prometheus.CounterValue, cmd) } + if cmd, count, sum, buckets, err := parseMetricsCommandStatsHist(fieldKey, fieldValue); err == nil { + e.registerHist(ch, "commands_duration_seconds_bucket", count, float64(sum)/1e6, buckets, cmd) + } } diff --git a/exporter/info_test.go b/exporter/info_test.go index 526bb7ca..c4379912 100644 --- a/exporter/info_test.go +++ b/exporter/info_test.go @@ -2,8 +2,10 @@ package exporter import ( "fmt" + "math" "net/http/httptest" "os" + "reflect" "regexp" "strings" "testing" @@ -41,7 +43,7 @@ func TestKeyspaceStringParser(t *testing.T) { } if ok && (kt != tst.keysTotal || kx != tst.keysEx || ttl != tst.avgTTL || kexp != tst.keysExpired) { - t.Errorf("values not matching, db:%s stats:%s %f %f %f", tst.db, tst.stats, kt, kx, ttl, kexp) + t.Errorf("values not matching, db:%s stats:%s %f %f %f %f", tst.db, tst.stats, kt, kx, ttl, kexp) } } } @@ -234,6 +236,11 @@ func TestParseCommandStats(t *testing.T) { fieldValue: "calls=75,usec=DEF,usec_per_call=16.80", wantSuccess: false, }, + { + fieldKey: "cmdstat_georadius_ro", + fieldValue: "calls=75,usec=DEF,usec_per_call=16.80", + wantSuccess: false, + }, } { t.Run(tst.fieldKey+tst.fieldValue, func(t *testing.T) { @@ -265,5 +272,164 @@ func TestParseCommandStats(t *testing.T) { } }) } +} + +func TestParseCommandStatsHist(t *testing.T) { + + for _, tst := range []struct { + fieldKey string + fieldValue string + + wantSuccess bool + wantCmd string + wantBuckets map[float64]uint64 + wantSum uint64 + wantCount uint64 + }{ + { + fieldKey: "cmdstathist_get", + fieldValue: "10=1191,20=1,50=0,70=0,100=0,150=0,inf=0,sum=10000,count=1192", + wantSuccess: true, + wantCmd: "get", + wantBuckets: map[float64]uint64{ + 0.00001: 1191, + 0.00002: 1192, + 0.00005: 1192, + 0.00007: 1192, + 0.0001: 1192, + 0.00015: 1192, + math.Inf(1): 1192, + }, + wantSum: 10000, + wantCount: 1192, + }, + { + fieldKey: "cmdstathist_hget", + fieldValue: "", + wantSuccess: true, + wantCmd: "hget", + wantBuckets: map[float64]uint64{}, + }, + { + fieldKey: "cmdstathis_hget", + fieldValue: "fd", + wantSuccess: false, + wantCmd: "hget", + }, + { + fieldKey: "cmdstathist_hget", + fieldValue: "fd", + wantSuccess: false, + wantCmd: "hget", + }, + { + fieldKey: "cmdstathist_hget", + fieldValue: "fd=malformed", + wantSuccess: false, + wantCmd: "hget", + }, + { + fieldKey: "cmdstathist_get", + fieldValue: "10=1191,20=1,50=0,70=0,100=0,150=0,inf=0,sum=,count=1192", + wantSuccess: false, + wantCmd: "get", + wantBuckets: map[float64]uint64{ + 0.00001: 1191, + 0.00002: 1192, + 0.00005: 1192, + 0.00007: 1192, + 0.0001: 1192, + 0.00015: 1192, + math.Inf(1): 1192, + }, + wantSum: 0, + wantCount: 1192, + }, + { + fieldKey: "cmdstathist_get", + fieldValue: "10=1191,20=1,50=0,70=0,100=0,150=0,inf=0,sum=aa,count=1192", + wantSuccess: false, + wantCmd: "get", + wantBuckets: map[float64]uint64{ + 0.00001: 1191, + 0.00002: 1192, + 0.00005: 1192, + 0.00007: 1192, + 0.0001: 1192, + 0.00015: 1192, + math.Inf(1): 1192, + }, + wantSum: 0, + wantCount: 1192, + }, + { + fieldKey: "cmdstathist_get", + fieldValue: "10=1191,20=1,50=0,70=0,100=0,150=0,inf=0,sum=10000,count=", + wantSuccess: false, + wantCmd: "get", + wantBuckets: map[float64]uint64{ + 0.00001: 1191, + 0.00002: 1192, + 0.00005: 1192, + 0.00007: 1192, + 0.0001: 1192, + 0.00015: 1192, + math.Inf(1): 1192, + }, + wantSum: 10000, + wantCount: 0, + }, + { + fieldKey: "cmdstathist_get", + fieldValue: "10=1191,20=1,50=0,70=0,100=0,150=0,inf=0,sum=10000,count=dasd", + wantSuccess: false, + wantCmd: "get", + wantBuckets: map[float64]uint64{ + 0.00001: 1191, + 0.00002: 1192, + 0.00005: 1192, + 0.00007: 1192, + 0.0001: 1192, + 0.00015: 1192, + math.Inf(1): 1192, + }, + wantSum: 10000, + wantCount: 0, + }, + } { + t.Run(tst.fieldKey+tst.fieldValue, func(t *testing.T) { + cmd, count, sum, buckets, err := parseMetricsCommandStatsHist(tst.fieldKey, tst.fieldValue) + + if tst.wantSuccess && err != nil { + t.Fatalf("err: %s", err) + return + } + + if !tst.wantSuccess && err == nil { + t.Fatalf("expected err!") + return + } + + if !tst.wantSuccess { + return + } + + if cmd != tst.wantCmd { + t.Fatalf("cmd not matching, got: %s, wanted: %s", cmd, tst.wantCmd) + } + + if !reflect.DeepEqual(buckets, tst.wantBuckets) { + t.Fatalf("cmd not matching, got: %v, wanted: %v", buckets, tst.wantBuckets) + } + + if count != tst.wantCount { + t.Fatalf("count not matching, got: %d, wanted: %d", count, tst.wantCount) + } + + if sum != tst.wantSum { + t.Fatalf("count not matching, got: %d, wanted: %d", sum, tst.wantSum) + } + }) + } } diff --git a/exporter/metrics.go b/exporter/metrics.go index b7d567a4..b4f93534 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -89,3 +89,14 @@ func (e *Exporter) registerConstMetric(ch chan<- prometheus.Metric, metric strin ch <- m } } + +func (e *Exporter) registerHist(ch chan<- prometheus.Metric, metric string, count uint64, sum float64, buckets map[float64]uint64, labelValues ...string) { + descr := e.metricDescriptions[metric] + if descr == nil { + descr = newMetricDescr(e.options.Namespace, metric, metric+" metric", labelValues) + } + + if m, err := prometheus.NewConstHistogram(descr, count, sum, buckets, labelValues...); err == nil { + ch <- m + } +}