From 1f324b47eaeee5d6c15092651ced8d70ab583172 Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Mon, 23 Sep 2024 15:02:21 +0200 Subject: [PATCH] Alerting: adapt sum of gauges to use options (#584) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * adapt sum of gauges to use options * add WithLabels function back and revert unit tests * add new function unit tests * Update metrics/tenant_registries.go Co-authored-by: Peter Štibraný * add changelog --------- Co-authored-by: Peter Štibraný --- CHANGELOG.md | 1 + metrics/tenant_registries.go | 18 +++-- metrics/tenant_registries_test.go | 120 ++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d43c2f562..ebce52e16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -229,6 +229,7 @@ * [ENHANCEMENT] `services.FailureWatcher` can now be closed, which unregisters all service and manager listeners, and closes channel used to receive errors. #564 * [ENHANCEMENT] Runtimeconfig: support gzip-compressed files with `.gz` extension. #571 * [ENHANCEMENT] grpcclient: Support custom gRPC compressors. #583 +* [ENHANCEMENT] Adapt `metrics.SendSumOfGaugesPerTenant` to use `metrics.MetricOption`. #584 * [CHANGE] Backoff: added `Backoff.ErrCause()` which is like `Backoff.Err()` but returns the context cause if backoff is terminated because the context has been canceled. #538 * [BUGFIX] spanlogger: Support multiple tenant IDs. #59 * [BUGFIX] Memberlist: fixed corrupted packets when sending compound messages with more than 255 messages or messages bigger than 64KB. #85 diff --git a/metrics/tenant_registries.go b/metrics/tenant_registries.go index e1dc153d2..c903877ae 100644 --- a/metrics/tenant_registries.go +++ b/metrics/tenant_registries.go @@ -219,25 +219,29 @@ func (d MetricFamiliesPerTenant) SendSumOfGaugesWithLabels(out chan<- prometheus // SendSumOfGaugesPerTenant provides metrics on a per-tenant basis. // This function assumes that `tenant` is the first label on the provided metric Desc. -func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenant(out chan<- prometheus.Metric, desc *prometheus.Desc, gauge string) { - d.SendSumOfGaugesPerTenantWithLabels(out, desc, gauge) -} +func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenant(out chan<- prometheus.Metric, desc *prometheus.Desc, metric string, options ...MetricOption) { + opts := applyMetricOptions(options...) -// SendSumOfGaugesPerTenantWithLabels provides metrics with the provided label names on a per-tenant basis. This function assumes that `tenant` is the -// first label on the provided metric Desc -func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenantWithLabels(out chan<- prometheus.Metric, desc *prometheus.Desc, metric string, labelNames ...string) { for _, tenantEntry := range d { if tenantEntry.tenant == "" { continue } result := singleValueWithLabelsMap{} - tenantEntry.metrics.sumOfSingleValuesWithLabels(metric, labelNames, gaugeValue, result.aggregateFn, false) + tenantEntry.metrics.sumOfSingleValuesWithLabels(metric, opts.labelNames, gaugeValue, result.aggregateFn, opts.skipZeroValueMetrics) result.prependTenantLabelValue(tenantEntry.tenant) result.WriteToMetricChannel(out, desc, prometheus.GaugeValue) } } +// SendSumOfGaugesPerTenantWithLabels provides metrics with the provided label names on a per-tenant basis. This function assumes that `tenant` is the +// first label on the provided metric Desc +// +// Deprecated: use SendSumOfGaugesPerTenant with WithLabels option instead. +func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenantWithLabels(out chan<- prometheus.Metric, desc *prometheus.Desc, metric string, labelNames ...string) { + d.SendSumOfGaugesPerTenant(out, desc, metric, WithLabels(labelNames...)) +} + func (d MetricFamiliesPerTenant) sumOfSingleValuesWithLabels(metric string, fn func(*dto.Metric) float64, labelNames []string, skipZeroValue bool) singleValueWithLabelsMap { result := singleValueWithLabelsMap{} for _, tenantEntry := range d { diff --git a/metrics/tenant_registries_test.go b/metrics/tenant_registries_test.go index f0776c9b8..d551cf69a 100644 --- a/metrics/tenant_registries_test.go +++ b/metrics/tenant_registries_test.go @@ -184,6 +184,126 @@ func BenchmarkGetMetricsWithLabelNames(b *testing.B) { } } +// TestSendSumOfGaugesPerTenant tests to ensure multiple metrics for the same tenant with a matching label are +// summed correctly. +func TestSendSumOfGaugesPerTenant(t *testing.T) { + user1Reg := prometheus.NewRegistry() + user2Reg := prometheus.NewRegistry() + user3Reg := prometheus.NewRegistry() + user1Metric := promauto.With(user1Reg).NewGaugeVec(prometheus.GaugeOpts{Name: "test_metric"}, []string{"label_one", "label_two"}) + user2Metric := promauto.With(user2Reg).NewGaugeVec(prometheus.GaugeOpts{Name: "test_metric"}, []string{"label_one", "label_two"}) + user3Metric := promauto.With(user3Reg).NewGaugeVec(prometheus.GaugeOpts{Name: "test_metric"}, []string{"label_one", "label_two"}) + user1Metric.WithLabelValues("a", "b").Set(100) + user1Metric.WithLabelValues("a", "c").Set(80) + user1Metric.WithLabelValues("b", "c").Set(0) + user2Metric.WithLabelValues("a", "b").Set(60) + user2Metric.WithLabelValues("a", "c").Set(40) + user2Metric.WithLabelValues("b", "c").Set(0) + user3Metric.WithLabelValues("a", "b").Set(0) + user3Metric.WithLabelValues("a", "c").Set(0) + user3Metric.WithLabelValues("b", "c").Set(20) + + regs := NewTenantRegistries(log.NewNopLogger()) + regs.AddTenantRegistry("user-1", user1Reg) + regs.AddTenantRegistry("user-2", user2Reg) + regs.AddTenantRegistry("user-3", user3Reg) + mf := regs.BuildMetricFamiliesPerTenant() + + t.Run("group metrics by user and label_one", func(t *testing.T) { + desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_one"}, nil) + actual := collectMetrics(t, func(out chan prometheus.Metric) { + mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_one")) + }) + expected := []*dto.Metric{ + {Label: makeLabels("label_one", "a", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(180)}}, + {Label: makeLabels("label_one", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(0)}}, + {Label: makeLabels("label_one", "a", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(100)}}, + {Label: makeLabels("label_one", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(0)}}, + {Label: makeLabels("label_one", "a", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(0)}}, + {Label: makeLabels("label_one", "b", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}}, + } + require.ElementsMatch(t, expected, actual) + }) + + t.Run("group metrics by user and label_one, and skip zero value metrics", func(t *testing.T) { + desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_one"}, nil) + actual := collectMetrics(t, func(out chan prometheus.Metric) { + mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_one"), WithSkipZeroValueMetrics) + }) + expected := []*dto.Metric{ + {Label: makeLabels("label_one", "a", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(180)}}, + {Label: makeLabels("label_one", "a", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(100)}}, + {Label: makeLabels("label_one", "b", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}}, + } + require.ElementsMatch(t, expected, actual) + }) + + t.Run("group metrics by user and label_two", func(t *testing.T) { + desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_two"}, nil) + actual := collectMetrics(t, func(out chan prometheus.Metric) { + mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_two")) + }) + expected := []*dto.Metric{ + {Label: makeLabels("label_two", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(100)}}, + {Label: makeLabels("label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(80)}}, + {Label: makeLabels("label_two", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(60)}}, + {Label: makeLabels("label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(40)}}, + {Label: makeLabels("label_two", "b", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(0)}}, + {Label: makeLabels("label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}}, + } + require.ElementsMatch(t, expected, actual) + }) + + t.Run("group metrics by user and label_two, and skip zero value metrics", func(t *testing.T) { + desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_two"}, nil) + actual := collectMetrics(t, func(out chan prometheus.Metric) { + mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_two"), WithSkipZeroValueMetrics) + }) + expected := []*dto.Metric{ + {Label: makeLabels("label_two", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(100)}}, + {Label: makeLabels("label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(80)}}, + {Label: makeLabels("label_two", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(60)}}, + {Label: makeLabels("label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(40)}}, + {Label: makeLabels("label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}}, + } + require.ElementsMatch(t, expected, actual) + }) + + t.Run("group metrics by user, label_one and label_two", func(t *testing.T) { + desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_one", "label_two"}, nil) + actual := collectMetrics(t, func(out chan prometheus.Metric) { + mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_one", "label_two")) + }) + expected := []*dto.Metric{ + {Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(100)}}, + {Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(80)}}, + {Label: makeLabels("label_one", "b", "label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(0)}}, + {Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(60)}}, + {Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(40)}}, + {Label: makeLabels("label_one", "b", "label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(0)}}, + {Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(0)}}, + {Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(0)}}, + {Label: makeLabels("label_one", "b", "label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}}, + } + require.ElementsMatch(t, expected, actual) + }) + + t.Run("group metrics by user, label_one and label_two, and skip zero value metrics", func(t *testing.T) { + desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_one", "label_two"}, nil) + actual := collectMetrics(t, func(out chan prometheus.Metric) { + mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_one", "label_two"), WithSkipZeroValueMetrics) + }) + expected := []*dto.Metric{ + {Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(100)}}, + {Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(80)}}, + {Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(60)}}, + {Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(40)}}, + {Label: makeLabels("label_one", "b", "label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}}, + } + require.ElementsMatch(t, expected, actual) + }) +} + // TestSendSumOfGaugesPerTenantWithLabels tests to ensure multiple metrics for the same tenant with a matching label are // summed correctly. func TestSendSumOfGaugesPerTenantWithLabels(t *testing.T) {