Skip to content

Commit

Permalink
Alerting: adapt sum of gauges to use options (#584)
Browse files Browse the repository at this point in the history 
* adapt sum of gauges to use options

* add WithLabels function back and revert unit tests

* add new function unit tests

* Update metrics/tenant_registries.go

Co-authored-by: Peter Štibraný <pstibrany@gmail.com>

* add changelog

---------

Co-authored-by: Peter Štibraný <pstibrany@gmail.com>
  • Loading branch information
@titolins @pstibrany
titolins and pstibrany authored Sep 23, 2024
1 parent 560bb26 commit 1f324b4
Show file tree
Hide file tree
Showing 3 changed files with 132 additions and 7 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@
* [ENHANCEMENT] `services.FailureWatcher` can now be closed, which unregisters all service and manager listeners, and closes channel used to receive errors. #564
* [ENHANCEMENT] Runtimeconfig: support gzip-compressed files with `.gz` extension. #571
* [ENHANCEMENT] grpcclient: Support custom gRPC compressors. #583
* [ENHANCEMENT] Adapt `metrics.SendSumOfGaugesPerTenant` to use `metrics.MetricOption`. #584
* [CHANGE] Backoff: added `Backoff.ErrCause()` which is like `Backoff.Err()` but returns the context cause if backoff is terminated because the context has been canceled. #538
* [BUGFIX] spanlogger: Support multiple tenant IDs. #59
* [BUGFIX] Memberlist: fixed corrupted packets when sending compound messages with more than 255 messages or messages bigger than 64KB. #85
Expand Down
18 changes: 11 additions & 7 deletions metrics/tenant_registries.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,25 +219,29 @@ func (d MetricFamiliesPerTenant) SendSumOfGaugesWithLabels(out chan<- prometheus

// SendSumOfGaugesPerTenant provides metrics on a per-tenant basis.
// This function assumes that `tenant` is the first label on the provided metric Desc.
func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenant(out chan<- prometheus.Metric, desc *prometheus.Desc, gauge string) {
d.SendSumOfGaugesPerTenantWithLabels(out, desc, gauge)
}
func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenant(out chan<- prometheus.Metric, desc *prometheus.Desc, metric string, options ...MetricOption) {
opts := applyMetricOptions(options...)

// SendSumOfGaugesPerTenantWithLabels provides metrics with the provided label names on a per-tenant basis. This function assumes that `tenant` is the
// first label on the provided metric Desc
func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenantWithLabels(out chan<- prometheus.Metric, desc *prometheus.Desc, metric string, labelNames ...string) {
for _, tenantEntry := range d {
if tenantEntry.tenant == "" {
continue
}

result := singleValueWithLabelsMap{}
tenantEntry.metrics.sumOfSingleValuesWithLabels(metric, labelNames, gaugeValue, result.aggregateFn, false)
tenantEntry.metrics.sumOfSingleValuesWithLabels(metric, opts.labelNames, gaugeValue, result.aggregateFn, opts.skipZeroValueMetrics)
result.prependTenantLabelValue(tenantEntry.tenant)
result.WriteToMetricChannel(out, desc, prometheus.GaugeValue)
}
}

// SendSumOfGaugesPerTenantWithLabels provides metrics with the provided label names on a per-tenant basis. This function assumes that `tenant` is the
// first label on the provided metric Desc
//
// Deprecated: use SendSumOfGaugesPerTenant with WithLabels option instead.
func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenantWithLabels(out chan<- prometheus.Metric, desc *prometheus.Desc, metric string, labelNames ...string) {
d.SendSumOfGaugesPerTenant(out, desc, metric, WithLabels(labelNames...))
}

func (d MetricFamiliesPerTenant) sumOfSingleValuesWithLabels(metric string, fn func(*dto.Metric) float64, labelNames []string, skipZeroValue bool) singleValueWithLabelsMap {
result := singleValueWithLabelsMap{}
for _, tenantEntry := range d {
Expand Down
120 changes: 120 additions & 0 deletions metrics/tenant_registries_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,126 @@ func BenchmarkGetMetricsWithLabelNames(b *testing.B) {
}
}

// TestSendSumOfGaugesPerTenant tests to ensure multiple metrics for the same tenant with a matching label are
// summed correctly.
func TestSendSumOfGaugesPerTenant(t *testing.T) {
user1Reg := prometheus.NewRegistry()
user2Reg := prometheus.NewRegistry()
user3Reg := prometheus.NewRegistry()
user1Metric := promauto.With(user1Reg).NewGaugeVec(prometheus.GaugeOpts{Name: "test_metric"}, []string{"label_one", "label_two"})
user2Metric := promauto.With(user2Reg).NewGaugeVec(prometheus.GaugeOpts{Name: "test_metric"}, []string{"label_one", "label_two"})
user3Metric := promauto.With(user3Reg).NewGaugeVec(prometheus.GaugeOpts{Name: "test_metric"}, []string{"label_one", "label_two"})
user1Metric.WithLabelValues("a", "b").Set(100)
user1Metric.WithLabelValues("a", "c").Set(80)
user1Metric.WithLabelValues("b", "c").Set(0)
user2Metric.WithLabelValues("a", "b").Set(60)
user2Metric.WithLabelValues("a", "c").Set(40)
user2Metric.WithLabelValues("b", "c").Set(0)
user3Metric.WithLabelValues("a", "b").Set(0)
user3Metric.WithLabelValues("a", "c").Set(0)
user3Metric.WithLabelValues("b", "c").Set(20)

regs := NewTenantRegistries(log.NewNopLogger())
regs.AddTenantRegistry("user-1", user1Reg)
regs.AddTenantRegistry("user-2", user2Reg)
regs.AddTenantRegistry("user-3", user3Reg)
mf := regs.BuildMetricFamiliesPerTenant()

t.Run("group metrics by user and label_one", func(t *testing.T) {
desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_one"}, nil)
actual := collectMetrics(t, func(out chan prometheus.Metric) {
mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_one"))
})
expected := []*dto.Metric{
{Label: makeLabels("label_one", "a", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(180)}},
{Label: makeLabels("label_one", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "a", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(100)}},
{Label: makeLabels("label_one", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "a", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "b", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}},
}
require.ElementsMatch(t, expected, actual)
})

t.Run("group metrics by user and label_one, and skip zero value metrics", func(t *testing.T) {
desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_one"}, nil)
actual := collectMetrics(t, func(out chan prometheus.Metric) {
mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_one"), WithSkipZeroValueMetrics)
})
expected := []*dto.Metric{
{Label: makeLabels("label_one", "a", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(180)}},
{Label: makeLabels("label_one", "a", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(100)}},
{Label: makeLabels("label_one", "b", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}},
}
require.ElementsMatch(t, expected, actual)
})

t.Run("group metrics by user and label_two", func(t *testing.T) {
desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_two"}, nil)
actual := collectMetrics(t, func(out chan prometheus.Metric) {
mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_two"))
})
expected := []*dto.Metric{
{Label: makeLabels("label_two", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(100)}},
{Label: makeLabels("label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(80)}},
{Label: makeLabels("label_two", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(60)}},
{Label: makeLabels("label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(40)}},
{Label: makeLabels("label_two", "b", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}},
}
require.ElementsMatch(t, expected, actual)
})

t.Run("group metrics by user and label_two, and skip zero value metrics", func(t *testing.T) {
desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_two"}, nil)
actual := collectMetrics(t, func(out chan prometheus.Metric) {
mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_two"), WithSkipZeroValueMetrics)
})
expected := []*dto.Metric{
{Label: makeLabels("label_two", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(100)}},
{Label: makeLabels("label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(80)}},
{Label: makeLabels("label_two", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(60)}},
{Label: makeLabels("label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(40)}},
{Label: makeLabels("label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}},
}
require.ElementsMatch(t, expected, actual)
})

t.Run("group metrics by user, label_one and label_two", func(t *testing.T) {
desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_one", "label_two"}, nil)
actual := collectMetrics(t, func(out chan prometheus.Metric) {
mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_one", "label_two"))
})
expected := []*dto.Metric{
{Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(100)}},
{Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(80)}},
{Label: makeLabels("label_one", "b", "label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(60)}},
{Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(40)}},
{Label: makeLabels("label_one", "b", "label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "b", "label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}},
}
require.ElementsMatch(t, expected, actual)
})

t.Run("group metrics by user, label_one and label_two, and skip zero value metrics", func(t *testing.T) {
desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_one", "label_two"}, nil)
actual := collectMetrics(t, func(out chan prometheus.Metric) {
mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_one", "label_two"), WithSkipZeroValueMetrics)
})
expected := []*dto.Metric{
{Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(100)}},
{Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(80)}},
{Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(60)}},
{Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(40)}},
{Label: makeLabels("label_one", "b", "label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}},
}
require.ElementsMatch(t, expected, actual)
})
}

// TestSendSumOfGaugesPerTenantWithLabels tests to ensure multiple metrics for the same tenant with a matching label are
// summed correctly.
func TestSendSumOfGaugesPerTenantWithLabels(t *testing.T) {
Expand Down

0 comments on commit 1f324b4

Please sign in to comment.