Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Alerting: adapt sum of gauges to use options #584

Merged
merged 5 commits into from
Sep 23, 2024
Merged

Alerting: adapt sum of gauges to use options #584

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7e9bdbc
adapt sum of gauges to use options
titolins Sep 20, 2024
2bdcd0c
add WithLabels function back and revert unit tests
titolins Sep 23, 2024
bb25f7a
add new function unit tests
titolins Sep 23, 2024
1747a3d
Update metrics/tenant_registries.go
titolins Sep 23, 2024
33e026c
add changelog
titolins Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions metrics/tenant_registries.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,25 +219,29 @@ func (d MetricFamiliesPerTenant) SendSumOfGaugesWithLabels(out chan<- prometheus

// SendSumOfGaugesPerTenant provides metrics on a per-tenant basis.
// This function assumes that `tenant` is the first label on the provided metric Desc.
func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenant(out chan<- prometheus.Metric, desc *prometheus.Desc, gauge string) {
d.SendSumOfGaugesPerTenantWithLabels(out, desc, gauge)
}
func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenant(out chan<- prometheus.Metric, desc *prometheus.Desc, metric string, options ...MetricOption) {
opts := applyMetricOptions(options...)

// SendSumOfGaugesPerTenantWithLabels provides metrics with the provided label names on a per-tenant basis. This function assumes that `tenant` is the
// first label on the provided metric Desc
func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenantWithLabels(out chan<- prometheus.Metric, desc *prometheus.Desc, metric string, labelNames ...string) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To avoid breaking clients, can we keep this function and reimplement it using SendSumOfGaugesPerTenant + WithLabels option? (We can mark it as deprecated if we want to get rid of it)

We can then also revert changes in the unit tests, to prove that function still works as before.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, no breaking changes then. Added it now and reverted unit tests also 👍

for _, tenantEntry := range d {
if tenantEntry.tenant == "" {
continue
}

result := singleValueWithLabelsMap{}
tenantEntry.metrics.sumOfSingleValuesWithLabels(metric, labelNames, gaugeValue, result.aggregateFn, false)
tenantEntry.metrics.sumOfSingleValuesWithLabels(metric, opts.labelNames, gaugeValue, result.aggregateFn, opts.skipZeroValueMetrics)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you please add unit test showing that skipping of zero values works?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added now 👍

result.prependTenantLabelValue(tenantEntry.tenant)
result.WriteToMetricChannel(out, desc, prometheus.GaugeValue)
}
}

// SendSumOfGaugesPerTenantWithLabels provides metrics with the provided label names on a per-tenant basis. This function assumes that `tenant` is the
// first label on the provided metric Desc
//
// Deprecated: use SendSumOfGaugesPerTenant instead.
titolins marked this conversation as resolved.
Show resolved Hide resolved
func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenantWithLabels(out chan<- prometheus.Metric, desc *prometheus.Desc, metric string, labelNames ...string) {
d.SendSumOfGaugesPerTenant(out, desc, metric, WithLabels(labelNames...))
}

func (d MetricFamiliesPerTenant) sumOfSingleValuesWithLabels(metric string, fn func(*dto.Metric) float64, labelNames []string, skipZeroValue bool) singleValueWithLabelsMap {
result := singleValueWithLabelsMap{}
for _, tenantEntry := range d {
Expand Down
120 changes: 120 additions & 0 deletions metrics/tenant_registries_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,126 @@ func BenchmarkGetMetricsWithLabelNames(b *testing.B) {
}
}

// TestSendSumOfGaugesPerTenant tests to ensure multiple metrics for the same tenant with a matching label are
// summed correctly.
func TestSendSumOfGaugesPerTenant(t *testing.T) {
user1Reg := prometheus.NewRegistry()
user2Reg := prometheus.NewRegistry()
user3Reg := prometheus.NewRegistry()
user1Metric := promauto.With(user1Reg).NewGaugeVec(prometheus.GaugeOpts{Name: "test_metric"}, []string{"label_one", "label_two"})
user2Metric := promauto.With(user2Reg).NewGaugeVec(prometheus.GaugeOpts{Name: "test_metric"}, []string{"label_one", "label_two"})
user3Metric := promauto.With(user3Reg).NewGaugeVec(prometheus.GaugeOpts{Name: "test_metric"}, []string{"label_one", "label_two"})
user1Metric.WithLabelValues("a", "b").Set(100)
user1Metric.WithLabelValues("a", "c").Set(80)
user1Metric.WithLabelValues("b", "c").Set(0)
user2Metric.WithLabelValues("a", "b").Set(60)
user2Metric.WithLabelValues("a", "c").Set(40)
user2Metric.WithLabelValues("b", "c").Set(0)
user3Metric.WithLabelValues("a", "b").Set(0)
user3Metric.WithLabelValues("a", "c").Set(0)
user3Metric.WithLabelValues("b", "c").Set(20)

regs := NewTenantRegistries(log.NewNopLogger())
regs.AddTenantRegistry("user-1", user1Reg)
regs.AddTenantRegistry("user-2", user2Reg)
regs.AddTenantRegistry("user-3", user3Reg)
mf := regs.BuildMetricFamiliesPerTenant()

t.Run("group metrics by user and label_one", func(t *testing.T) {
desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_one"}, nil)
actual := collectMetrics(t, func(out chan prometheus.Metric) {
mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_one"))
})
expected := []*dto.Metric{
{Label: makeLabels("label_one", "a", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(180)}},
{Label: makeLabels("label_one", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "a", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(100)}},
{Label: makeLabels("label_one", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "a", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "b", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}},
}
require.ElementsMatch(t, expected, actual)
})

t.Run("group metrics by user and label_one, and skip zero value metrics", func(t *testing.T) {
desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_one"}, nil)
actual := collectMetrics(t, func(out chan prometheus.Metric) {
mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_one"), WithSkipZeroValueMetrics)
})
expected := []*dto.Metric{
{Label: makeLabels("label_one", "a", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(180)}},
{Label: makeLabels("label_one", "a", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(100)}},
{Label: makeLabels("label_one", "b", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}},
}
require.ElementsMatch(t, expected, actual)
})

t.Run("group metrics by user and label_two", func(t *testing.T) {
desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_two"}, nil)
actual := collectMetrics(t, func(out chan prometheus.Metric) {
mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_two"))
})
expected := []*dto.Metric{
{Label: makeLabels("label_two", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(100)}},
{Label: makeLabels("label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(80)}},
{Label: makeLabels("label_two", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(60)}},
{Label: makeLabels("label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(40)}},
{Label: makeLabels("label_two", "b", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}},
}
require.ElementsMatch(t, expected, actual)
})

t.Run("group metrics by user and label_two, and skip zero value metrics", func(t *testing.T) {
desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_two"}, nil)
actual := collectMetrics(t, func(out chan prometheus.Metric) {
mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_two"), WithSkipZeroValueMetrics)
})
expected := []*dto.Metric{
{Label: makeLabels("label_two", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(100)}},
{Label: makeLabels("label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(80)}},
{Label: makeLabels("label_two", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(60)}},
{Label: makeLabels("label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(40)}},
{Label: makeLabels("label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}},
}
require.ElementsMatch(t, expected, actual)
})

t.Run("group metrics by user, label_one and label_two", func(t *testing.T) {
desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_one", "label_two"}, nil)
actual := collectMetrics(t, func(out chan prometheus.Metric) {
mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_one", "label_two"))
})
expected := []*dto.Metric{
{Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(100)}},
{Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(80)}},
{Label: makeLabels("label_one", "b", "label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(60)}},
{Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(40)}},
{Label: makeLabels("label_one", "b", "label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(0)}},
{Label: makeLabels("label_one", "b", "label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}},
}
require.ElementsMatch(t, expected, actual)
})

t.Run("group metrics by user, label_one and label_two, and skip zero value metrics", func(t *testing.T) {
desc := prometheus.NewDesc("test_metric", "", []string{"user", "label_one", "label_two"}, nil)
actual := collectMetrics(t, func(out chan prometheus.Metric) {
mf.SendSumOfGaugesPerTenant(out, desc, "test_metric", WithLabels("label_one", "label_two"), WithSkipZeroValueMetrics)
})
expected := []*dto.Metric{
{Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(100)}},
{Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-1"), Gauge: &dto.Gauge{Value: proto.Float64(80)}},
{Label: makeLabels("label_one", "a", "label_two", "b", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(60)}},
{Label: makeLabels("label_one", "a", "label_two", "c", "user", "user-2"), Gauge: &dto.Gauge{Value: proto.Float64(40)}},
{Label: makeLabels("label_one", "b", "label_two", "c", "user", "user-3"), Gauge: &dto.Gauge{Value: proto.Float64(20)}},
}
require.ElementsMatch(t, expected, actual)
})
}

// TestSendSumOfGaugesPerTenantWithLabels tests to ensure multiple metrics for the same tenant with a matching label are
// summed correctly.
func TestSendSumOfGaugesPerTenantWithLabels(t *testing.T) {
Expand Down