From 8f1be0e86b0ced53e73cb30d228aa736b1380d89 Mon Sep 17 00:00:00 2001 From: Piotr <17101802+thampiotr@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:10:00 +0100 Subject: [PATCH] Support extra label selectors on mimir.rules.kubernetes (#1773) * support extra labels on mimir rules queries * Add tests and expose query matchers to users. * Docs * Changelog * Apply suggestions from code review Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --------- Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- CHANGELOG.md | 5 + .../mimir/mimir.rules.kubernetes.md | 132 +++++++++----- .../mimir/rules/kubernetes/events.go | 103 +++++++++-- .../mimir/rules/kubernetes/events_test.go | 161 ++++++++++++++---- .../component/mimir/rules/kubernetes/rules.go | 25 +-- .../mimir/rules/kubernetes/rules_test.go | 94 ++++++++-- .../component/mimir/rules/kubernetes/types.go | 46 +++++ 7 files changed, 447 insertions(+), 119 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9e0be81fb..4511af790d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,11 @@ Main (unreleased) - Add the function `path_join` to the stdlib. (@wildum) - Add support to `loki.source.syslog` for the RFC3164 format ("BSD syslog"). (@sushain97) +### Enhancements + +- The `mimir.rules.kubernetes` component now supports adding extra label matchers + to all queries discovered via `PrometheusRule` CRDs. (@thampiotr) + ### Bugfixes - Update windows_exporter from v0.27.2 vo v0.27.3: (@jkroepke) diff --git a/docs/sources/reference/components/mimir/mimir.rules.kubernetes.md b/docs/sources/reference/components/mimir/mimir.rules.kubernetes.md index 07e87a2fca..8652a3b35c 100644 --- a/docs/sources/reference/components/mimir/mimir.rules.kubernetes.md +++ b/docs/sources/reference/components/mimir/mimir.rules.kubernetes.md @@ -50,25 +50,25 @@ mimir.rules.kubernetes "LABEL" { `mimir.rules.kubernetes` supports the following arguments: -Name | Type | Description | Default | Required --------------------------|---------------------|--------------------------------------------------------------------------------------------------|---------------|--------- -`address` | `string` | URL of the Mimir ruler. | | yes -`tenant_id` | `string` | Mimir tenant ID. | | no -`use_legacy_routes` | `bool` | Whether to use [deprecated][gem-2_2] ruler API endpoints. | false | no -`prometheus_http_prefix` | `string` | Path prefix for [Mimir's Prometheus endpoint][gem-path-prefix]. | `/prometheus` | no -`sync_interval` | `duration` | Amount of time between reconciliations with Mimir. | "5m" | no -`mimir_namespace_prefix` | `string` | Prefix used to differentiate multiple {{< param "PRODUCT_NAME" >}} deployments. | "alloy" | no -`bearer_token_file` | `string` | File containing a bearer token to authenticate with. | | no -`bearer_token` | `secret` | Bearer token to authenticate with. | | no -`enable_http2` | `bool` | Whether HTTP2 is supported for requests. | `true` | no -`follow_redirects` | `bool` | Whether redirects returned by the server should be followed. | `true` | no -`proxy_url` | `string` | HTTP proxy to send requests through. | | no -`no_proxy` | `string` | Comma-separated list of IP addresses, CIDR notations, and domain names to exclude from proxying. | | no -`proxy_from_environment` | `bool` | Use the proxy URL indicated by environment variables. | `false` | no -`proxy_connect_header` | `map(list(secret))` | Specifies headers to send to proxies during CONNECT requests. | | no -`external_labels` | `map(string)` | Labels to add to each rule. | `{}` | no - - At most, one of the following can be provided: +| Name | Type | Description | Default | Required | +|--------------------------|---------------------|--------------------------------------------------------------------------------------------------|---------------|----------| +| `address` | `string` | URL of the Mimir ruler. | | yes | +| `tenant_id` | `string` | Mimir tenant ID. | | no | +| `use_legacy_routes` | `bool` | Whether to use [deprecated][gem-2_2] ruler API endpoints. | false | no | +| `prometheus_http_prefix` | `string` | Path prefix for [Mimir's Prometheus endpoint][gem-path-prefix]. | `/prometheus` | no | +| `sync_interval` | `duration` | Amount of time between reconciliations with Mimir. | "5m" | no | +| `mimir_namespace_prefix` | `string` | Prefix used to differentiate multiple {{< param "PRODUCT_NAME" >}} deployments. | "alloy" | no | +| `bearer_token_file` | `string` | File containing a bearer token to authenticate with. | | no | +| `bearer_token` | `secret` | Bearer token to authenticate with. | | no | +| `enable_http2` | `bool` | Whether HTTP2 is supported for requests. | `true` | no | +| `follow_redirects` | `bool` | Whether redirects returned by the server should be followed. | `true` | no | +| `proxy_url` | `string` | HTTP proxy to send requests through. | | no | +| `no_proxy` | `string` | Comma-separated list of IP addresses, CIDR notations, and domain names to exclude from proxying. | | no | +| `proxy_from_environment` | `bool` | Use the proxy URL indicated by environment variables. | `false` | no | +| `proxy_connect_header` | `map(list(secret))` | Specifies headers to send to proxies during CONNECT requests. | | no | +| `external_labels` | `map(string)` | Labels to add to each rule. | `{}` | no | + +At most, one of the following can be provided: - [`bearer_token` argument](#arguments). - [`bearer_token_file` argument](#arguments). - [`basic_auth` block][basic_auth]. @@ -105,17 +105,19 @@ This is useful if you configure Mimir to use a different [prefix][gem-path-prefi The following blocks are supported inside the definition of `mimir.rules.kubernetes`: -Hierarchy | Block | Description | Required --------------------------------------------|------------------------|----------------------------------------------------------|--------- -rule_namespace_selector | [label_selector][] | Label selector for `Namespace` resources. | no -rule_namespace_selector > match_expression | [match_expression][] | Label match expression for `Namespace` resources. | no -rule_selector | [label_selector][] | Label selector for `PrometheusRule` resources. | no -rule_selector > match_expression | [match_expression][] | Label match expression for `PrometheusRule` resources. | no -basic_auth | [basic_auth][] | Configure basic_auth for authenticating to the endpoint. | no -authorization | [authorization][] | Configure generic authorization to the endpoint. | no -oauth2 | [oauth2][] | Configure OAuth2 for authenticating to the endpoint. | no -oauth2 > tls_config | [tls_config][] | Configure TLS settings for connecting to the endpoint. | no -tls_config | [tls_config][] | Configure TLS settings for connecting to the endpoint. | no +| Hierarchy | Block | Description | Required | +|--------------------------------------------|--------------------------|----------------------------------------------------------|----------| +| extra_query_matchers | [extra_query_matchers][] | Additional label matchers to add to each query. | no | +| extra_query_matchers > matcher | [matcher][] | A label matcher to add to query. | no | +| rule_namespace_selector | [label_selector][] | Label selector for `Namespace` resources. | no | +| rule_namespace_selector > match_expression | [match_expression][] | Label match expression for `Namespace` resources. | no | +| rule_selector | [label_selector][] | Label selector for `PrometheusRule` resources. | no | +| rule_selector > match_expression | [match_expression][] | Label match expression for `PrometheusRule` resources. | no | +| basic_auth | [basic_auth][] | Configure basic_auth for authenticating to the endpoint. | no | +| authorization | [authorization][] | Configure generic authorization to the endpoint. | no | +| oauth2 | [oauth2][] | Configure OAuth2 for authenticating to the endpoint. | no | +| oauth2 > tls_config | [tls_config][] | Configure TLS settings for connecting to the endpoint. | no | +| tls_config | [tls_config][] | Configure TLS settings for connecting to the endpoint. | no | The `>` symbol indicates deeper levels of nesting. For example, `oauth2 > tls_config` refers to a `tls_config` block defined inside @@ -127,6 +129,28 @@ an `oauth2` block. [tls_config]: #tls_config-block [label_selector]: #label_selector-block [match_expression]: #match_expression-block +[extra_query_matchers]: #extra_query_matchers-block +[matcher]: #matcher-block + +### extra_query_matchers block + +The `extra_query_matchers` block has no attributes. It contains zero or more [matcher][] blocks. +These blocks allow you to add extra label matchers to all queries that are discovered by `mimir.rules.kubernetes` +component. The algorithm of adding the label matchers to queries is the same as the one provided by +`promtool promql label-matchers set` command in [promtool](https://github.com/prometheus/prometheus/tree/main/cmd/promtool). + +### matcher block + +The `matcher` block describes a label matcher that will be added to each query found in `PrometheusRule` CRDs. + +The following arguments are supported: + +| Name | Type | Description | Default | Required | +|--------------|----------|----------------------------------------------------|---------|----------| +| `name` | `string` | Name of the label to match. | | yes | +| `match_type` | `string` | The type of match. One of `=`, `!=`, `=~` and `!~`. | | yes | +| `value` | `string` | Value of the label to match. | | yes | + ### label_selector block @@ -134,9 +158,9 @@ The `label_selector` block describes a Kubernetes label selector for rule or nam The following arguments are supported: -Name | Type | Description | Default | Required ----------------|---------------|---------------------------------------------------|-----------------------------|--------- -`match_labels` | `map(string)` | Label keys and values used to discover resources. | `{}` | yes +| Name | Type | Description | Default | Required | +|----------------|---------------|---------------------------------------------------|---------|----------| +| `match_labels` | `map(string)` | Label keys and values used to discover resources. | `{}` | yes | When the `match_labels` argument is empty, all resources will be matched. @@ -146,11 +170,11 @@ The `match_expression` block describes a Kubernetes label match expression for r The following arguments are supported: -Name | Type | Description | Default | Required ------------|----------------|----------------------------------------------------|---------|--------- -`key` | `string` | The label name to match against. | | yes -`operator` | `string` | The operator to use when matching. | | yes -`values` | `list(string)` | The values used when matching. | | no +| Name | Type | Description | Default | Required | +|------------|----------------|------------------------------------|---------|----------| +| `key` | `string` | The label name to match against. | | yes | +| `operator` | `string` | The operator to use when matching. | | yes | +| `values` | `list(string)` | The values used when matching. | | no | The `operator` argument should be one of the following strings: @@ -204,13 +228,13 @@ actually exist. ## Debug metrics -Metric Name | Type | Description -----------------------------------------------|-------------|------------------------------------------------------------------------- -`mimir_rules_config_updates_total` | `counter` | Number of times the configuration has been updated. -`mimir_rules_events_total` | `counter` | Number of events processed, partitioned by event type. -`mimir_rules_events_failed_total` | `counter` | Number of events that failed to be processed, partitioned by event type. -`mimir_rules_events_retried_total` | `counter` | Number of events that were retried, partitioned by event type. -`mimir_rules_client_request_duration_seconds` | `histogram` | Duration of requests to the Mimir API. +| Metric Name | Type | Description | +|-----------------------------------------------|-------------|--------------------------------------------------------------------------| +| `mimir_rules_config_updates_total` | `counter` | Number of times the configuration has been updated. | +| `mimir_rules_events_total` | `counter` | Number of events processed, partitioned by event type. | +| `mimir_rules_events_failed_total` | `counter` | Number of events that failed to be processed, partitioned by event type. | +| `mimir_rules_events_retried_total` | `counter` | Number of events that were retried, partitioned by event type. | +| `mimir_rules_client_request_duration_seconds` | `histogram` | Duration of requests to the Mimir API. | ## Example @@ -253,6 +277,24 @@ mimir.rules.kubernetes "default" { } ``` +This example adds label matcher `{cluster=~"prod-.*"}` to all the queries discovered by `mimir.rules.kubernetes`. + +```alloy +mimir.rules.kubernetes "default" { + address = "GRAFANA_CLOUD_METRICS_URL" + extra_query_matchers { + matcher { + name = "cluster" + match_type = "=~" + value = "prod-.*" + } + } +} +``` + +If a query in the form of `up != 1` is found in `PrometheusRule` CRDs, +it will be modified to `up{cluster=~"prod-.*"} != 1` before sending it to Mimir. + The following example is an RBAC configuration for Kubernetes. It authorizes {{< param "PRODUCT_NAME" >}} to query the Kubernetes REST API: ```yaml diff --git a/internal/component/mimir/rules/kubernetes/events.go b/internal/component/mimir/rules/kubernetes/events.go index 9d16d77618..800d86bdd4 100644 --- a/internal/component/mimir/rules/kubernetes/events.go +++ b/internal/component/mimir/rules/kubernetes/events.go @@ -9,17 +9,20 @@ import ( "time" "github.com/go-kit/log" - "github.com/grafana/alloy/internal/component/common/kubernetes" - "github.com/grafana/alloy/internal/mimir/client" - "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/hashicorp/go-multierror" promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" promListers "github.com/prometheus-operator/prometheus-operator/pkg/client/listers/monitoring/v1" + promlabels "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/rulefmt" + "github.com/prometheus/prometheus/promql/parser" "k8s.io/apimachinery/pkg/labels" coreListers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/yaml" // Used for CRD compatibility instead of gopkg.in/yaml.v2 + + "github.com/grafana/alloy/internal/component/common/kubernetes" + "github.com/grafana/alloy/internal/mimir/client" + "github.com/grafana/alloy/internal/runtime/logging/level" ) const ( @@ -31,13 +34,14 @@ type eventProcessor struct { stopChan chan struct{} health healthReporter - mimirClient client.Interface - namespaceLister coreListers.NamespaceLister - ruleLister promListers.PrometheusRuleLister - namespaceSelector labels.Selector - ruleSelector labels.Selector - namespacePrefix string - externalLabels map[string]string + mimirClient client.Interface + namespaceLister coreListers.NamespaceLister + ruleLister promListers.PrometheusRuleLister + namespaceSelector labels.Selector + ruleSelector labels.Selector + namespacePrefix string + externalLabels map[string]string + extraQueryMatchers *ExtraQueryMatchers metrics *metrics logger log.Logger @@ -183,14 +187,27 @@ func (e *eventProcessor) desiredStateFromKubernetes() (kubernetes.RuleGroupsByNa } if len(e.externalLabels) > 0 { - for _, rule_group := range groups { + for _, ruleGroup := range groups { // Refer to the slice element via its index, // to make sure we mutate on the original and not a copy. - for i := range rule_group.Rules { - if rule_group.Rules[i].Labels == nil { - rule_group.Rules[i].Labels = make(map[string]string, len(e.externalLabels)) + for i := range ruleGroup.Rules { + if ruleGroup.Rules[i].Labels == nil { + ruleGroup.Rules[i].Labels = make(map[string]string, len(e.externalLabels)) + } + maps.Copy(ruleGroup.Rules[i].Labels, e.externalLabels) + } + } + } + + if e.extraQueryMatchers != nil { + for _, ruleGroup := range groups { + for i := range ruleGroup.Rules { + query := ruleGroup.Rules[i].Expr.Value + newQuery, err := addMatchersToQuery(query, e.extraQueryMatchers.Matchers) + if err != nil { + level.Error(e.logger).Log("msg", "failed to add labels to PrometheusRule query", "query", query, "err", err) } - maps.Copy(rule_group.Rules[i].Labels, e.externalLabels) + ruleGroup.Rules[i].Expr.Value = newQuery } } } @@ -202,6 +219,62 @@ func (e *eventProcessor) desiredStateFromKubernetes() (kubernetes.RuleGroupsByNa return desiredState, nil } +func addMatchersToQuery(query string, matchers []Matcher) (string, error) { + var err error + for _, s := range matchers { + query, err = labelsSetPromQL(query, s.MatchType, s.Name, s.Value) + if err != nil { + return "", err + } + } + return query, nil +} + +// Lifted from: https://github.com/prometheus/prometheus/blob/79a6238e195ecc1c20937036c1e3b4e3bdaddc49/cmd/promtool/main.go#L1242 +func labelsSetPromQL(query, labelMatchType, name, value string) (string, error) { + expr, err := parser.ParseExpr(query) + if err != nil { + return query, err + } + + var matchType promlabels.MatchType + switch labelMatchType { + case parser.ItemType(parser.EQL).String(): + matchType = promlabels.MatchEqual + case parser.ItemType(parser.NEQ).String(): + matchType = promlabels.MatchNotEqual + case parser.ItemType(parser.EQL_REGEX).String(): + matchType = promlabels.MatchRegexp + case parser.ItemType(parser.NEQ_REGEX).String(): + matchType = promlabels.MatchNotRegexp + default: + return query, fmt.Errorf("invalid label match type: %s", labelMatchType) + } + + parser.Inspect(expr, func(node parser.Node, path []parser.Node) error { + if n, ok := node.(*parser.VectorSelector); ok { + var found bool + for i, l := range n.LabelMatchers { + if l.Name == name { + n.LabelMatchers[i].Type = matchType + n.LabelMatchers[i].Value = value + found = true + } + } + if !found { + n.LabelMatchers = append(n.LabelMatchers, &promlabels.Matcher{ + Type: matchType, + Name: name, + Value: value, + }) + } + } + return nil + }) + + return expr.String(), nil +} + func convertCRDRuleGroupToRuleGroup(crd promv1.PrometheusRuleSpec) ([]rulefmt.RuleGroup, error) { buf, err := yaml.Marshal(crd) if err != nil { diff --git a/internal/component/mimir/rules/kubernetes/events_test.go b/internal/component/mimir/rules/kubernetes/events_test.go index 8bd09bd6b8..1b5cf82c03 100644 --- a/internal/component/mimir/rules/kubernetes/events_test.go +++ b/internal/component/mimir/rules/kubernetes/events_test.go @@ -8,8 +8,6 @@ import ( "time" "github.com/go-kit/log" - "github.com/grafana/alloy/internal/component/common/kubernetes" - mimirClient "github.com/grafana/alloy/internal/mimir/client" v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" promListers "github.com/prometheus-operator/prometheus-operator/pkg/client/listers/monitoring/v1" "github.com/prometheus/prometheus/model/rulefmt" @@ -23,6 +21,9 @@ import ( coreListers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + + "github.com/grafana/alloy/internal/component/common/kubernetes" + mimirClient "github.com/grafana/alloy/internal/mimir/client" ) type fakeMimirClient struct { @@ -86,17 +87,8 @@ func (m *fakeMimirClient) ListRules(_ context.Context, namespace string) (map[st } func TestEventLoop(t *testing.T) { - nsIndexer := cache.NewIndexer( - cache.DeletionHandlingMetaNamespaceKeyFunc, - cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, - ) - nsLister := coreListers.NewNamespaceLister(nsIndexer) - - ruleIndexer := cache.NewIndexer( - cache.DeletionHandlingMetaNamespaceKeyFunc, - cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, - ) - ruleLister := promListers.NewPrometheusRuleLister(ruleIndexer) + nsIndexer := testNamespaceIndexer() + ruleIndexer := testRuleIndexer() ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -131,8 +123,8 @@ func TestEventLoop(t *testing.T) { stopChan: make(chan struct{}), health: &fakeHealthReporter{}, mimirClient: newFakeMimirClient(), - namespaceLister: nsLister, - ruleLister: ruleLister, + namespaceLister: coreListers.NewNamespaceLister(nsIndexer), + ruleLister: promListers.NewPrometheusRuleLister(ruleIndexer), namespaceSelector: labels.Everything(), ruleSelector: labels.Everything(), namespacePrefix: "alloy", @@ -190,17 +182,8 @@ func TestEventLoop(t *testing.T) { } func TestAdditionalLabels(t *testing.T) { - nsIndexer := cache.NewIndexer( - cache.DeletionHandlingMetaNamespaceKeyFunc, - cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, - ) - nsLister := coreListers.NewNamespaceLister(nsIndexer) - - ruleIndexer := cache.NewIndexer( - cache.DeletionHandlingMetaNamespaceKeyFunc, - cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, - ) - ruleLister := promListers.NewPrometheusRuleLister(ruleIndexer) + nsIndexer := testNamespaceIndexer() + ruleIndexer := testRuleIndexer() ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -228,7 +211,7 @@ func TestAdditionalLabels(t *testing.T) { Alert: "alert2", Expr: intstr.FromString("expr2"), Labels: map[string]string{ - //This label should get overridden. + // This label should get overridden. "foo": "lalalala", }, }, @@ -243,8 +226,8 @@ func TestAdditionalLabels(t *testing.T) { stopChan: make(chan struct{}), health: &fakeHealthReporter{}, mimirClient: newFakeMimirClient(), - namespaceLister: nsLister, - ruleLister: ruleLister, + namespaceLister: coreListers.NewNamespaceLister(nsIndexer), + ruleLister: promListers.NewPrometheusRuleLister(ruleIndexer), namespaceSelector: labels.Everything(), ruleSelector: labels.Everything(), namespacePrefix: "alloy", @@ -298,3 +281,123 @@ func TestAdditionalLabels(t *testing.T) { require.YAMLEq(t, expectedRule, string(ruleBuf)) } } + +func TestExtraQueryMatchers(t *testing.T) { + nsIndexer := testNamespaceIndexer() + ruleIndexer := testRuleIndexer() + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "namespace", + UID: types.UID("33f8860c-bd06-4c0d-a0b1-a114d6b9937b"), + }, + } + + rule := &v1.PrometheusRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: "name", + Namespace: "namespace", + UID: types.UID("64aab764-c95e-4ee9-a932-cd63ba57e6cf"), + }, + Spec: v1.PrometheusRuleSpec{ + Groups: []v1.RuleGroup{ + { + Name: "group1", + Rules: []v1.Rule{ + { + Record: "record_rule_1", + Expr: intstr.FromString("sum by (namespace) (rate(success{\"job\"=\"bad\"}[10m]) / rate(total{}[10m]))"), + }, + { + Alert: "alert_1", + Expr: intstr.FromString("sum by (namespace) (rate(success{\"foo\"=\"bar\"}[10m]) / (rate(success{\"job\"!~\"bad\"}[10m]) + rate(failure[10m]))) < 0.995"), + }, + }, + }, + }, + }, + } + + processor := &eventProcessor{ + queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), + stopChan: make(chan struct{}), + health: &fakeHealthReporter{}, + mimirClient: newFakeMimirClient(), + namespaceLister: coreListers.NewNamespaceLister(nsIndexer), + ruleLister: promListers.NewPrometheusRuleLister(ruleIndexer), + namespaceSelector: labels.Everything(), + ruleSelector: labels.Everything(), + namespacePrefix: "alloy", + metrics: newMetrics(), + logger: log.With(log.NewLogfmtLogger(os.Stdout), "ts", log.DefaultTimestampUTC), + extraQueryMatchers: &ExtraQueryMatchers{Matchers: []Matcher{ + { + Name: "cluster", + MatchType: "=~", + Value: "prod-.*", + }, + { + Name: "job", + MatchType: "=", + Value: "good", + }, + }}, + } + + ctx := context.Background() + + // Do an initial sync of the Mimir ruler state before starting the event processing loop. + require.NoError(t, processor.syncMimir(ctx)) + go processor.run(ctx) + defer processor.stop() + + eventHandler := kubernetes.NewQueuedEventHandler(processor.logger, processor.queue) + + // Add a namespace and rule to kubernetes + require.NoError(t, nsIndexer.Add(ns)) + require.NoError(t, ruleIndexer.Add(rule)) + eventHandler.OnAdd(rule, false) + + // Wait for the rule to be added to mimir + rules := map[string][]rulefmt.RuleGroup{} + require.Eventually(t, func() bool { + var err error + rules, err = processor.mimirClient.ListRules(ctx, "") + require.NoError(t, err) + require.Equal(t, 1, len(rules)) + return len(rules) == 1 + }, 3*time.Second, 10*time.Millisecond) + + // The map of rules has only one element. + for ruleName, rule := range rules { + require.Equal(t, "alloy/namespace/name/64aab764-c95e-4ee9-a932-cd63ba57e6cf", ruleName) + + ruleBuf, err := yaml.Marshal(rule) + require.NoError(t, err) + + expectedRule := `- name: group1 + rules: + - expr: "sum by (namespace) (rate(success{cluster=~\"prod-.*\",job=\"good\"}[10m]) / rate(total{cluster=~\"prod-.*\",job=\"good\"}[10m]))" + record: record_rule_1 + - alert: alert_1 + expr: "sum by (namespace) (rate(success{cluster=~\"prod-.*\",foo=\"bar\",job=\"good\"}[10m]) / (rate(success{cluster=~\"prod-.*\",job=\"good\"}[10m]) + rate(failure{cluster=~\"prod-.*\",job=\"good\"}[10m]))) < 0.995" +` + require.YAMLEq(t, expectedRule, string(ruleBuf)) + } +} + +func testRuleIndexer() cache.Indexer { + ruleIndexer := cache.NewIndexer( + cache.DeletionHandlingMetaNamespaceKeyFunc, + cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, + ) + return ruleIndexer +} + +func testNamespaceIndexer() cache.Indexer { + nsIndexer := cache.NewIndexer( + cache.DeletionHandlingMetaNamespaceKeyFunc, + cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, + ) + return nsIndexer +} diff --git a/internal/component/mimir/rules/kubernetes/rules.go b/internal/component/mimir/rules/kubernetes/rules.go index bade96a8bc..d90263746b 100644 --- a/internal/component/mimir/rules/kubernetes/rules.go +++ b/internal/component/mimir/rules/kubernetes/rules.go @@ -445,18 +445,19 @@ func (c *Component) newEventProcessor(queue workqueue.RateLimitingInterface, sto maps.Copy(externalLabels, c.args.ExternalLabels) return &eventProcessor{ - queue: queue, - stopChan: stopChan, - health: c, - mimirClient: c.mimirClient, - namespaceLister: namespaceLister, - ruleLister: ruleLister, - namespaceSelector: c.namespaceSelector, - ruleSelector: c.ruleSelector, - namespacePrefix: c.args.MimirNameSpacePrefix, - metrics: c.metrics, - logger: c.log, - externalLabels: externalLabels, + queue: queue, + stopChan: stopChan, + health: c, + mimirClient: c.mimirClient, + namespaceLister: namespaceLister, + ruleLister: ruleLister, + namespaceSelector: c.namespaceSelector, + ruleSelector: c.ruleSelector, + namespacePrefix: c.args.MimirNameSpacePrefix, + metrics: c.metrics, + logger: c.log, + externalLabels: externalLabels, + extraQueryMatchers: c.args.ExtraQueryMatchers, } } diff --git a/internal/component/mimir/rules/kubernetes/rules_test.go b/internal/component/mimir/rules/kubernetes/rules_test.go index 132497ae08..602ce025ff 100644 --- a/internal/component/mimir/rules/kubernetes/rules_test.go +++ b/internal/component/mimir/rules/kubernetes/rules_test.go @@ -20,32 +20,90 @@ import ( "github.com/grafana/alloy/syntax" ) -func TestAlloyConfig(t *testing.T) { - var exampleAlloyConfig = ` +func TestAlloyConfigs(t *testing.T) { + var testCases = []struct { + name string + config string + expectedErrorContains string + }{ + { + name: "basic working config", + config: ` address = "GRAFANA_CLOUD_METRICS_URL" basic_auth { username = "GRAFANA_CLOUD_USER" password = "GRAFANA_CLOUD_API_KEY" } - external_labels = {"label1" = "value1"} -` - - var args Arguments - err := syntax.Unmarshal([]byte(exampleAlloyConfig), &args) - require.NoError(t, err) -} - -func TestBadAlloyConfig(t *testing.T) { - var exampleAlloyConfig = ` + external_labels = {"label1" = "value1"}`, + }, + { + name: "invalid http config", + config: ` address = "GRAFANA_CLOUD_METRICS_URL" bearer_token = "token" - bearer_token_file = "/path/to/file.token" -` + bearer_token_file = "/path/to/file.token"`, + expectedErrorContains: `at most one of basic_auth, authorization, oauth2, bearer_token & bearer_token_file must be configured`, + }, + { + name: "query matchers valid", + config: ` + address = "GRAFANA_CLOUD_METRICS_URL" + extra_query_matchers { + matcher { + name = "job" + match_type = "!=" + value = "bar" + } + matcher { + name = "namespace" + match_type = "=" + value = "all" + } + matcher { + name = "namespace" + match_type = "!~" + value = ".+" + } + matcher { + name = "cluster" + match_type = "=~" + value = "prod-.*" + } + } +`, + }, + { + name: "query matchers empty", + config: ` + address = "GRAFANA_CLOUD_METRICS_URL" + extra_query_matchers {}`, + }, + { + name: "query matchers invalid", + config: ` + address = "GRAFANA_CLOUD_METRICS_URL" + extra_query_matchers { + matcher { + name = "job" + match_type = "!!" + value = "bar" + } + }`, + expectedErrorContains: `invalid match type`, + }, + } - // Make sure the squashed HTTPClientConfig Validate function is being utilized correctly - var args Arguments - err := syntax.Unmarshal([]byte(exampleAlloyConfig), &args) - require.ErrorContains(t, err, "at most one of basic_auth, authorization, oauth2, bearer_token & bearer_token_file must be configured") + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var args Arguments + err := syntax.Unmarshal([]byte(tc.config), &args) + if tc.expectedErrorContains == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tc.expectedErrorContains) + } + }) + } } type fakeCluster struct{} diff --git a/internal/component/mimir/rules/kubernetes/types.go b/internal/component/mimir/rules/kubernetes/types.go index ecbc8fbb15..c872a0d8f1 100644 --- a/internal/component/mimir/rules/kubernetes/types.go +++ b/internal/component/mimir/rules/kubernetes/types.go @@ -1,13 +1,27 @@ package rules import ( + "errors" "fmt" + "slices" "time" + promlabels "github.com/prometheus/prometheus/model/labels" + "github.com/grafana/alloy/internal/component/common/config" "github.com/grafana/alloy/internal/component/common/kubernetes" ) +var ( + // This should contain all valid match types for extra query matchers. + validMatchTypes = []string{ + promlabels.MatchEqual.String(), + promlabels.MatchNotEqual.String(), + promlabels.MatchRegexp.String(), + promlabels.MatchNotRegexp.String(), + } +) + type Arguments struct { Address string `alloy:"address,attr"` TenantID string `alloy:"tenant_id,attr,optional"` @@ -17,6 +31,7 @@ type Arguments struct { SyncInterval time.Duration `alloy:"sync_interval,attr,optional"` MimirNameSpacePrefix string `alloy:"mimir_namespace_prefix,attr,optional"` ExternalLabels map[string]string `alloy:"external_labels,attr,optional"` + ExtraQueryMatchers *ExtraQueryMatchers `alloy:"extra_query_matchers,block,optional"` RuleSelector kubernetes.LabelSelector `alloy:"rule_selector,block,optional"` RuleNamespaceSelector kubernetes.LabelSelector `alloy:"rule_namespace_selector,block,optional"` @@ -42,7 +57,38 @@ func (args *Arguments) Validate() error { if args.MimirNameSpacePrefix == "" { return fmt.Errorf("mimir_namespace_prefix must not be empty") } + if err := args.ExtraQueryMatchers.Validate(); err != nil { + return err + } // We must explicitly Validate because HTTPClientConfig is squashed and it won't run otherwise return args.HTTPClientConfig.Validate() } + +type ExtraQueryMatchers struct { + Matchers []Matcher `alloy:"matcher,block,optional"` +} + +func (e *ExtraQueryMatchers) Validate() error { + if e == nil { + return nil + } + var errs error + for _, matcher := range e.Matchers { + errs = errors.Join(errs, matcher.Validate()) + } + return errs +} + +type Matcher struct { + Name string `alloy:"name,attr"` + Value string `alloy:"value,attr"` + MatchType string `alloy:"match_type,attr"` +} + +func (m Matcher) Validate() error { + if !slices.Contains(validMatchTypes, m.MatchType) { + return fmt.Errorf("invalid match type: %q", m.MatchType) + } + return nil +}