Feat/integration updates (#1130)

* Aligned Loki Integration w/ Loki-Mixin * Default Grafana Integration Label Selector * Updated Tests for Non-Capturing Groups * Added MySQL Logs Test * Rebuilt * Updated Meta-Monitoring Example * Updated Integration Test * Rebuilt * Rebuilt
grafana · Jan 22, 2025 · 96562ed · 96562ed
1 parent c073190
commit 96562ed
Show file tree

Hide file tree

Showing 31 changed files with 1,220 additions and 135 deletions.
diff --git a/charts/k8s-monitoring/charts/feature-integrations/docs/integrations/grafana.md b/charts/k8s-monitoring/charts/feature-integrations/docs/integrations/grafana.md
@@ -7,7 +7,7 @@
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | fieldSelectors | list | `[]` | Discover Grafana instances based on field selectors. |
-| labelSelectors | object | `{}` | Discover Grafana instances based on label selectors. |
+| labelSelectors | object | `{"app.kubernetes.io/name":"grafana"}` | Discover Grafana instances based on label selectors. |
 | metrics.portName | string | `"grafana"` | Name of the port to scrape metrics from. |
 | namespaces | list | `[]` | The namespaces to look for Grafana instances in. Will automatically look for Grafana instances in all namespaces unless specified here |
 

diff --git a/charts/k8s-monitoring/charts/feature-integrations/docs/integrations/loki.md b/charts/k8s-monitoring/charts/feature-integrations/docs/integrations/loki.md
@@ -7,17 +7,10 @@
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | fieldSelectors | list | `[]` | Discover Loki instances based on field selectors. |
-| labelSelectors | object | `{}` | Discover Loki instances based on label selectors. |
+| labelSelectors | object | `{"app.kubernetes.io/name":["loki","enterprise-logs"]}` | Discover Loki instances based on label selectors. |
 | metrics.portName | string | `"http-metrics"` | Name of the port to scrape metrics from. |
 | namespaces | list | `[]` | Namespaces to look for Loki instances in. Will automatically look for Loki instances in all namespaces unless specified here |
 
-### General Settings
-
-| Key | Type | Default | Description |
-|-----|------|---------|-------------|
-| jobLabel | string | `"integrations/loki"` | The value of the job label for scraped metrics and logs |
-| name | string | `""` | Name for this Loki instance. |
-
 ### Logs Settings
 
 | Key | Type | Default | Description |
@@ -49,3 +42,9 @@
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | metrics.scrapeInterval | string | `60s` | How frequently to scrape metrics from Loki. |
+
+### General Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| name | string | `""` | Name for this Loki instance. |
diff --git a/charts/k8s-monitoring/charts/feature-integrations/integrations/grafana-values.yaml b/charts/k8s-monitoring/charts/feature-integrations/integrations/grafana-values.yaml
@@ -18,7 +18,8 @@ fieldSelectors: []
 
 # -- Discover Grafana instances based on label selectors.
 # @section -- Discovery Settings
-labelSelectors: {}
+labelSelectors:
+  app.kubernetes.io/name: grafana
 
 # Settings for metrics collection
 metrics:

diff --git a/charts/k8s-monitoring/charts/feature-integrations/integrations/loki-values.yaml b/charts/k8s-monitoring/charts/feature-integrations/integrations/loki-values.yaml
@@ -3,10 +3,6 @@
 # @section -- General Settings
 name: ""
 
-# -- The value of the job label for scraped metrics and logs
-# @section -- General Settings
-jobLabel: integrations/loki
-
 # -- Namespaces to look for Loki instances in.
 # Will automatically look for Loki instances in all namespaces unless specified here
 # @section -- Discovery Settings
@@ -18,7 +14,8 @@ fieldSelectors: []
 
 # -- Discover Loki instances based on label selectors.
 # @section -- Discovery Settings
-labelSelectors: {}
+labelSelectors:
+  app.kubernetes.io/name: [loki, enterprise-logs]
 
 # Settings for metrics collection
 metrics:

diff --git a/...oring/charts/feature-integrations/schema-mods/definitions/grafana-integration.schema.json b/...oring/charts/feature-integrations/schema-mods/definitions/grafana-integration.schema.json
@@ -8,7 +8,12 @@
             "type": "string"
         },
         "labelSelectors": {
-            "type": "object"
+            "type": "object",
+            "properties": {
+                "app.kubernetes.io/name": {
+                    "type": "string"
+                }
+            }
         },
         "logs": {
             "type": "object",

diff --git a/...nitoring/charts/feature-integrations/schema-mods/definitions/loki-integration.schema.json b/...nitoring/charts/feature-integrations/schema-mods/definitions/loki-integration.schema.json
@@ -4,11 +4,16 @@
         "fieldSelectors": {
             "type": "array"
         },
-        "jobLabel": {
-            "type": "string"
-        },
         "labelSelectors": {
-            "type": "object"
+            "type": "object",
+            "properties": {
+                "app.kubernetes.io/name": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
         },
         "logs": {
             "type": "object",

diff --git a/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_grafana_logs.tpl b/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_grafana_logs.tpl
@@ -19,28 +19,28 @@
         {{- $valueList := list }}
         {{- if .namespaces }}
           {{- $labelList = append $labelList "__meta_kubernetes_namespace" -}}
-          {{- $valueList = append $valueList (printf "(%s)" (join "|" .namespaces)) -}}
+          {{- $valueList = append $valueList (printf "(?:%s)" (join "|" .namespaces)) -}}
         {{- end }}
         {{- range $k, $v := .labelSelectors }}
           {{- if kindIs "slice" $v }}
             {{- $labelList = append $labelList (include "pod_label" $k) -}}
-            {{- $valueList = append $valueList (printf "(%s)" (join "|" $v)) -}}
+            {{- $valueList = append $valueList (printf "(?:%s)" (join "|" $v)) -}}
           {{- else }}
             {{- $labelList = append $labelList (include "pod_label" $k) -}}
-            {{- $valueList = append $valueList $v -}}
+            {{- $valueList = append $valueList (printf "(?:%s)" $v) -}}
           {{- end }}
         {{- end }}
 rule {
-  source_labels = {{ $labelList | sortAlpha | toJson }}
+  source_labels = {{ $labelList | toJson }}
   separator = ";"
-  regex = {{ $valueList | sortAlpha | join ";" | quote }}
+  regex = {{ $valueList | join ";" | quote }}
   target_label = "job"
   replacement = "integrations/grafana"
 }
 rule {
-  source_labels = {{ $labelList | sortAlpha | toJson }}
+  source_labels = {{ $labelList | toJson }}
   separator = ";"
-  regex = {{ $valueList | sortAlpha | join ";" | quote }}
+  regex = {{ $valueList | join ";" | quote }}
   target_label = "instance"
   replacement = {{ $instance.name | quote }}
 }

diff --git a/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_loki_logs.tpl b/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_loki_logs.tpl
@@ -19,30 +19,43 @@
         {{- $valueList := list }}
         {{- if .namespaces }}
           {{- $labelList = append $labelList "__meta_kubernetes_namespace" -}}
-          {{- $valueList = append $valueList (printf "(%s)" (join "|" .namespaces)) -}}
+          {{- $valueList = append $valueList (printf "(?:%s)" (join "|" .namespaces)) -}}
         {{- end }}
         {{- range $k, $v := .labelSelectors }}
           {{- if kindIs "slice" $v }}
             {{- $labelList = append $labelList (include "pod_label" $k) -}}
-            {{- $valueList = append $valueList (printf "(%s)" (join "|" $v)) -}}
+            {{- $valueList = append $valueList (printf "(?:%s)" (join "|" $v)) -}}
           {{- else }}
             {{- $labelList = append $labelList (include "pod_label" $k) -}}
-            {{- $valueList = append $valueList $v -}}
+            {{- $valueList = append $valueList (printf "(?:%s)" $v) -}}
           {{- end }}
         {{- end }}
+// add static label of integration="loki" and instance="name" to pods that match the selector so they can be identified in the loki.process stages
 rule {
-  source_labels = {{ $labelList | sortAlpha | toJson }}
+  source_labels = {{ $labelList | toJson }}
   separator = ";"
-  regex = {{ $valueList | sortAlpha | join ";" | quote }}
-  target_label = "job"
-  replacement = "integrations/loki"
+  regex = {{ $valueList | join ";" | quote }}
+  target_label = "integration"
+  replacement = "loki"
 }
 rule {
-  source_labels = {{ $labelList | sortAlpha | toJson }}
+  source_labels = {{ $labelList | toJson }}
   separator = ";"
-  regex = {{ $valueList | sortAlpha | join ";" | quote }}
+  regex = {{ $valueList | join ";" | quote }}
   target_label = "instance"
   replacement = {{ $instance.name | quote }}
+}
+{{- $labelList = append $labelList "__meta_kubernetes_namespace" -}}
+{{- $valueList = append $valueList "([^;]+)" -}}
+{{- $labelList = append $labelList (include "pod_label" "component") -}}
+{{- $valueList = append $valueList "([^;]+)" }}
+// override the job label to be namespace/component so it aligns to the loki-mixin
+rule {
+  source_labels = {{ $labelList | toJson }}
+  separator = ";"
+  regex = {{ $valueList | join ";" | quote }}
+  target_label = "job"
+  replacement = "$1/$2"
 }
       {{- end }}
     {{- end }}
@@ -58,9 +71,9 @@ rule {
         {{- if .logs.enabled }}
 stage.match {
   {{- if $instance.namespaces }}
-  selector = "{job=\"integrations/loki\",instance=\"{{ $instance.name }}\",namespace=~\"{{ $instance.namespaces | join "|" }}\"}"
+  selector = "{integration=\"loki\",instance=\"{{ $instance.name }}\",namespace=~\"{{ $instance.namespaces | join "|" }}\"}"
   {{- else }}
-  selector = "{job=\"integrations/loki\",instance=\"{{ $instance.name }}\"}"
+  selector = "{integration=\"loki\",instance=\"{{ $instance.name }}\"}"
   {{- end }}
 
   // extract some of the fields from the log line

diff --git a/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_loki_metrics.tpl b/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_loki_metrics.tpl
@@ -74,6 +74,13 @@ declare "loki_integration" {
         action = "keep"
       }
 
+      // the loki-mixin expects the job label to be namespace/component
+      rule {
+        source_labels = ["__meta_kubernetes_namespace","__meta_kubernetes_pod_label_app_kubernetes_io_component"]
+        separator = "/"
+        target_label = "job"
+      }
+
       {{ include "commonRelabelings" . | nindent 4 }}
     }
 
@@ -140,9 +147,36 @@ declare "loki_integration" {
       // drop metrics that match the drop_metrics regex
       rule {
         source_labels = ["__name__"]
-        regex = coalesce(argument.drop_metrics.value, "(^(go|process)_.+$)")
+        regex = coalesce(argument.drop_metrics.value, "")
         action = "drop"
       }
+
+      // keep only metrics that match the keep_metrics regex
+      rule {
+        source_labels = ["__name__"]
+        regex = coalesce(argument.keep_metrics.value, "(.+)")
+        action = "keep"
+      }
+
+      // the loki-mixin expects the instance label to be the node name
+      rule {
+        source_labels = ["node"]
+        target_label = "instance"
+        replacement = "$1"
+      }
+      rule {
+        action = "labeldrop"
+        regex = "node"
+      }
+
+      // set the memcached exporter container name from container="exporter" to container="memcached"
+      rule {
+        source_labels = ["component", "container"]
+        separator = ";"
+        regex = "memcached-[^;]+;exporter"
+        target_label = "container"
+        replacement = "memcached"
+      }
     }
   }
   {{- range $instance := $.Values.loki.instances }}
@@ -177,7 +211,7 @@ loki_integration_discovery {{ include "helper.alloy_name" .name | quote }} {
 
 loki_integration_scrape  {{ include "helper.alloy_name" .name | quote }} {
   targets = loki_integration_discovery.{{ include "helper.alloy_name" .name }}.output
-  job_label = {{ .jobLabel | quote }}
+  job_label = "integrations/loki"
   clustering = true
 {{- if $metricAllowList }}
   keep_metrics = {{ $metricAllowList | join "|" | quote }}

diff --git a/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_mysql_logs.tpl b/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_mysql_logs.tpl
@@ -17,28 +17,28 @@
       {{- $valueList := list }}
       {{- if .logs.namespaces }}
         {{- $labelList = append $labelList "__meta_kubernetes_namespace" -}}
-        {{- $valueList = append $valueList (printf "(%s)" (join "|" .logs.namespaces)) -}}
+        {{- $valueList = append $valueList (printf "(?:%s)" (join "|" .logs.namespaces)) -}}
       {{- end }}
       {{- range $k, $v := .logs.labelSelectors }}
         {{- if kindIs "slice" $v }}
           {{- $labelList = append $labelList (include "pod_label" $k) -}}
-          {{- $valueList = append $valueList (printf "(%s)" (join "|" $v)) -}}
+          {{- $valueList = append $valueList (printf "(?:%s)" (join "|" $v)) -}}
         {{- else }}
           {{- $labelList = append $labelList (include "pod_label" $k) -}}
-          {{- $valueList = append $valueList $v -}}
+          {{- $valueList = append $valueList (printf "(?:%s)" $v) -}}
         {{- end }}
       {{- end }}
 rule {
-  source_labels = {{ $labelList | sortAlpha | toJson }}
+  source_labels = {{ $labelList | toJson }}
   separator = ";"
-  regex = {{ $valueList | sortAlpha | join ";" | quote }}
+  regex = {{ $valueList | join ";" | quote }}
   target_label = "integration"
   replacement = "mysql"
 }
 rule {
-  source_labels = {{ $labelList | sortAlpha | toJson }}
+  source_labels = {{ $labelList | toJson }}
   separator = ";"
-  regex = {{ $valueList | sortAlpha | join ";" | quote }}
+  regex = {{ $valueList | join ";" | quote }}
   target_label = "instance"
   replacement = {{ $instance.name | quote }}
 }

diff --git a/charts/k8s-monitoring/charts/feature-integrations/tests/grafana_logs_test.yaml b/charts/k8s-monitoring/charts/feature-integrations/tests/grafana_logs_test.yaml
@@ -23,14 +23,14 @@ tests:
               rule {
                 source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
                 separator = ";"
-                regex = "grafana"
+                regex = "(?:grafana)"
                 target_label = "job"
                 replacement = "integrations/grafana"
               }
               rule {
                 source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
                 separator = ";"
-                regex = "grafana"
+                regex = "(?:grafana)"
                 target_label = "instance"
                 replacement = "grafana"
               }
@@ -84,7 +84,22 @@ tests:
           of: ConfigMap
       - matchRegex:
           path: data["logs.alloy"]
-          pattern: namespace=~\\"k8smon\\"
+          # The pattern should look like this, but since the regex is escaped, it will be a bit different
+          # rule {
+          #   source_labels = ["__meta_kubernetes_namespace","__meta_kubernetes_pod_label_app_kubernetes_io_name"]
+          #   separator = ";"
+          #   regex = "(?:k8smon);(?:grafana)"
+          #   target_label = "job"
+          #   replacement = "integrations/grafana"
+          # }
+          pattern: |-
+            \s*rule {
+            \s*  source_labels = \["__meta_kubernetes_namespace","__meta_kubernetes_pod_label_app_kubernetes_io_name"\]
+            \s*  separator = ";"
+            \s*  regex = "\(\?:k8smon\)\;\(\?:grafana\)"
+            \s*  target_label = "job"
+            \s*  replacement = "integrations/grafana"
+            \s*}
 
   - it: should allow you to set the timestamp format
     set: