diff --git a/charts/k8s-monitoring/charts/feature-integrations/docs/integrations/grafana.md b/charts/k8s-monitoring/charts/feature-integrations/docs/integrations/grafana.md index c03ff7bd8..8f61c6d63 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/docs/integrations/grafana.md +++ b/charts/k8s-monitoring/charts/feature-integrations/docs/integrations/grafana.md @@ -7,7 +7,7 @@ | Key | Type | Default | Description | |-----|------|---------|-------------| | fieldSelectors | list | `[]` | Discover Grafana instances based on field selectors. | -| labelSelectors | object | `{}` | Discover Grafana instances based on label selectors. | +| labelSelectors | object | `{"app.kubernetes.io/name":"grafana"}` | Discover Grafana instances based on label selectors. | | metrics.portName | string | `"grafana"` | Name of the port to scrape metrics from. | | namespaces | list | `[]` | The namespaces to look for Grafana instances in. Will automatically look for Grafana instances in all namespaces unless specified here | diff --git a/charts/k8s-monitoring/charts/feature-integrations/docs/integrations/loki.md b/charts/k8s-monitoring/charts/feature-integrations/docs/integrations/loki.md index ec3a5dd72..46bf3411e 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/docs/integrations/loki.md +++ b/charts/k8s-monitoring/charts/feature-integrations/docs/integrations/loki.md @@ -7,17 +7,10 @@ | Key | Type | Default | Description | |-----|------|---------|-------------| | fieldSelectors | list | `[]` | Discover Loki instances based on field selectors. | -| labelSelectors | object | `{}` | Discover Loki instances based on label selectors. | +| labelSelectors | object | `{"app.kubernetes.io/name":["loki","enterprise-logs"]}` | Discover Loki instances based on label selectors. | | metrics.portName | string | `"http-metrics"` | Name of the port to scrape metrics from. | | namespaces | list | `[]` | Namespaces to look for Loki instances in. Will automatically look for Loki instances in all namespaces unless specified here | -### General Settings - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| jobLabel | string | `"integrations/loki"` | The value of the job label for scraped metrics and logs | -| name | string | `""` | Name for this Loki instance. | - ### Logs Settings | Key | Type | Default | Description | @@ -49,3 +42,9 @@ | Key | Type | Default | Description | |-----|------|---------|-------------| | metrics.scrapeInterval | string | `60s` | How frequently to scrape metrics from Loki. | + +### General Settings + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| name | string | `""` | Name for this Loki instance. | diff --git a/charts/k8s-monitoring/charts/feature-integrations/integrations/grafana-values.yaml b/charts/k8s-monitoring/charts/feature-integrations/integrations/grafana-values.yaml index d39edc922..b6472e4df 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/integrations/grafana-values.yaml +++ b/charts/k8s-monitoring/charts/feature-integrations/integrations/grafana-values.yaml @@ -18,7 +18,8 @@ fieldSelectors: [] # -- Discover Grafana instances based on label selectors. # @section -- Discovery Settings -labelSelectors: {} +labelSelectors: + app.kubernetes.io/name: grafana # Settings for metrics collection metrics: diff --git a/charts/k8s-monitoring/charts/feature-integrations/integrations/loki-values.yaml b/charts/k8s-monitoring/charts/feature-integrations/integrations/loki-values.yaml index ecaced09e..10c342cd9 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/integrations/loki-values.yaml +++ b/charts/k8s-monitoring/charts/feature-integrations/integrations/loki-values.yaml @@ -3,10 +3,6 @@ # @section -- General Settings name: "" -# -- The value of the job label for scraped metrics and logs -# @section -- General Settings -jobLabel: integrations/loki - # -- Namespaces to look for Loki instances in. # Will automatically look for Loki instances in all namespaces unless specified here # @section -- Discovery Settings @@ -18,7 +14,8 @@ fieldSelectors: [] # -- Discover Loki instances based on label selectors. # @section -- Discovery Settings -labelSelectors: {} +labelSelectors: + app.kubernetes.io/name: [loki, enterprise-logs] # Settings for metrics collection metrics: diff --git a/charts/k8s-monitoring/charts/feature-integrations/schema-mods/definitions/grafana-integration.schema.json b/charts/k8s-monitoring/charts/feature-integrations/schema-mods/definitions/grafana-integration.schema.json index f3c7c1ac3..1d0313bb2 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/schema-mods/definitions/grafana-integration.schema.json +++ b/charts/k8s-monitoring/charts/feature-integrations/schema-mods/definitions/grafana-integration.schema.json @@ -8,7 +8,12 @@ "type": "string" }, "labelSelectors": { - "type": "object" + "type": "object", + "properties": { + "app.kubernetes.io/name": { + "type": "string" + } + } }, "logs": { "type": "object", diff --git a/charts/k8s-monitoring/charts/feature-integrations/schema-mods/definitions/loki-integration.schema.json b/charts/k8s-monitoring/charts/feature-integrations/schema-mods/definitions/loki-integration.schema.json index 6fce2cf65..e8f738f86 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/schema-mods/definitions/loki-integration.schema.json +++ b/charts/k8s-monitoring/charts/feature-integrations/schema-mods/definitions/loki-integration.schema.json @@ -4,11 +4,16 @@ "fieldSelectors": { "type": "array" }, - "jobLabel": { - "type": "string" - }, "labelSelectors": { - "type": "object" + "type": "object", + "properties": { + "app.kubernetes.io/name": { + "type": "array", + "items": { + "type": "string" + } + } + } }, "logs": { "type": "object", diff --git a/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_grafana_logs.tpl b/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_grafana_logs.tpl index e0c7f5363..890ef6732 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_grafana_logs.tpl +++ b/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_grafana_logs.tpl @@ -19,28 +19,28 @@ {{- $valueList := list }} {{- if .namespaces }} {{- $labelList = append $labelList "__meta_kubernetes_namespace" -}} - {{- $valueList = append $valueList (printf "(%s)" (join "|" .namespaces)) -}} + {{- $valueList = append $valueList (printf "(?:%s)" (join "|" .namespaces)) -}} {{- end }} {{- range $k, $v := .labelSelectors }} {{- if kindIs "slice" $v }} {{- $labelList = append $labelList (include "pod_label" $k) -}} - {{- $valueList = append $valueList (printf "(%s)" (join "|" $v)) -}} + {{- $valueList = append $valueList (printf "(?:%s)" (join "|" $v)) -}} {{- else }} {{- $labelList = append $labelList (include "pod_label" $k) -}} - {{- $valueList = append $valueList $v -}} + {{- $valueList = append $valueList (printf "(?:%s)" $v) -}} {{- end }} {{- end }} rule { - source_labels = {{ $labelList | sortAlpha | toJson }} + source_labels = {{ $labelList | toJson }} separator = ";" - regex = {{ $valueList | sortAlpha | join ";" | quote }} + regex = {{ $valueList | join ";" | quote }} target_label = "job" replacement = "integrations/grafana" } rule { - source_labels = {{ $labelList | sortAlpha | toJson }} + source_labels = {{ $labelList | toJson }} separator = ";" - regex = {{ $valueList | sortAlpha | join ";" | quote }} + regex = {{ $valueList | join ";" | quote }} target_label = "instance" replacement = {{ $instance.name | quote }} } diff --git a/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_loki_logs.tpl b/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_loki_logs.tpl index f195fdcf5..6de3bd29e 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_loki_logs.tpl +++ b/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_loki_logs.tpl @@ -19,30 +19,43 @@ {{- $valueList := list }} {{- if .namespaces }} {{- $labelList = append $labelList "__meta_kubernetes_namespace" -}} - {{- $valueList = append $valueList (printf "(%s)" (join "|" .namespaces)) -}} + {{- $valueList = append $valueList (printf "(?:%s)" (join "|" .namespaces)) -}} {{- end }} {{- range $k, $v := .labelSelectors }} {{- if kindIs "slice" $v }} {{- $labelList = append $labelList (include "pod_label" $k) -}} - {{- $valueList = append $valueList (printf "(%s)" (join "|" $v)) -}} + {{- $valueList = append $valueList (printf "(?:%s)" (join "|" $v)) -}} {{- else }} {{- $labelList = append $labelList (include "pod_label" $k) -}} - {{- $valueList = append $valueList $v -}} + {{- $valueList = append $valueList (printf "(?:%s)" $v) -}} {{- end }} {{- end }} +// add static label of integration="loki" and instance="name" to pods that match the selector so they can be identified in the loki.process stages rule { - source_labels = {{ $labelList | sortAlpha | toJson }} + source_labels = {{ $labelList | toJson }} separator = ";" - regex = {{ $valueList | sortAlpha | join ";" | quote }} - target_label = "job" - replacement = "integrations/loki" + regex = {{ $valueList | join ";" | quote }} + target_label = "integration" + replacement = "loki" } rule { - source_labels = {{ $labelList | sortAlpha | toJson }} + source_labels = {{ $labelList | toJson }} separator = ";" - regex = {{ $valueList | sortAlpha | join ";" | quote }} + regex = {{ $valueList | join ";" | quote }} target_label = "instance" replacement = {{ $instance.name | quote }} +} +{{- $labelList = append $labelList "__meta_kubernetes_namespace" -}} +{{- $valueList = append $valueList "([^;]+)" -}} +{{- $labelList = append $labelList (include "pod_label" "component") -}} +{{- $valueList = append $valueList "([^;]+)" }} +// override the job label to be namespace/component so it aligns to the loki-mixin +rule { + source_labels = {{ $labelList | toJson }} + separator = ";" + regex = {{ $valueList | join ";" | quote }} + target_label = "job" + replacement = "$1/$2" } {{- end }} {{- end }} @@ -58,9 +71,9 @@ rule { {{- if .logs.enabled }} stage.match { {{- if $instance.namespaces }} - selector = "{job=\"integrations/loki\",instance=\"{{ $instance.name }}\",namespace=~\"{{ $instance.namespaces | join "|" }}\"}" + selector = "{integration=\"loki\",instance=\"{{ $instance.name }}\",namespace=~\"{{ $instance.namespaces | join "|" }}\"}" {{- else }} - selector = "{job=\"integrations/loki\",instance=\"{{ $instance.name }}\"}" + selector = "{integration=\"loki\",instance=\"{{ $instance.name }}\"}" {{- end }} // extract some of the fields from the log line diff --git a/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_loki_metrics.tpl b/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_loki_metrics.tpl index d2d90d315..a8a18e58e 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_loki_metrics.tpl +++ b/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_loki_metrics.tpl @@ -74,6 +74,13 @@ declare "loki_integration" { action = "keep" } + // the loki-mixin expects the job label to be namespace/component + rule { + source_labels = ["__meta_kubernetes_namespace","__meta_kubernetes_pod_label_app_kubernetes_io_component"] + separator = "/" + target_label = "job" + } + {{ include "commonRelabelings" . | nindent 4 }} } @@ -140,9 +147,36 @@ declare "loki_integration" { // drop metrics that match the drop_metrics regex rule { source_labels = ["__name__"] - regex = coalesce(argument.drop_metrics.value, "(^(go|process)_.+$)") + regex = coalesce(argument.drop_metrics.value, "") action = "drop" } + + // keep only metrics that match the keep_metrics regex + rule { + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(.+)") + action = "keep" + } + + // the loki-mixin expects the instance label to be the node name + rule { + source_labels = ["node"] + target_label = "instance" + replacement = "$1" + } + rule { + action = "labeldrop" + regex = "node" + } + + // set the memcached exporter container name from container="exporter" to container="memcached" + rule { + source_labels = ["component", "container"] + separator = ";" + regex = "memcached-[^;]+;exporter" + target_label = "container" + replacement = "memcached" + } } } {{- range $instance := $.Values.loki.instances }} @@ -177,7 +211,7 @@ loki_integration_discovery {{ include "helper.alloy_name" .name | quote }} { loki_integration_scrape {{ include "helper.alloy_name" .name | quote }} { targets = loki_integration_discovery.{{ include "helper.alloy_name" .name }}.output - job_label = {{ .jobLabel | quote }} + job_label = "integrations/loki" clustering = true {{- if $metricAllowList }} keep_metrics = {{ $metricAllowList | join "|" | quote }} diff --git a/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_mysql_logs.tpl b/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_mysql_logs.tpl index 2ddad7091..4c9d52aa4 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_mysql_logs.tpl +++ b/charts/k8s-monitoring/charts/feature-integrations/templates/_integration_mysql_logs.tpl @@ -17,28 +17,28 @@ {{- $valueList := list }} {{- if .logs.namespaces }} {{- $labelList = append $labelList "__meta_kubernetes_namespace" -}} - {{- $valueList = append $valueList (printf "(%s)" (join "|" .logs.namespaces)) -}} + {{- $valueList = append $valueList (printf "(?:%s)" (join "|" .logs.namespaces)) -}} {{- end }} {{- range $k, $v := .logs.labelSelectors }} {{- if kindIs "slice" $v }} {{- $labelList = append $labelList (include "pod_label" $k) -}} - {{- $valueList = append $valueList (printf "(%s)" (join "|" $v)) -}} + {{- $valueList = append $valueList (printf "(?:%s)" (join "|" $v)) -}} {{- else }} {{- $labelList = append $labelList (include "pod_label" $k) -}} - {{- $valueList = append $valueList $v -}} + {{- $valueList = append $valueList (printf "(?:%s)" $v) -}} {{- end }} {{- end }} rule { - source_labels = {{ $labelList | sortAlpha | toJson }} + source_labels = {{ $labelList | toJson }} separator = ";" - regex = {{ $valueList | sortAlpha | join ";" | quote }} + regex = {{ $valueList | join ";" | quote }} target_label = "integration" replacement = "mysql" } rule { - source_labels = {{ $labelList | sortAlpha | toJson }} + source_labels = {{ $labelList | toJson }} separator = ";" - regex = {{ $valueList | sortAlpha | join ";" | quote }} + regex = {{ $valueList | join ";" | quote }} target_label = "instance" replacement = {{ $instance.name | quote }} } diff --git a/charts/k8s-monitoring/charts/feature-integrations/tests/grafana_logs_test.yaml b/charts/k8s-monitoring/charts/feature-integrations/tests/grafana_logs_test.yaml index fca0fb872..afd1ab9e7 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/tests/grafana_logs_test.yaml +++ b/charts/k8s-monitoring/charts/feature-integrations/tests/grafana_logs_test.yaml @@ -23,14 +23,14 @@ tests: rule { source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] separator = ";" - regex = "grafana" + regex = "(?:grafana)" target_label = "job" replacement = "integrations/grafana" } rule { source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] separator = ";" - regex = "grafana" + regex = "(?:grafana)" target_label = "instance" replacement = "grafana" } @@ -84,7 +84,22 @@ tests: of: ConfigMap - matchRegex: path: data["logs.alloy"] - pattern: namespace=~\\"k8smon\\" + # The pattern should look like this, but since the regex is escaped, it will be a bit different + # rule { + # source_labels = ["__meta_kubernetes_namespace","__meta_kubernetes_pod_label_app_kubernetes_io_name"] + # separator = ";" + # regex = "(?:k8smon);(?:grafana)" + # target_label = "job" + # replacement = "integrations/grafana" + # } + pattern: |- + \s*rule { + \s* source_labels = \["__meta_kubernetes_namespace","__meta_kubernetes_pod_label_app_kubernetes_io_name"\] + \s* separator = ";" + \s* regex = "\(\?:k8smon\)\;\(\?:grafana\)" + \s* target_label = "job" + \s* replacement = "integrations/grafana" + \s*} - it: should allow you to set the timestamp format set: diff --git a/charts/k8s-monitoring/charts/feature-integrations/tests/loki_logs_test.yaml b/charts/k8s-monitoring/charts/feature-integrations/tests/loki_logs_test.yaml index 394b1c49e..177a42844 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/tests/loki_logs_test.yaml +++ b/charts/k8s-monitoring/charts/feature-integrations/tests/loki_logs_test.yaml @@ -20,27 +20,36 @@ tests: value: |- // Integration rules discovery.relabel "test" { + // add static label of integration="loki" and instance="name" to pods that match the selector so they can be identified in the loki.process stages rule { source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] separator = ";" - regex = "loki" - target_label = "job" - replacement = "integrations/loki" + regex = "(?:loki)" + target_label = "integration" + replacement = "loki" } rule { source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] separator = ";" - regex = "loki" + regex = "(?:loki)" target_label = "instance" replacement = "loki" } + // override the job label to be namespace/component so it aligns to the loki-mixin + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name","__meta_kubernetes_namespace","__meta_kubernetes_pod_label_component"] + separator = ";" + regex = "(?:loki);([^;]+);([^;]+)" + target_label = "job" + replacement = "$1/$2" + } } // Processing stages loki.process "test" { // Integration: Loki stage.match { - selector = "{job=\"integrations/loki\",instance=\"loki\"}" + selector = "{integration=\"loki\",instance=\"loki\"}" // extract some of the fields from the log line stage.logfmt { @@ -85,7 +94,22 @@ tests: of: ConfigMap - matchRegex: path: data["logs.alloy"] - pattern: namespace=~\\"k8smon\\" + # The pattern should look like this, but since the regex is escaped, it will be a bit different + # rule { + # source_labels = ["__meta_kubernetes_namespace","__meta_kubernetes_pod_label_app_kubernetes_io_name"] + # separator = ";" + # regex = "(?:k8smon);(?:loki)" + # target_label = "integration" + # replacement = "loki" + # } + pattern: |- + \s*rule { + \s* source_labels = \["__meta_kubernetes_namespace","__meta_kubernetes_pod_label_app_kubernetes_io_name"\] + \s* separator = ";" + \s* regex = "\(\?:k8smon\)\;\(\?:loki\)" + \s* target_label = "integration" + \s* replacement = "loki" + \s*} - it: should allow you to set the timestamp format set: diff --git a/charts/k8s-monitoring/charts/feature-integrations/tests/loki_metrics_test.yaml b/charts/k8s-monitoring/charts/feature-integrations/tests/loki_metrics_test.yaml index 1e8e0e87c..d595ec1c0 100644 --- a/charts/k8s-monitoring/charts/feature-integrations/tests/loki_metrics_test.yaml +++ b/charts/k8s-monitoring/charts/feature-integrations/tests/loki_metrics_test.yaml @@ -76,6 +76,13 @@ tests: action = "keep" } + // the loki-mixin expects the job label to be namespace/component + rule { + source_labels = ["__meta_kubernetes_namespace","__meta_kubernetes_pod_label_app_kubernetes_io_component"] + separator = "/" + target_label = "job" + } + rule { @@ -214,9 +221,36 @@ tests: // drop metrics that match the drop_metrics regex rule { source_labels = ["__name__"] - regex = coalesce(argument.drop_metrics.value, "(^(go|process)_.+$)") + regex = coalesce(argument.drop_metrics.value, "") action = "drop" } + + // keep only metrics that match the keep_metrics regex + rule { + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(.+)") + action = "keep" + } + + // the loki-mixin expects the instance label to be the node name + rule { + source_labels = ["node"] + target_label = "instance" + replacement = "$1" + } + rule { + action = "labeldrop" + regex = "node" + } + + // set the memcached exporter container name from container="exporter" to container="memcached" + rule { + source_labels = ["component", "container"] + separator = ";" + regex = "memcached-[^;]+;exporter" + target_label = "container" + replacement = "memcached" + } } } diff --git a/charts/k8s-monitoring/charts/feature-integrations/tests/mysql_logs_test.yaml b/charts/k8s-monitoring/charts/feature-integrations/tests/mysql_logs_test.yaml new file mode 100644 index 000000000..40b4909a7 --- /dev/null +++ b/charts/k8s-monitoring/charts/feature-integrations/tests/mysql_logs_test.yaml @@ -0,0 +1,108 @@ +--- +# yamllint disable rule:document-start rule:line-length rule:trailing-spaces rule:empty-lines +suite: Test MySQL integration +templates: + - configmap.yaml +tests: + - it: should add the MySQL processing stage to the logs config + set: + deployAsConfigMap: true + mysql: + instances: + - name: mysql + logs: + enabled: true + labelSelectors: + app.kubernetes.io/name: mysql + asserts: + - isKind: + of: ConfigMap + - equal: + path: data["logs.alloy"] + value: |- + // Integration rules + discovery.relabel "test" { + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + separator = ";" + regex = "(?:mysql)" + target_label = "integration" + replacement = "mysql" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + separator = ";" + regex = "(?:mysql)" + target_label = "instance" + replacement = "mysql" + } + } + + // Processing stages + loki.process "test" { + // Integration: MySQL + stage.match { + selector = "{integration=\"mysql\"}" + + stage.regex { + expression = `(?P.+) (?P[\d]+) \[(?P