Skip to content

Commit 68bc50a

Browse files
petewallpetewall
authored
Add exclusion rules for prom operator objects and static label helpers (#1127)
* Add exclusion rules for prom operator objects and static label helpers Signed-off-by: Pete Wall <pete.wall@grafana.com> * Fix bad section name Signed-off-by: Pete Wall <pete.wall@grafana.com> * Add the integration label to the labels to keep list. It'll be dropped later, and it's required for the integration matcher later Signed-off-by: Pete Wall <pete.wall@grafana.com> --------- Signed-off-by: Pete Wall <pete.wall@grafana.com>
1 parent 430bcfa commit 68bc50a

File tree

61 files changed

+381
-140
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+381
-140
lines changed

‎charts/k8s-monitoring/charts/feature-pod-logs/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ Be sure perform actual integration testing in a live environment in the main [k8
4444
| extraLogProcessingStages | string | `""` | Stage blocks to be added to the loki.process component for pod logs. ([docs](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#blocks)) This value is templated so that you can refer to other values from this file. |
4545
| labels | object | `{"app_kubernetes_io_name":"app.kubernetes.io/name"}` | Log labels to set with values copied from the Kubernetes Pod labels. Format: `<log_label>: <kubernetes_label>`. |
4646
| labelsToKeep | list | `["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name"]` | The list of labels to keep on the logs, all other pipeline labels will be dropped. |
47+
| staticLabels | object | `{}` | Log labels to set with static values. |
48+
| staticLabelsFrom | object | `{}` | Log labels to set with static values, not quoted so it can reference config components. |
4749

4850
### Pod Discovery
4951

‎charts/k8s-monitoring/charts/feature-pod-logs/templates/_common_log_processing.alloy.tpl

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -37,28 +37,43 @@ loki.process "pod_logs" {
3737
]
3838
}
3939

40-
{{- if .Values.extraLogProcessingStages }}
41-
{{ tpl .Values.extraLogProcessingStages $ | indent 2 }}
42-
{{- end -}}
43-
44-
{{- /* the stage.structured_metadata block needs to be conditionalized because the support for enabling structured metadata can be disabled */ -}}
45-
{{- /* through the loki limits_conifg on a per-tenant basis, even if there are no values defined or there are values defined but it is disabled */ -}}
46-
{{- /* in Loki, the write will fail. */ -}}
47-
{{- if gt (len (keys .Values.structuredMetadata)) 0 }}
40+
{{- /* the stage.structured_metadata block needs to be conditionalized because the support for enabling structured metadata can be disabled */ -}}
41+
{{- /* through the loki limits_conifg on a per-tenant basis, even if there are no values defined or there are values defined but it is disabled */ -}}
42+
{{- /* in Loki, the write will fail. */ -}}
43+
{{- if .Values.structuredMetadata }}
4844
// set the structured metadata values
4945
stage.structured_metadata {
5046
values = {
51-
{{- range $key, $value := .Values.structuredMetadata }}
47+
{{- range $key, $value := .Values.structuredMetadata }}
5248
{{ $key | quote }} = {{ if $value }}{{ $value | quote }}{{ else }}{{ $key | quote }}{{ end }},
53-
{{- end }}
49+
{{- end }}
5450
}
5551
}
56-
{{- end }}
52+
{{- end }}
53+
{{- with .Values.labelsToKeep }}
5754

5855
// Only keep the labels that are defined in the `keepLabels` list.
5956
stage.label_keep {
60-
values = {{ .Values.labelsToKeep | toJson }}
57+
values = {{ append . "integration" | toJson }}
58+
}
59+
{{- end }}
60+
61+
{{- if or .Values.staticLabels .Values.staticLabelsFrom }}
62+
63+
stage.static_labels {
64+
values = {
65+
{{- range $key, $value := .Values.staticLabels }}
66+
{{ $key }} = {{ $value | quote }},
67+
{{- end }}
68+
{{- range $key, $value := .Values.staticLabelsFrom }}
69+
{{ $key }} = {{ $value }},
70+
{{- end }}
71+
}
6172
}
73+
{{- end }}
74+
{{- if .Values.extraLogProcessingStages }}
75+
{{ tpl .Values.extraLogProcessingStages $ | indent 2 }}
76+
{{- end }}
6277

6378
forward_to = argument.logs_destinations.value
6479
}

‎charts/k8s-monitoring/charts/feature-pod-logs/tests/default_test.yaml

Lines changed: 160 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ tests:
142142
143143
// Only keep the labels that are defined in the `keepLabels` list.
144144
stage.label_keep {
145-
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name"]
145+
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","integration"]
146146
}
147147
148148
forward_to = argument.logs_destinations.value
@@ -293,14 +293,13 @@ tests:
293293
294294
// Only keep the labels that are defined in the `keepLabels` list.
295295
stage.label_keep {
296-
values = ["k8s_container_name","k8s_namespace","k8s_pod_name","k8s_pod_label_app_kubernetes_io_name"]
296+
values = ["k8s_container_name","k8s_namespace","k8s_pod_name","k8s_pod_label_app_kubernetes_io_name","integration"]
297297
}
298298
299299
forward_to = argument.logs_destinations.value
300300
}
301301
}
302302
303-
304303
- it: should allow setting structured metadata
305304
set:
306305
deployAsConfigMap: true
@@ -450,9 +449,166 @@ tests:
450449
451450
// Only keep the labels that are defined in the `keepLabels` list.
452451
stage.label_keep {
453-
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name"]
452+
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","integration"]
454453
}
455454
456455
forward_to = argument.logs_destinations.value
457456
}
458457
}
458+
459+
- it: should allow setting static labels
460+
set:
461+
deployAsConfigMap: true
462+
staticLabels:
463+
region: central
464+
staticLabelsFrom:
465+
color: sys.env("COLOR")
466+
asserts:
467+
- isKind:
468+
of: ConfigMap
469+
- equal:
470+
path: data["module.alloy"]
471+
value: |-
472+
declare "pod_logs" {
473+
argument "logs_destinations" {
474+
comment = "Must be a list of log destinations where collected logs should be forwarded to"
475+
}
476+
477+
discovery.relabel "filtered_pods" {
478+
targets = discovery.kubernetes.pods.targets
479+
rule {
480+
source_labels = ["__meta_kubernetes_namespace"]
481+
action = "replace"
482+
target_label = "namespace"
483+
}
484+
rule {
485+
source_labels = ["__meta_kubernetes_pod_name"]
486+
action = "replace"
487+
target_label = "pod"
488+
}
489+
rule {
490+
source_labels = ["__meta_kubernetes_pod_container_name"]
491+
action = "replace"
492+
target_label = "container"
493+
}
494+
rule {
495+
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
496+
separator = "/"
497+
action = "replace"
498+
replacement = "$1"
499+
target_label = "job"
500+
}
501+
502+
// set the container runtime as a label
503+
rule {
504+
action = "replace"
505+
source_labels = ["__meta_kubernetes_pod_container_id"]
506+
regex = "^(\\S+):\\/\\/.+$"
507+
replacement = "$1"
508+
target_label = "tmp_container_runtime"
509+
}
510+
511+
// set the job label from the k8s.grafana.com/logs.job annotation if it exists
512+
rule {
513+
source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_logs_job"]
514+
regex = "(.+)"
515+
target_label = "job"
516+
}
517+
518+
// make all labels on the pod available to the pipeline as labels,
519+
// they are omitted before write to loki via stage.label_keep unless explicitly set
520+
rule {
521+
action = "labelmap"
522+
regex = "__meta_kubernetes_pod_label_(.+)"
523+
}
524+
525+
// make all annotations on the pod available to the pipeline as labels,
526+
// they are omitted before write to loki via stage.label_keep unless explicitly set
527+
rule {
528+
action = "labelmap"
529+
regex = "__meta_kubernetes_pod_annotation_(.+)"
530+
}
531+
}
532+
533+
discovery.kubernetes "pods" {
534+
role = "pod"
535+
selectors {
536+
role = "pod"
537+
field = "spec.nodeName=" + sys.env("HOSTNAME")
538+
}
539+
}
540+
541+
discovery.relabel "filtered_pods_with_paths" {
542+
targets = discovery.relabel.filtered_pods.output
543+
544+
rule {
545+
source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
546+
separator = "/"
547+
action = "replace"
548+
replacement = "/var/log/pods/*$1/*.log"
549+
target_label = "__path__"
550+
}
551+
}
552+
553+
local.file_match "pod_logs" {
554+
path_targets = discovery.relabel.filtered_pods_with_paths.output
555+
}
556+
557+
loki.source.file "pod_logs" {
558+
targets = local.file_match.pod_logs.targets
559+
forward_to = [loki.process.pod_logs.receiver]
560+
}
561+
562+
loki.process "pod_logs" {
563+
stage.match {
564+
selector = "{tmp_container_runtime=~\"containerd|cri-o\"}"
565+
// the cri processing stage extracts the following k/v pairs: log, stream, time, flags
566+
stage.cri {}
567+
568+
// Set the extract flags and stream values as labels
569+
stage.labels {
570+
values = {
571+
flags = "",
572+
stream = "",
573+
}
574+
}
575+
}
576+
577+
stage.match {
578+
selector = "{tmp_container_runtime=\"docker\"}"
579+
// the docker processing stage extracts the following k/v pairs: log, stream, time
580+
stage.docker {}
581+
582+
// Set the extract stream value as a label
583+
stage.labels {
584+
values = {
585+
stream = "",
586+
}
587+
}
588+
}
589+
590+
// Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster,
591+
// namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary
592+
// container runtime label as it is no longer needed.
593+
stage.label_drop {
594+
values = [
595+
"filename",
596+
"tmp_container_runtime",
597+
]
598+
}
599+
600+
// Only keep the labels that are defined in the `keepLabels` list.
601+
stage.label_keep {
602+
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","integration"]
603+
}
604+
605+
stage.static_labels {
606+
values = {
607+
region = "central",
608+
color = sys.env("COLOR"),
609+
}
610+
}
611+
612+
forward_to = argument.logs_destinations.value
613+
}
614+
}

‎charts/k8s-monitoring/charts/feature-pod-logs/values.schema.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,12 @@
6060
"namespaces": {
6161
"type": "array"
6262
},
63+
"staticLabels": {
64+
"type": "object"
65+
},
66+
"staticLabelsFrom": {
67+
"type": "object"
68+
},
6369
"structuredMetadata": {
6470
"type": "object"
6571
},

‎charts/k8s-monitoring/charts/feature-pod-logs/values.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,14 @@ extraDiscoveryRules: ""
3434
labels:
3535
app_kubernetes_io_name: app.kubernetes.io/name
3636

37+
# -- Log labels to set with static values.
38+
# @section -- Log Processing
39+
staticLabels: {}
40+
41+
# -- Log labels to set with static values, not quoted so it can reference config components.
42+
# @section -- Log Processing
43+
staticLabelsFrom: {}
44+
3745
# -- Log labels to set with values copied from the Kubernetes Pod annotations.
3846
# Format: `<log_label>: <kubernetes_annotation>`.
3947
# @section -- Log Processing

‎charts/k8s-monitoring/charts/feature-prometheus-operator-objects/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ Be sure perform actual integration testing in a live environment in the main [k8
7373
| Key | Type | Default | Description |
7474
|-----|------|---------|-------------|
7575
| podMonitors.enabled | bool | `true` | Enable discovery of Prometheus Operator PodMonitor objects. |
76+
| podMonitors.excludeNamespaces | list | `[]` | Which namespaces to not look for PodMonitor objects. |
7677
| podMonitors.extraDiscoveryRules | string | `""` | Rule blocks to be added to the prometheus.operator.podmonitors component for PodMonitors. These relabeling rules are applied pre-scrape against the targets from service discovery. The relabelings defined in the PodMonitor object are applied first, then these relabelings are applied. Before the scrape, any remaining target labels that start with `__` (i.e. `__meta_kubernetes*`) are dropped. ([docs](https://grafana.com/docs/alloy/latest/reference/components/discovery/discovery.relabel/#rule-block)) |
7778
| podMonitors.extraMetricProcessingRules | string | `""` | Rule blocks to be added to the prometheus.relabel component for PodMonitor objects. These relabeling rules are applied post-scrape against the metrics returned from the scraped target, no `__meta*` labels are present. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#rule-block)) |
7879
| podMonitors.maxCacheSize | string | `nil` | Sets the max_cache_size for cadvisor prometheus.relabel component. This should be at least 2x-5x your largest scrape target or samples appended rate. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#arguments)) Overrides global.maxCacheSize |
@@ -87,6 +88,7 @@ Be sure perform actual integration testing in a live environment in the main [k8
8788
| Key | Type | Default | Description |
8889
|-----|------|---------|-------------|
8990
| probes.enabled | bool | `true` | Enable discovery of Prometheus Operator Probe objects. |
91+
| probes.excludeNamespaces | list | `[]` | Which namespaces to not look for Probe objects. |
9092
| probes.extraDiscoveryRules | string | `""` | Rule blocks to be added to the prometheus.operator.probes component for Probes. These relabeling rules are applied pre-scrape against the targets from service discovery. The relabelings defined in the PodMonitor object are applied first, then these relabelings are applied. Before the scrape, any remaining target labels that start with `__` (i.e. `__meta_kubernetes*`) are dropped. ([docs](https://grafana.com/docs/alloy/latest/reference/components/discovery/discovery.relabel/#rule-block)) |
9193
| probes.extraMetricProcessingRules | string | `""` | Rule blocks to be added to the prometheus.relabel component for Probe objects. These relabeling rules are applied post-scrape against the metrics returned from the scraped target, no `__meta*` labels are present. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#rule-block)) |
9294
| probes.maxCacheSize | string | `nil` | Sets the max_cache_size for cadvisor prometheus.relabel component. This should be at least 2x-5x your largest scrape target or samples appended rate. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#arguments)) Overrides global.maxCacheSize |
@@ -101,6 +103,7 @@ Be sure perform actual integration testing in a live environment in the main [k8
101103
| Key | Type | Default | Description |
102104
|-----|------|---------|-------------|
103105
| serviceMonitors.enabled | bool | `true` | Enable discovery of Prometheus Operator ServiceMonitor objects. |
106+
| serviceMonitors.excludeNamespaces | list | `[]` | Which namespaces to not look for ServiceMonitor objects. |
104107
| serviceMonitors.extraDiscoveryRules | string | `""` | Rule blocks to be added to the prometheus.operator.probes component for Probes. These relabeling rules are applied pre-scrape against the targets from service discovery. The relabelings defined in the PodMonitor object are applied first, then these relabelings are applied. Before the scrape, any remaining target labels that start with `__` (i.e. `__meta_kubernetes*`) are dropped. ([docs](https://grafana.com/docs/alloy/latest/reference/components/discovery/discovery.relabel/#rule-block)) |
105108
| serviceMonitors.extraMetricProcessingRules | string | `""` | Rule blocks to be added to the prometheus.relabel component for ServiceMonitor objects. These relabeling rules are applied post-scrape against the metrics returned from the scraped target, no `__meta*` labels are present. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#rule-block)) |
106109
| serviceMonitors.maxCacheSize | string | `nil` | Sets the max_cache_size for cadvisor prometheus.relabel component. This should be at least 2x-5x your largest scrape target or samples appended rate. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#arguments)) Overrides global.maxCacheSize |

‎charts/k8s-monitoring/charts/feature-prometheus-operator-objects/templates/_pod_monitors.alloy.tpl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,13 @@ prometheus.operator.podmonitors "pod_monitors" {
1919
default_scrape_interval = {{ .Values.podMonitors.scrapeInterval | default .Values.global.scrapeInterval | quote }}
2020
}
2121

22+
{{- with .Values.podMonitors.excludeNamespaces }}
23+
rule {
24+
source_labels = ["__meta_kubernetes_namespace"]
25+
regex = {{ . | join "|" | quote }}
26+
action = "drop"
27+
}
28+
{{- end }}
2229
{{- if .Values.podMonitors.extraDiscoveryRules }}
2330
{{ .Values.podMonitors.extraDiscoveryRules | indent 2 }}
2431
{{- end }}

‎charts/k8s-monitoring/charts/feature-prometheus-operator-objects/templates/_probes.alloy.tpl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,13 @@ prometheus.operator.probes "pod_monitors" {
1919
default_scrape_interval = {{ .Values.probes.scrapeInterval | default .Values.global.scrapeInterval | quote }}
2020
}
2121

22+
{{- with .Values.probes.excludeNamespaces }}
23+
rule {
24+
source_labels = ["__meta_kubernetes_namespace"]
25+
regex = {{ . | join "|" | quote }}
26+
action = "drop"
27+
}
28+
{{- end }}
2229
{{- if .Values.probes.extraDiscoveryRules }}
2330
{{ .Values.probes.extraDiscoveryRules | indent 2 }}
2431
{{- end }}

‎charts/k8s-monitoring/charts/feature-prometheus-operator-objects/templates/_service_monitors.alloy.tpl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,13 @@ prometheus.operator.servicemonitors "service_monitors" {
1919
default_scrape_interval = {{ .Values.serviceMonitors.scrapeInterval | default .Values.global.scrapeInterval | quote }}
2020
}
2121

22+
{{- with .Values.serviceMonitors.excludeNamespaces }}
23+
rule {
24+
source_labels = ["__meta_kubernetes_namespace"]
25+
regex = {{ . | join "|" | quote }}
26+
action = "drop"
27+
}
28+
{{- end }}
2229
{{- if .Values.serviceMonitors.extraDiscoveryRules }}
2330
{{ .Values.serviceMonitors.extraDiscoveryRules | indent 2 }}
2431
{{- end }}

‎charts/k8s-monitoring/charts/feature-prometheus-operator-objects/values.schema.json

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@
3636
"enabled": {
3737
"type": "boolean"
3838
},
39+
"excludeNamespaces": {
40+
"type": "array"
41+
},
3942
"extraDiscoveryRules": {
4043
"type": "string"
4144
},
@@ -73,6 +76,9 @@
7376
"enabled": {
7477
"type": "boolean"
7578
},
79+
"excludeNamespaces": {
80+
"type": "array"
81+
},
7682
"extraDiscoveryRules": {
7783
"type": "string"
7884
},
@@ -110,6 +116,9 @@
110116
"enabled": {
111117
"type": "boolean"
112118
},
119+
"excludeNamespaces": {
120+
"type": "array"
121+
},
113122
"extraDiscoveryRules": {
114123
"type": "string"
115124
},

0 commit comments

Comments
 (0)