Skip to content

Commit

Permalink
Add the auto-instrumentation feature (#834)
Browse files Browse the repository at this point in the history
* Add auto-instrumentation feature, which uses Grafana Beyla

Signed-off-by: Pete Wall <pete.wall@grafana.com>

* Add tests for beyla metrics and traces:

Signed-off-by: Pete Wall <pete.wall@grafana.com>

---------

Signed-off-by: Pete Wall <pete.wall@grafana.com>
  • Loading branch information
petewall authored Nov 13, 2024
1 parent b9f8745 commit 5bdc1d2
Show file tree
Hide file tree
Showing 61 changed files with 3,350 additions and 76 deletions.
6 changes: 6 additions & 0 deletions charts/feature-auto-instrumentation/.helmignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
docs
schema-mods
tests
Makefile
README.md
README.md.gotmpl
31 changes: 31 additions & 0 deletions charts/feature-auto-instrumentation/.updatecli-beyla.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
---
name: Update dependency "beyla" for Helm chart "feature-auto-instrumentation"
sources:
beyla:
name: Get latest "beyla" Helm chart version
kind: helmchart
spec:
name: beyla
url: https://grafana.github.io/helm-charts
versionfilter:
kind: semver
pattern: '*'
conditions:
beyla:
name: Ensure Helm chart dependency "beyla" is specified
kind: yaml
spec:
file: charts/feature-auto-instrumentation/Chart.yaml
key: $.dependencies[0].name
value: beyla
disablesourceinput: true
targets:
beyla:
name: Bump Helm chart dependency "beyla" for Helm chart "feature-auto-instrumentation"
kind: helmchart
spec:
file: Chart.yaml
key: $.dependencies[0].version
name: charts/feature-auto-instrumentation
versionincrement: none
sourceid: beyla
6 changes: 6 additions & 0 deletions charts/feature-auto-instrumentation/Chart.lock
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
dependencies:
- name: beyla
repository: https://grafana.github.io/helm-charts
version: 1.4.11
digest: sha256:9b033f8958e90acc3d4d51f14cadf332be42b9c83a62a1492d2db421f946c650
generated: "2024-11-12T16:42:11.373589-07:00"
17 changes: 17 additions & 0 deletions charts/feature-auto-instrumentation/Chart.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
---
apiVersion: v2
name: k8s-monitoring-feature-auto-instrumentation
description: Gathers telemetry data via automatic instrumentation
icon: https://raw.githubusercontent.com/grafana/grafana/main/public/img/grafana_icon.svg
sources:
- https://github.com/grafana/k8s-monitoring-helm/tree/main/charts/feature-annotation-autodiscovery
version: 1.0.0
appVersion: 1.0.0
maintainers:
- email: pete.wall@grafana.com
name: petewall
dependencies:
- name: beyla
version: 1.4.11
repository: https://grafana.github.io/helm-charts
condition: beyla.enabled
40 changes: 40 additions & 0 deletions charts/feature-auto-instrumentation/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
HAS_HELM_DOCS := $(shell command -v helm-docs;)
HAS_HELM_UNITTEST := $(shell helm plugin list | grep unittest 2> /dev/null)
UPDATECLI_FILES := $(shell yq -e '.dependencies[] | select(.repository == "http*") | ".updatecli-" + .name + ".yaml"' Chart.yaml 2>/dev/null | sort | uniq)

.SECONDEXPANSION:
README.md: values.yaml Chart.yaml $$(wildcard README.md.gotmpl)
ifdef HAS_HELM_DOCS
helm-docs
else
docker run --rm --volume "$(shell pwd):/helm-docs" -u $(shell id -u) jnorwood/helm-docs:latest
endif

Chart.lock: Chart.yaml
helm dependency update .
@touch Chart.lock # Ensure the timestamp is updated

values.schema.json: values.yaml $$(wildcard schema-mods/*)
../../scripts/schema-gen.sh .

.updatecli-%.yaml: Chart.yaml
../../scripts/charts-to-updatecli.sh Chart.yaml

.PHONY: clean
clean:
rm -f README.md values.schema.json $(UPDATECLI_FILES)

.PHONY: build
build: README.md Chart.lock values.schema.json $(UPDATECLI_FILES)

.PHONY: test
test: build
helm repo add grafana https://grafana.github.io/helm-charts

helm lint .
ct lint --lint-conf ../../.configs/lintconf.yaml --helm-dependency-extra-args=--skip-refresh --charts .
ifdef HAS_HELM_UNITTEST
helm unittest .
else
docker run --rm --volume $(shell pwd):/apps helmunittest/helm-unittest .
endif
73 changes: 73 additions & 0 deletions charts/feature-auto-instrumentation/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
<!--
(NOTE: Do not edit README.md directly. It is a generated file!)
( To make changes, please modify README.md.gotmpl and run `helm-docs`)
-->

# k8s-monitoring-feature-auto-instrumentation

![Version: 1.0.0](https://img.shields.io/badge/Version-1.0.0-informational?style=flat-square) ![AppVersion: 1.0.0](https://img.shields.io/badge/AppVersion-1.0.0-informational?style=flat-square)
Gathers telemetry data via automatic instrumentation

The auto-instrumentation feature deploys Grafana Beyla to automatically instrument programs running on this cluster
using eBPF.

## Testing

This chart contains unit tests to verify the generated configuration. A hidden value, `deployAsConfigMap`, will render
the generated configuration into a ConfigMap object. This ConfigMap is not used during regular operation, but it is
useful for showing the outcome of a given values file.

The unit tests use this to create an object with the configuration that can be asserted against. To run the tests, use
`helm test`.

Actual integration testing in a live environment should be done in the main [k8s-monitoring](../k8s-monitoring) chart.

## Maintainers

| Name | Email | Url |
| ---- | ------ | --- |
| petewall | <pete.wall@grafana.com> | |
<!-- markdownlint-disable no-bare-urls -->
<!-- markdownlint-disable list-marker-space -->
## Source Code

* <https://github.com/grafana/k8s-monitoring-helm/tree/main/charts/feature-annotation-autodiscovery>

## Requirements

| Repository | Name | Version |
|------------|------|---------|
| https://grafana.github.io/helm-charts | beyla | 1.4.11 |
<!-- markdownlint-enable list-marker-space -->
<!-- markdownlint-enable no-bare-urls -->
<!-- markdownlint-disable no-space-in-emphasis -->
## Values

### Beyla

| Key | Type | Default | Description |
|-----|------|---------|-------------|
| beyla.extraDiscoveryRules | string | `""` | Rule blocks to be added to the discovery.relabel component for Beyla. These relabeling rules are applied pre-scrape against the targets from service discovery. Before the scrape, any remaining target labels that start with __ (i.e. __meta_kubernetes*) are dropped. ([docs](https://grafana.com/docs/alloy/latest/reference/components/discovery.relabel/#rule-block)) |
| beyla.extraMetricProcessingRules | string | `""` | Rule blocks to be added to the prometheus.relabel component for Beyla. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus.relabel/#rule-block)) These relabeling rules are applied post-scrape against the metrics returned from the scraped target, no __meta* labels are present. |
| beyla.labelMatchers | object | `{"app.kubernetes.io/name":"beyla"}` | Label matchers used to select the Beyla pods for scraping metrics. |
| beyla.maxCacheSize | string | 100000 | Sets the max_cache_size for the prometheus.relabel component for Beyla. This should be at least 2x-5x your largest scrape target or samples appended rate. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus.relabel/#arguments)) Overrides metrics.maxCacheSize |
| beyla.metricsTuning.excludeMetrics | list | `[]` | Metrics to drop. Can use regular expressions. |
| beyla.metricsTuning.includeMetrics | list | `[]` | Metrics to keep. Can use regular expressions. |
| beyla.preset | string | `"application"` | The configuration preset to use. Valid options are "application" or "network". |
| beyla.scrapeInterval | string | 60s | How frequently to scrape metrics from Beyla. Overrides metrics.scrapeInterval |
| beyla.service | object | `{"targetPort":9090}` | The port number for the Beyla service. |

### General settings

| Key | Type | Default | Description |
|-----|------|---------|-------------|
| fullnameOverride | string | `""` | Full name override |
| nameOverride | string | `""` | Name override |

### Global Settings

| Key | Type | Default | Description |
|-----|------|---------|-------------|
| global.maxCacheSize | int | `100000` | Sets the max_cache_size for every prometheus.relabel component. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus.relabel/#arguments)) This should be at least 2x-5x your largest scrape target or samples appended rate. |
| global.scrapeInterval | string | `"60s"` | How frequently to scrape metrics. |
<!-- markdownlint-enable no-space-in-emphasis -->
36 changes: 36 additions & 0 deletions charts/feature-auto-instrumentation/README.md.gotmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
<!--
(NOTE: Do not edit README.md directly. It is a generated file!)
( To make changes, please modify README.md.gotmpl and run `helm-docs`)
-->

{{ template "chart.header" . }}
{{ template "chart.deprecationWarning" . }}
{{ template "chart.badgesSection" . }}
{{ template "chart.description" . }}
{{ template "chart.homepageLine" . }}

The auto-instrumentation feature deploys Grafana Beyla to automatically instrument programs running on this cluster
using eBPF.

## Testing

This chart contains unit tests to verify the generated configuration. A hidden value, `deployAsConfigMap`, will render
the generated configuration into a ConfigMap object. This ConfigMap is not used during regular operation, but it is
useful for showing the outcome of a given values file.

The unit tests use this to create an object with the configuration that can be asserted against. To run the tests, use
`helm test`.

Actual integration testing in a live environment should be done in the main [k8s-monitoring](../k8s-monitoring) chart.

{{ template "chart.maintainersSection" . }}
<!-- markdownlint-disable no-bare-urls -->
<!-- markdownlint-disable list-marker-space -->
{{ template "chart.sourcesSection" . }}

{{ template "chart.requirementsSection" . }}
<!-- markdownlint-enable list-marker-space -->
<!-- markdownlint-enable no-bare-urls -->
<!-- markdownlint-disable no-space-in-emphasis -->
{{ template "chart.valuesSection" . }}
<!-- markdownlint-enable no-space-in-emphasis -->
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"properties": {
"beyla": {"properties": {"preset": {"enum": ["application", "network"]}}}
}
}
29 changes: 29 additions & 0 deletions charts/feature-auto-instrumentation/templates/_helpers.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{{/*
Create a default fully qualified name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "feature.autoInstrumentation.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" | lower }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride | lower }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" | lower }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" | lower }}
{{- end }}
{{- end }}
{{- end }}

{{- define "escape_annotation" -}}
{{ . | replace "-" "_" | replace "." "_" | replace "/" "_" }}
{{- end }}

{{- define "pod_annotation" -}}
{{ printf "__meta_kubernetes_pod_annotation_%s" (include "escape_annotation" .) }}
{{- end }}

{{- define "service_annotation" -}}
{{ printf "__meta_kubernetes_service_annotation_%s" (include "escape_annotation" .) }}
{{- end }}
89 changes: 89 additions & 0 deletions charts/feature-auto-instrumentation/templates/_module.alloy.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
{{- define "feature.autoInstrumentation.module" }}
{{- $metricAllowList := .Values.beyla.metricsTuning.includeMetrics }}
{{- $metricDenyList := .Values.beyla.metricsTuning.excludeMetrics }}
{{- $labelSelectors := list }}
{{- range $k, $v := .Values.beyla.labelMatchers }}
{{- $labelSelectors = append $labelSelectors (printf "%s=%s" $k $v) }}
{{- end }}
declare "auto_instrumentation" {
argument "metrics_destinations" {
comment = "Must be a list of metric destinations where collected metrics should be forwarded to"
}

discovery.kubernetes "beyla_pods" {
role = "pod"
namespaces {
own_namespace = true
}
selectors {
role = "pod"
label = {{ $labelSelectors | join "," | quote }}
}
}

discovery.relabel "beyla_pods" {
targets = discovery.kubernetes.beyla_pods.targets
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
action = "replace"
target_label = "instance"
}

{{- if .Values.beyla.extraDiscoveryRules }}
{{ .Values.beyla.extraDiscoveryRules | indent 4 }}
{{- end }}
}

prometheus.scrape "beyla_applications" {
targets = discovery.relabel.beyla_pods.output
honor_labels = true
scrape_interval = {{ .Values.beyla.scrapeInterval | default .Values.global.scrapeInterval | quote }}
clustering {
enabled = true
}
{{- if or $metricAllowList $metricDenyList .Values.beyla.extraMetricProcessingRules }}
forward_to = [prometheus.relabel.beyla.receiver]
{{- else }}
forward_to = argument.metrics_destinations.value
{{- end }}
}

prometheus.scrape "beyla_internal" {
targets = discovery.relabel.beyla_pods.output
metrics_path = "/internal/metrics"
job_name = "integrations/beyla"
honor_labels = true
scrape_interval = {{ .Values.beyla.scrapeInterval | default .Values.global.scrapeInterval | quote }}
clustering {
enabled = true
}
{{- if or $metricAllowList $metricDenyList .Values.beyla.extraMetricProcessingRules }}
forward_to = [prometheus.relabel.beyla.receiver]
}

prometheus.relabel "beyla" {
max_cache_size = {{ .Values.beyla.maxCacheSize | default .Values.global.maxCacheSize | int }}
{{- if $metricAllowList }}
rule {
source_labels = ["__name__"]
regex = "up|{{ $metricAllowList | join "|" }}"
action = "keep"
}
{{- end }}
{{- if $metricDenyList }}
rule {
source_labels = ["__name__"]
regex = {{ $metricDenyList | join "|" | quote }}
action = "drop"
}
{{- end }}
{{- if .Values.beyla.extraMetricProcessingRules }}
{{ .Values.beyla.extraMetricProcessingRules | indent 4 }}
{{- end }}
{{- end }}
forward_to = argument.metrics_destinations.value
}
}
{{- end -}}

{{- define "feature.autoInstrumentation.alloyModules" }}{{- end }}
13 changes: 13 additions & 0 deletions charts/feature-auto-instrumentation/templates/_notes.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{{- define "feature.autoInstrumentation.notes.deployments" }}
* Grafana Beyla (Daemonset)
{{- end }}

{{- define "feature.autoInstrumentation.notes.task" }}
Automatically instrument applications and services running in the cluster with Grafana Beyla
{{- end }}

{{- define "feature.autoInstrumentation.notes.actions" }}{{- end }}

{{- define "feature.autoInstrumentation.summary" -}}
version: {{ .Chart.Version }}
{{- end }}
11 changes: 11 additions & 0 deletions charts/feature-auto-instrumentation/templates/configmap.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{{- if .Values.deployAsConfigMap }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "feature.autoInstrumentation.fullname" . }}
namespace: {{ .Release.Namespace }}
data:
module.alloy: |-
{{- include "feature.autoInstrumentation.module" . | indent 4 }}
{{- end }}
Empty file.
Loading

0 comments on commit 5bdc1d2

Please sign in to comment.