grafana · petewall · Feb 6, 2025 · Jan 24, 2025 · Jan 24, 2025 · Jan 24, 2025
@@ -130,6 +130,12 @@ Be sure perform actual integration testing in a live environment in the main [k8
 |-----|------|---------|-------------|
 | mysql | object | `{"instances":[]}` | Scrape metrics/logs from MySQL |
 
+### Integration: Tempo
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| tempo | object | `{"instances":[]}` | Scrape metrics/logs from Tempo |
+
 ## Contributing
 
 To contribute integrations to this feature, you must create or modify a few files:

@@ -0,0 +1,77 @@
+---
+# The set of metrics from Grafana Tempo required for the Grafana Tempo integration
+## TODO: Populate this list
+- container_cpu_usage_seconds_total
+- container_memory_working_set_bytes
+- container_network_receive_bytes_total
+- container_network_transmit_bytes_total
+- container_spec_cpu_period
+- container_spec_cpu_quota
+- container_spec_memory_limit_bytes
+- gauge_memberlist_health_score
+- go_gc_duration_seconds_count
+- go_goroutines
+- go_memstats_heap_inuse_bytes
+- kube_deployment_spec_replicas
+- kube_deployment_status_replicas_unavailable
+- kube_deployment_status_replicas_updated
+- kube_pod_container_info
+- kube_pod_container_resource_requests
+- kube_pod_container_status_restarts_total
+- kube_statefulset_replicas
+- kube_statefulset_status_replicas_current
+- kube_statefulset_status_replicas_ready
+- kube_statefulset_status_replicas_updated
+- kubelet_volume_stats_available_bytes
+- promtail_custom_bad_words_total
+- tempo_build_info
+- tempo_discarded_spans_total
+- tempo_distributor_bytes_received_total
+- tempo_distributor_metrics_generator_pushes_failures_total
+- tempo_distributor_push_duration_seconds_bucket
+- tempo_distributor_push_duration_seconds_count
+- tempo_distributor_push_duration_seconds_sum
+- tempo_distributor_queue_length
+- tempo_distributor_spans_received_total
+- tempo_ingester_blocks_cleared_total
+- tempo_ingester_blocks_flushed_total
+- tempo_ingester_failed_flushes_total
+- tempo_ingester_flush_duration_seconds_bucket
+- tempo_ingester_live_traces
+- tempo_ingester_traces_created_total
+- tempo_limits_defaults
+- tempo_limits_overrides
+- tempo_memberlist_client_cluster_members_count
+- tempo_memberlist_client_cluster_node_health_score
+- tempo_memberlist_client_kv_store_count
+- tempo_memcache_request_duration_seconds_bucket
+- tempo_memcache_request_duration_seconds_count
+- tempo_memcache_request_duration_seconds_sum
+- tempo_metrics_generator_bytes_received_total
+- tempo_metrics_generator_registry_active_series
+- tempo_metrics_generator_spans_discarded_total
+- tempo_metrics_generator_spans_received_total
+- tempo_query_frontend_queries_total
+- tempo_receiver_accepted_spans
+- tempo_receiver_refused_spans
+- tempo_request_duration_seconds_bucket
+- tempo_request_duration_seconds_count
+- tempo_request_duration_seconds_sum
+- tempo_vulture_trace_error_total
+- tempo_vulture_trace_total
+- tempodb_backend_request_duration_seconds_bucket
+- tempodb_backend_request_duration_seconds_count
+- tempodb_backend_request_duration_seconds_sum
+- tempodb_blocklist_length
+- tempodb_blocklist_poll_errors_total
+- tempodb_compaction_blocks_total
+- tempodb_compaction_bytes_written_total
+- tempodb_compaction_errors_total
+- tempodb_compaction_objects_combined_total
+- tempodb_compaction_objects_written_total
+- tempodb_compaction_outstanding_blocks
+- tempodb_retention_deleted_total
+- tempodb_retention_errors_total
+- tempodb_retention_marked_for_deletion_total
+- tempodb_work_queue_length
+- tempodb_work_queue_max
@@ -0,0 +1,50 @@
+# tempo
+
+## Values
+
+### Discovery Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| fieldSelectors | list | `[]` | Discover Tempo instances based on field selectors. |
+| labelSelectors | object | `{"app.kubernetes.io/name":["tempo","enterprise-traces"]}` | Discover Tempo instances based on label selectors. |
+| metrics.portName | string | `"http-metrics"` | Name of the port to scrape metrics from. |
+| namespaces | list | `[]` | Namespaces to look for Tempo instances in. Will automatically look for Tempo instances in all namespaces unless specified here |
+
+### Logs Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| logs.enabled | bool | `true` | Whether to enable special processing of Tempo pod logs. |
+| logs.tuning.dropLogLevels | list | `[]` | The log levels to drop. Will automatically keep all log levels unless specified here. |
+| logs.tuning.excludeLines | list | `[]` | Line patterns (valid RE2 regular expression)to exclude from the logs. |
+| logs.tuning.scrubTimestamp | bool | `true` | Whether the timestamp should be scrubbed from the log line |
+| logs.tuning.structuredMetadata | object | `{}` | The structured metadata mappings to set. To not set any structured metadata, set this to an empty object (e.g. `{}`) |
+| logs.tuning.timestampFormat | string | `"RFC3339Nano"` | The timestamp format to use for the log line, if not set the default timestamp which is the collection will be used for the log line |
+
+### Metrics Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| metrics.enabled | bool | `true` | Whether to enable metrics collection from Tempo. |
+
+### Metric Processing Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| metrics.maxCacheSize | string | `100000` | Sets the max_cache_size for prometheus.relabel component. This should be at least 2x-5x your largest scrape target or samples appended rate. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus.relabel/#arguments)) Overrides global.maxCacheSize |
+| metrics.tuning.excludeMetrics | list | `[]` | Metrics to drop. Can use regular expressions. |
+| metrics.tuning.includeMetrics | list | `[]` | Metrics to keep. Can use regular expressions. |
+| metrics.tuning.useDefaultAllowList | bool | `true` | Filter the list of metrics from Grafana Tempo to the minimal set required for the Grafana Tempo integration. |
+
+### Scrape Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| metrics.scrapeInterval | string | `60s` | How frequently to scrape metrics from Tempo. |
+
+### General Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| name | string | `""` | Name for this Tempo instance. |
@@ -0,0 +1,84 @@
+---
+# -- Name for this Tempo instance.
+# @section -- General Settings
+name: ""
+
+# -- Namespaces to look for Tempo instances in.
+# Will automatically look for Tempo instances in all namespaces unless specified here
+# @section -- Discovery Settings
+namespaces: []
+
+# -- Discover Tempo instances based on field selectors.
+# @section -- Discovery Settings
+fieldSelectors: []
+
+# -- Discover Tempo instances based on label selectors.
+# @section -- Discovery Settings
+labelSelectors:
+  app.kubernetes.io/name: [tempo, enterprise-traces]
+
+# Settings for metrics collection
+metrics:
+  # -- Whether to enable metrics collection from Tempo.
+  # @section -- Metrics Settings
+  enabled: true
+
+  # -- Name of the port to scrape metrics from.
+  # @section -- Discovery Settings
+  portName: http-metrics
+
+  # -- How frequently to scrape metrics from Tempo.
+  # @default -- `60s`
+  # @section -- Scrape Settings
+  scrapeInterval:
+
+  # -- Sets the max_cache_size for prometheus.relabel component.
+  # This should be at least 2x-5x your largest scrape target or samples appended rate.
+  # ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus.relabel/#arguments))
+  # Overrides global.maxCacheSize
+  # @default -- `100000`
+  # @section -- Metric Processing Settings
+  maxCacheSize:
+
+  # Adjustments to the scraped metrics to filter the amount of data sent to storage.
+  tuning:
+    # -- Filter the list of metrics from Grafana Tempo to the minimal set required for the Grafana Tempo integration.
+    # @section -- Metric Processing Settings
+    useDefaultAllowList: true
+    # -- Metrics to keep. Can use regular expressions.
+    # @section -- Metric Processing Settings
+    includeMetrics: []
+    # -- Metrics to drop. Can use regular expressions.
+    # @section -- Metric Processing Settings
+    excludeMetrics: []
+
+# Settings for log gathering using the Pod Logs feature
+logs:
+  # -- Whether to enable special processing of Tempo pod logs.
+  # @section -- Logs Settings
+  enabled: true
+
+  # Adjustments to the scraped logs to filter the amount of data sent to storage.
+  tuning:
+    # -- The structured metadata mappings to set.
+    # To not set any structured metadata, set this to an empty object (e.g. `{}`)
+    # @section -- Logs Settings
+    structuredMetadata: {}
+
+    # -- The timestamp format to use for the log line, if not set the default timestamp which is the collection
+    # will be used for the log line
+    # @section -- Logs Settings
+    timestampFormat: "RFC3339Nano"
+
+    # -- Whether the timestamp should be scrubbed from the log line
+    # @section -- Logs Settings
+    scrubTimestamp: true
+
+    # -- The log levels to drop.
+    # Will automatically keep all log levels unless specified here.
+    # @section -- Logs Settings
+    dropLogLevels: []
+
+    # -- Line patterns (valid RE2 regular expression)to exclude from the logs.
+    # @section -- Logs Settings
+    excludeLines: []
@@ -0,0 +1,88 @@
+{
+    "type": "object",
+    "properties": {
+        "fieldSelectors": {
+            "type": "array"
+        },
+        "labelSelectors": {
+            "type": "object",
+            "properties": {
+                "app.kubernetes.io/name": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
+        "logs": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "boolean"
+                },
+                "tuning": {
+                    "type": "object",
+                    "properties": {
+                        "dropLogLevels": {
+                            "type": "array"
+                        },
+                        "excludeLines": {
+                            "type": "array"
+                        },
+                        "scrubTimestamp": {
+                            "type": "boolean"
+                        },
+                        "structuredMetadata": {
+                            "type": "object"
+                        },
+                        "timestampFormat": {
+                            "type": "string"
+                        }
+                    }
+                }
+            }
+        },
+        "metrics": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "boolean"
+                },
+                "maxCacheSize": {
+                    "type": "null"
+                },
+                "portName": {
+                    "type": "string"
+                },
+                "scrapeInterval": {
+                    "type": "null"
+                },
+                "tuning": {
+                    "type": "object",
+                    "properties": {
+                        "excludeMetrics": {
+                            "type": "array"
+                        },
+                        "includeMetrics": {
+                            "type": "array"
+                        },
+                        "useDefaultAllowList": {
+                            "type": "boolean"
+                        }
+                    }
+                }
+            }
+        },
+        "name": {
+            "type": "string"
+        },
+        "namespaces": {
+            "type": "array"
+        },
+        "type": {
+            "type": "string",
+            "const": "tempo"
+        }
+    }
+}
@@ -10,7 +10,8 @@
           { "$ref": "#/definitions/grafana-integration"},
           { "$ref": "#/definitions/loki-integration"},
           { "$ref": "#/definitions/mimir-integration"},
-          { "$ref": "#/definitions/mysql-integration"}
+          { "$ref": "#/definitions/mysql-integration"},
+          { "$ref": "#/definitions/tempo-integration"}
         ]
       }
     }

@@ -0,0 +1,22 @@
+{{- define "integrations.tempo.validate" }}
+  {{- range $instance := $.Values.tempo.instances }}
+    {{- include "integrations.tempo.instance.validate" (merge $ (dict "instance" $instance)) | nindent 2 }}
+  {{- end }}
+{{- end }}
+
+{{- define "integrations.tempo.instance.validate" }}
+  {{- if not .instance.labelSelectors }}
+    {{- $msg := list "" "The tempo integration requires a label selector" }}
+    {{- $msg = append $msg "For example, please set:" }}
+    {{- $msg = append $msg "integrations:" }}
+    {{- $msg = append $msg "  tempo:" }}
+    {{- $msg = append $msg "    instances:" }}
+    {{- $msg = append $msg (printf "      - name: %s" .instance.name) }}
+    {{- $msg = append $msg "        labelSelectors:" }}
+    {{- $msg = append $msg (printf "          app.kubernetes.io/name: %s" .instance.name) }}
+    {{- $msg = append $msg "OR" }}
+    {{- $msg = append $msg "        labelSelectors:" }}
+    {{- $msg = append $msg "          app.kubernetes.io/name: [tempo-one, tempo-two]" }}
+    {{- fail (join "\n" $msg) }}
+  {{- end }}
+{{- end }}