Add MySQL integration. Metrics only so far.

Signed-off-by: Pete Wall <pete.wall@grafana.com>

CONTRIBUTING.md

@@ -41,10 +41,9 @@ Each component has its own guide for contributing:
 <!-- [Main Helm chart](./charts/k8s-monitoring/README.md#Contributing) -->
 
 -   [Feature subcharts](./charts/k8s-monitoring/docs/Features.md#Contributing)
+-   [Integrations](./charts/feature-integrations/README.md#Contributing)
 -   [Destinations](./charts/k8s-monitoring/docs/destinations/README.md#Contributing)
 
-<!-- [Integrations](./charts/feature-integrations/README.md#Contributing) -->
-
 ## Building
 
 Lots of files in this repository are generated in order to reduce duplication and ensure consistency. To build these

charts/feature-integrations/README.md

@@ -103,3 +103,46 @@ Actual integration testing in a live environment should be done in the main [k8s
 |-----|------|---------|-------------|
 | global.maxCacheSize | int | `100000` | Sets the max_cache_size for every prometheus.relabel component. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus.relabel/#arguments)) This should be at least 2x-5x your largest scrape target or samples appended rate. |
 | global.scrapeInterval | string | `"60s"` | How frequently to scrape metrics. |
+
+### Integration: MySQL
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| mysql | object | `{"instances":[]}` | Scrape metrics from MySQL |
+
+## Contributing
+
+To contribute integrations to this feature, there are a few files that need to be created or modified:
+
+-   `values.yaml` - The main feature chart's values file. Add a section for your integration. It must contain an
+    `instance` array and any settings that apply to every instance of the integration. For example:
+    ```yaml
+    <slug>:
+      instances: []
+      globalSetting: value
+    ```
+-   `integrations/<slug>-values.yaml` - The values that will be used for each instance. This must include `name` to
+    differentiate it from other instances, and any other settings that are specific to that instance. For example:
+    ```yaml
+    name: ""
+    labelSelectors:
+      app.kubernetes.io/name: my-service
+    protocol: http
+    ...
+    ```
+-   `templates/_integration<=_<slug>.tpl` - The file that contains template functions that build the configuration to
+    discover, gather, process, and deliver the telemetry data. This file is required to implement the following template
+    functions:
+    -   `integrations.<slug>.type.metrics` - Returns true if this integration scrapes metrics.
+    -   `integrations.<slug>.type.logs` - Returns true if this integration gathers logs.
+    -   `integrations.<slug>.module` - Returns the configuration that is included once if this integration is used. This
+        is typically the module definition.
+    -   `integrations.<slug>.include.metrics` - Returns the configuration that is included for each instance of the
+        integration that scrapes metrics.
+    -   `integrations.<slug>.include.logs` - Returns the configuration that is included for each instance of the
+        integration that gathers logs.
+    -   `integrations.<slug>.exclude.logs` - Returns a rule that can be used by other Log-gathering features to ensure
+        that logs that are gathered from this integration are not collected twice. Typically the inverse of a rule in
+        the `integrations.<slug>.include.logs` function.
+    -   `default-allow-lists/<slug>.yaml` - If the integration scrapes metrics, a common pattern is to provide a list of
+        metrics that should be allowed. This minimizes the amount of metrics delivered to a useful minimal set.

charts/feature-integrations/README.md.gotmpl

@@ -66,3 +66,40 @@ Actual integration testing in a live environment should be done in the main [k8s
 <!-- markdownlint-enable no-bare-urls -->
 {{ template "chart.requirementsSection" . }}
 {{ template "chart.valuesSection" . }}
+
+## Contributing
+
+To contribute integrations to this feature, there are a few files that need to be created or modified:
+
+-   `values.yaml` - The main feature chart's values file. Add a section for your integration. It must contain an
+    `instance` array and any settings that apply to every instance of the integration. For example:
+    ```yaml
+    <slug>:
+      instances: []
+      globalSetting: value
+    ```
+-   `integrations/<slug>-values.yaml` - The values that will be used for each instance. This must include `name` to
+    differentiate it from other instances, and any other settings that are specific to that instance. For example:
+    ```yaml
+    name: ""
+    labelSelectors:
+      app.kubernetes.io/name: my-service
+    protocol: http
+    ...
+    ```
+-   `templates/_integration<=_<slug>.tpl` - The file that contains template functions that build the configuration to
+    discover, gather, process, and deliver the telemetry data. This file is required to implement the following template
+    functions:
+    -   `integrations.<slug>.type.metrics` - Returns true if this integration scrapes metrics.
+    -   `integrations.<slug>.type.logs` - Returns true if this integration gathers logs.
+    -   `integrations.<slug>.module` - Returns the configuration that is included once if this integration is used. This
+        is typically the module definition.
+    -   `integrations.<slug>.include.metrics` - Returns the configuration that is included for each instance of the
+        integration that scrapes metrics.
+    -   `integrations.<slug>.include.logs` - Returns the configuration that is included for each instance of the
+        integration that gathers logs.
+    -   `integrations.<slug>.exclude.logs` - Returns a rule that can be used by other Log-gathering features to ensure
+        that logs that are gathered from this integration are not collected twice. Typically the inverse of a rule in
+        the `integrations.<slug>.include.logs` function.
+    -   `default-allow-lists/<slug>.yaml` - If the integration scrapes metrics, a common pattern is to provide a list of
+        metrics that should be allowed. This minimizes the amount of metrics delivered to a useful minimal set.

charts/feature-integrations/docs/integrations/alloy.md

@@ -28,4 +28,4 @@
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| scrapeInterval | string | `60s` | How frequently to scrape metrics from Windows Exporter. |
+| scrapeInterval | string | `60s` | How frequently to scrape metrics from Alloy. |

charts/feature-integrations/docs/integrations/mysql.md

@@ -0,0 +1,42 @@
+# mysql
+
+## Values
+
+### Discovery Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| labelSelectors | object | `{}` | Discover MySQL Exporter instances based on label selectors, if not using the exporter |
+
+### Metric Processing Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| maxCacheSize | string | `100000` | Sets the max_cache_size for prometheus.relabel component. This should be at least 2x-5x your largest scrape target or samples appended rate. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus.relabel/#arguments)) Overrides global.maxCacheSize |
+| metricsTuning.excludeMetrics | list | `[]` | Metrics to drop. Can use regular expressions. |
+| metricsTuning.includeMetrics | list | `[]` | Metrics to keep. Can use regular expressions. |
+
+### General Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| name | string | `""` | Name for this MySQL instance. |
+
+### Scrape Settings
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| scrapeInterval | string | `60s` | How frequently to scrape metrics from MySQL Exporter. |
+
+### Other Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| exporter.collectors[0] | string | `"heartbeat"` |  |
+| exporter.collectors[1] | string | `"mysql.user"` |  |
+| exporter.dataSource.host | string | `""` |  |
+| exporter.dataSource.password | string | `""` |  |
+| exporter.dataSource.port | int | `3306` |  |
+| exporter.dataSource.username | string | `""` |  |
+| exporter.dataSourceName | string | `""` |  |
+| exporter.enabled | bool | `true` |  |

charts/feature-integrations/integrations/alloy-values.yaml

@@ -8,7 +8,7 @@ name: ""
 labelSelectors:
   app.kubernetes.io/name: alloy
 
-# -- How frequently to scrape metrics from Windows Exporter.
+# -- How frequently to scrape metrics from Alloy.
 # @default -- `60s`
 # @section -- Scrape Settings
 scrapeInterval:

charts/feature-integrations/integrations/mysql-values.yaml

@@ -0,0 +1,65 @@
+---
+# -- Name for this MySQL instance.
+# @section -- General Settings
+name: ""
+
+exporter:
+  enabled: true
+
+  # TODO: Utilize the secret functions to use from
+  dataSourceName: ""
+  dataSource:
+    host: ""
+    port: 3306
+    auth:
+      username: ""
+      usernameKey: ""
+      usernameFrom: ""
+      password: ""
+      passwordKey: ""
+      passwordFrom: ""
+
+    secret:
+      # -- (bool) Whether to create a secret for this MySQL integration instance.
+      # @default -- `true`
+      # @section -- Secret
+      create:
+      # -- If true, skip secret creation and embed the credentials directly into the configuration.
+      # @section -- Secret
+      embed: false
+      # -- The name of the secret to create.
+      # @section -- Secret
+      name: ""
+      # -- The namespace for the secret.
+      # @section -- Secret
+      namespace: ""
+
+  collectors:
+    - heartbeat
+    - mysql.user
+
+# -- Discover MySQL Exporter instances based on label selectors, if not using the exporter
+# @section -- Discovery Settings
+labelSelectors: {}
+
+# -- How frequently to scrape metrics from MySQL Exporter.
+# @default -- `60s`
+# @section -- Scrape Settings
+scrapeInterval:
+
+# Adjustments to the scraped metrics to filter the amount of data sent to storage.
+metricsTuning:
+  # -- Metrics to keep. Can use regular expressions.
+  # @section -- Metric Processing Settings
+  includeMetrics: []
+  # -- Metrics to drop. Can use regular expressions.
+  # @section -- Metric Processing Settings
+  excludeMetrics: []
+
+# -- Sets the max_cache_size for prometheus.relabel component.
+# This should be at least 2x-5x your largest scrape target or samples appended rate.
+# ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus.relabel/#arguments))
+# Overrides global.maxCacheSize
+# @default -- `100000`
+# @section -- Metric Processing Settings
+maxCacheSize:

charts/feature-integrations/schema-mods/definitions/mysql-integration.schema.json

@@ -0,0 +1,66 @@
+{
+    "type": "object",
+    "properties": {
+        "exporter": {
+            "type": "object",
+            "properties": {
+                "collectors": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "dataSource": {
+                    "type": "object",
+                    "properties": {
+                        "host": {
+                            "type": "string"
+                        },
+                        "password": {
+                            "type": "string"
+                        },
+                        "port": {
+                            "type": "integer"
+                        },
+                        "username": {
+                            "type": "string"
+                        }
+                    }
+                },
+                "dataSourceName": {
+                    "type": "string"
+                },
+                "enabled": {
+                    "type": "boolean"
+                }
+            }
+        },
+        "labelSelectors": {
+            "type": "object"
+        },
+        "maxCacheSize": {
+            "type": "null"
+        },
+        "metricsTuning": {
+            "type": "object",
+            "properties": {
+                "excludeMetrics": {
+                    "type": "array"
+                },
+                "includeMetrics": {
+                    "type": "array"
+                }
+            }
+        },
+        "name": {
+            "type": "string"
+        },
+        "scrapeInterval": {
+            "type": "null"
+        },
+        "type": {
+            "type": "string",
+            "const": "mysql"
+        }
+    }
+}

charts/feature-integrations/schema-mods/integration-list.json

@@ -6,7 +6,8 @@
         "anyOf": [
           { "$ref": "#/definitions/alloy-integration"},
           { "$ref": "#/definitions/cert-manager-integration"},
-          { "$ref": "#/definitions/etcd-integration"}
+          { "$ref": "#/definitions/etcd-integration"},
+          { "$ref": "#/definitions/mysql-integration"}
         ]
       }
     }

charts/feature-integrations/templates/_integration_alloy.tpl

@@ -288,7 +288,7 @@ declare "alloy_integration" {
     }
   }
   {{- range $instance := $.Values.alloy.instances }}
-    {{- include "integrations.alloy.include.metrics" (dict "integration" $instance "Values" $.Values "Files" $.Files) | nindent 2 }}
+    {{- include "integrations.alloy.include.metrics" (deepCopy $ | merge (dict "integration" $instance)) | nindent 2 }}
   {{- end }}
 }
 {{- end }}
@@ -297,7 +297,7 @@ declare "alloy_integration" {
 {{/* Inputs: integration (Alloy integration definition), Values (all values), Files (Files object) */}}
 {{- define "integrations.alloy.include.metrics" }}
 {{- $defaultValues := "integrations/alloy-values.yaml" | .Files.Get | fromYaml }}
-{{- with merge .integration $defaultValues }}
+{{- with deepCopy .integration | merge $defaultValues }}
 {{- $metricAllowList := include "integrations.alloy.allowList" (dict "integration" . "Files" $.Files) }}
 {{- $metricDenyList := .excludeMetrics }}
 {{- $labelSelectors := list }}

charts/feature-integrations/templates/_integration_cert-manager.tpl

@@ -17,7 +17,7 @@ declare "cert_manager_integration" {
   }
 
   {{- range $instance := (index $.Values "cert-manager").instances }}
-    {{- include "integrations.cert-manager.include.metrics" (dict "integration" $instance "Values" $.Values "Files" $.Files) | nindent 2 }}
+    {{- include "integrations.cert-manager.include.metrics" (deepCopy $ | merge (dict "integration" $instance)) | nindent 2 }}
   {{- end }}
 }
 {{- end }}
@@ -26,7 +26,7 @@ declare "cert_manager_integration" {
 {{/* Inputs: integration (cert-manager integration definition), Values (all values), Files (Files object) */}}
 {{- define "integrations.cert-manager.include.metrics" }}
 {{- $defaultValues := "integrations/cert-manager-values.yaml" | .Files.Get | fromYaml }}
-{{- with merge .integration $defaultValues }}
+{{- with deepCopy .integration | merge $defaultValues }}
 {{- $metricAllowList := .metricsTuning.includeMetrics }}
 {{- $metricDenyList := .metricsTuning.excludeMetrics }}
 {{- $labelSelectors := list }}

charts/feature-integrations/templates/_integration_etcd.tpl

@@ -17,7 +17,7 @@ declare "etcd_integration" {
   }
 
   {{- range $instance := (index $.Values "etcd").instances }}
-    {{- include "integrations.etcd.include.metrics" (dict "integration" $instance "Values" $.Values "Files" $.Files) | nindent 2 }}
+    {{- include "integrations.etcd.include.metrics" (deepCopy $ | merge (dict "integration" $instance)) | nindent 2 }}
   {{- end }}
 }
 {{- end }}
@@ -26,7 +26,7 @@ declare "etcd_integration" {
 {{/* Inputs: integration (etcd integration definition), Values (all values), Files (Files object) */}}
 {{- define "integrations.etcd.include.metrics" }}
 {{- $defaultValues := "integrations/etcd-values.yaml" | .Files.Get | fromYaml }}
-{{- with merge .integration $defaultValues }}
+{{- with deepCopy .integration | merge $defaultValues }}
 {{- $metricAllowList := .metricsTuning.includeMetrics }}
 {{- $metricDenyList := .metricsTuning.excludeMetrics }}
 {{- $labelSelectors := list }}