Use monitoring-mixins for dashboards and alerts

Monitoring-mixins ([1]) allow upstream collaboration on alerts and
grafana dashboards and convergence on monitoring best practices through
bundling mixins with the corresponding software.
Downstream customization of alerts and dashboards may be done using
jsonnet ([2]).
Usage of jsonnet-bundler ([3]) gives a clear reference of where
dashboards and alerts originated and which version is used through
`.src/jsonnetfile.json` and `.src/jsonnetfile.lock.json`.

[1]
https://monitoring.mixins.dev/

[2]
https://github.com/google/jsonnet

[3]
https://github.com/jsonnet-bundler/jsonnet-bundler

Signed-off-by: Jan Horstmann <horstmann@osism.tech>

.gitignore

@@ -0,0 +1 @@
+.src/vendor

.src/Containerfile

@@ -0,0 +1,27 @@
+From python:3.12-slim-bookworm
+
+ARG JB_VERSION=v0.6.0
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+        ca-certificates \
+        curl \
+        git \
+        jsonnet \
+    && curl --location --output /usr/local/bin/jb https://github.com/jsonnet-bundler/jsonnet-bundler/releases/download/${JB_VERSION}/jb-linux-amd64 \
+    && chmod +x /usr/local/bin/jb \
+    && apt-get remove -y \
+        curl \
+    && apt-get clean \
+    && apt-get autoremove -y \
+    && rm -rf \
+      /var/lib/apt/lists/* \
+      /var/tmp/*
+
+COPY files/mixins.py /mixins.py
+
+WORKDIR /srv/.src
+CMD ["/mixins.py"]
+

.src/Dockerfile

@@ -0,0 +1 @@
+Containerfile

.src/files/mixins.py

@@ -0,0 +1,69 @@
+#!/usr/local/bin/python3
+
+import os
+import subprocess
+
+dashboards_out_prefix = "../grafana/dashboards"
+rules_out_prefix = "../prometheus"
+
+print("Installing dependencies...")
+try:
+    out = subprocess.run(["/usr/local/bin/jb", "install"], capture_output=True)
+except subprocess.SubprocessError as e:
+    print(e.stderr)
+    raise e
+print(out.stdout.decode())
+print(out.stderr.decode())
+
+for path, _, filenames in os.walk("mixins"):
+    for file in filenames:
+        dashboards_out = os.path.join(
+            dashboards_out_prefix, *os.path.normpath(path).split(os.sep)[1:]
+        )
+        if file.endswith(".libsonnet"):
+            name = file.removesuffix(".libsonnet")
+            print("Processing mixin " + name)
+            print("Creating dashboard(s)...")
+            os.makedirs(dashboards_out, exist_ok=True)
+            try:
+                subprocess.run(
+                    [
+                        "jsonnet",
+                        "-J",
+                        "vendor",
+                        "-m",
+                        dashboards_out,
+                        "-e",
+                        '(import "' + os.path.join(path, file) + '").grafanaDashboards',
+                    ]
+                )
+            except subprocess.SubprocessError as e:
+                print(e.stderr)
+                raise e
+            for kind, suffix in [
+                ("prometheusAlerts", ".rules"),
+                ("prometheusRules", ".rec.rules"),
+            ]:
+                print("Creating " + kind + "...")
+                prometheus_out = os.path.join(rules_out_prefix, name + suffix)
+                with open(prometheus_out, "w") as f:
+                    try:
+                        subprocess.run(
+                            [
+                                "jsonnet",
+                                "-J",
+                                "vendor",
+                                "-S",
+                                "-e",
+                                'std.manifestYamlDoc((import "'
+                                + os.path.join(path, file)
+                                + '").'
+                                + kind
+                                + ")",
+                            ],
+                            stdout=f,
+                        )
+                    except subprocess.SubprocessError as e:
+                        print(e.stderr)
+                        raise e
+                print(prometheus_out)

.src/jsonnetfile.json

@@ -0,0 +1,42 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/alertmanager.git",
+          "subdir": "doc/alertmanager-mixin"
+        }
+      },
+      "version": "main"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/ceph/ceph.git",
+          "subdir": "monitoring/ceph-mixin"
+        }
+      },
+      "version": "main"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/node_exporter.git",
+          "subdir": "docs/node-mixin"
+        }
+      },
+      "version": "master"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/prometheus.git",
+          "subdir": "documentation/prometheus-mixin"
+        }
+      },
+      "version": "main"
+    }
+  ],
+  "legacyImports": true
+}

.src/jsonnetfile.lock.json

@@ -0,0 +1,96 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/ceph/ceph.git",
+          "subdir": "monitoring/ceph-mixin"
+        }
+      },
+      "version": "14463aa7a493dc6903d18af960aac24e8ab8153b",
+      "sum": "ZnyCIu25NBI6Q3Ru7QK1DHf7DBMEURSMQdEJXzCyIgA="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafonnet-lib.git",
+          "subdir": "grafonnet"
+        }
+      },
+      "version": "a1d61cce1da59c71409b99b5c7568511fec661ea",
+      "sum": "342u++/7rViR/zj2jeJOjshzglkZ1SY+hFNuyCBFMdc="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafonnet.git",
+          "subdir": "gen/grafonnet-latest"
+        }
+      },
+      "version": "d20e609202733790caf5b554c9945d049f243ae3",
+      "sum": "V9vAj21qJOc2DlMPDgB1eEjSQU4A+sAA4AXuJ6bd4xc="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafonnet.git",
+          "subdir": "gen/grafonnet-v11.4.0"
+        }
+      },
+      "version": "d20e609202733790caf5b554c9945d049f243ae3",
+      "sum": "aVAX09paQYNOoCSKVpuk1exVIyBoMt/C50QJI+Q/3nA="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/jsonnet-libs/docsonnet.git",
+          "subdir": "doc-util"
+        }
+      },
+      "version": "6ac6c69685b8c29c54515448eaca583da2d88150",
+      "sum": "BrAL/k23jq+xy9oA7TWIhUx07dsA/QLm3g7ktCwe//U="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/jsonnet-libs/xtd.git",
+          "subdir": ""
+        }
+      },
+      "version": "1199b50e9d2ff53d4bb5fb2304ad1fb69d38e609",
+      "sum": "LfbgcJbilu4uBdKYZSvmkoOTPwEAzg10L3/VqKAIWtA="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/alertmanager.git",
+          "subdir": "doc/alertmanager-mixin"
+        }
+      },
+      "version": "3b61ae81a7b859ae4e9ddd34c88d71b4c1691a9e",
+      "sum": "Mf4h1BYLle2nrgjf/HXrBbl0Zk8N+xaoEM017o0BC+k="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/node_exporter.git",
+          "subdir": "docs/node-mixin"
+        }
+      },
+      "version": "8f9a914bee023a614f4eea1005baf2e112bbadb0",
+      "sum": "xYj6VYFT/eafsbleNlC+Z2VfLy1CndyYrJs9BcTmnX8="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/prometheus.git",
+          "subdir": "documentation/prometheus-mixin"
+        }
+      },
+      "version": "d173c0b61c6658be00d5502c76c9226b7f643017",
+      "sum": "CwaQpW66lHx+++sY2g4BgrUTEFZtlDnQzFjo0AlgfIg="
+    }
+  ],
+  "legacyImports": false
+}

.src/mixins/ceph/ceph.libsonnet

@@ -0,0 +1,7 @@
+local ceph = import "ceph-mixin/mixin.libsonnet";
+
+ceph {
+  prometheusRules+: {},
+  prometheusAlerts+: {},
+  grafanaDashboards+: {}
+}

.src/mixins/infrastructure/alertmanager.libsonnet

@@ -0,0 +1,7 @@
+local alertmanager = import "alertmanager-mixin/mixin.libsonnet";
+
+alertmanager {
+  prometheusRules+: {},
+  prometheusAlerts+: {},
+  grafanaDashboards+: {}
+}

.src/mixins/infrastructure/node.libsonnet

@@ -0,0 +1,11 @@
+local node = import "node-mixin/mixin.libsonnet";
+
+node {
+  prometheusRules+: {},
+  prometheusAlerts+: {},
+  grafanaDashboards+: {
+    # Hide unused dashboards
+    'nodes-darwin.json':: super['nodes-darwin.json'],
+    'nodes-aix.json':: super['nodes-aix.json']
+  }
+}

.src/mixins/infrastructure/prometheus.libsonnet

@@ -0,0 +1,7 @@
+local prometheus = import "prometheus-mixin/mixin.libsonnet";
+
+prometheus {
+  prometheusRules+: {},
+  prometheusAlerts+: {},
+  grafanaDashboards+: {}
+}

README.md

@@ -3,5 +3,49 @@
 Repository for Grafana dashboards and Prometheus alerting rules.
 For use with the Prometheus exporters from Kolla.
 
-The alerts here build on those from:
-https://github.com/samber/awesome-prometheus-alerts
+## Usage
+
+While it is possible to directly place `.rules` files into the `prometheus/` folder  and grafana dashboards into `grafana/dashboards` or a subfolder thereof, use of [monitoring-mixins](https://monitoring.mixins.dev) is encouraged.
+
+### Build the jsonnet container
+
+Build the `mixin` container to manage mixins
+
+```
+podman build -t mixins .src
+```
+
+### Building mixins
+
+[jsonnet-bundler](https://github.com/jsonnet-bundler/jsonnet-bundler) is used to fetch mixins and their dependencies from upstream.
+
+* Install mixin dependencies
+
+  ```
+  podman run -it -v .:/srv localhost/mixins:latest jb install
+  ```
+
+* Build alerts, rules and dashboards
+  ```
+  podman run -it -v .:/srv localhost/mixins:latest
+  ```
+
+* Check all changes into git, including the generated alerts, rules and dashboards
+
+### Adding new mixins
+
+* Add any new upstream mixins to `.src/jsonnetfile.json
+* Add a local libsonnet for any new mixins in `.src/mixins/$DASHBOARDSUBFOLDER/$NAME.libsonnet
+* Apply any [customizations](https://monitoring.mixins.dev/#customising-the-mixin) to the local `$NAME.libsonnet`
+* [Build mixins](#building-mixins) 
+* Check all changes into git, including the `.src/jsonnetfile.lock.json`
+
+### Updating mixins
+
+* Update the jsonnet-bundler dependencies in `.src/jsonnetfile.lock.json
+
+  ```
+  podman run -it -v .:/srv localhost/mixins:latest jb update
+  ```
+
+* Check all changes into git, including the `.src/jsonnetfile.lock.json`

grafana/dashboards/ceph/multi-cluster-overview.json

@@ -22,7 +22,20 @@
    "graphTooltip": 0,
    "hideControls": false,
    "id": null,
-   "links": [ ],
+   "links": [
+      {
+         "asDropdown": true,
+         "icon": "external link",
+         "includeVars": true,
+         "keepTime": true,
+         "tags": [ ],
+         "targetBlank": false,
+         "title": "Browse Dashboards",
+         "tooltip": "",
+         "type": "dashboards",
+         "url": ""
+      }
+   ],
    "panels": [
       {
          "collapse": false,