Add alerting

Author: bastjan

class/defaults.yml

@@ -2,6 +2,9 @@ parameters:
   rotating_bucket_backup:
     =_metadata: {}
     namespace: syn-rotating-bucket-backup
+    namespace_metadata:
+      annotations: {}
+      labels: {}
 
     images:
       mc:
@@ -14,3 +17,28 @@ parameters:
       hour: 1
 
     jobs: {}
+
+    monitoring:
+      enabled: true
+      alerts:
+        RotatingClusterBackup_JobFailed:
+          enabled: true
+          rule:
+            annotations:
+              description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert.
+            expr: |
+              kube_job_failed{namespace="${rotating_bucket_backup:namespace}"} > 0
+            for: 15m
+            labels:
+              severity: warning
+
+        RotatingClusterBackup_JobCompletion_12h:
+          enabled: true
+          rule:
+            annotations:
+              description: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than 12 hours to complete. Job did not complete in time
+            expr: |
+              kube_job_spec_completions{namespace="${rotating_bucket_backup:namespace}"} - kube_job_status_succeeded{namespace="${rotating_bucket_backup:namespace}"} > 0
+            for: 12h
+            labels:
+              severity: warning

component/main.jsonnet

@@ -1,12 +1,45 @@
 // main template for rotating-bucket-backup
+local com = import 'lib/commodore.libjsonnet';
 local kap = import 'lib/kapitan.libjsonnet';
 local kube = import 'lib/kube.libjsonnet';
 local inv = kap.inventory();
 // The hiera parameters for the component
 local params = inv.parameters.rotating_bucket_backup;
 
+local alertlabels = {
+  syn: 'true',
+  syn_component: 'rotating-bucket-backup',
+};
+
+local alerts = params.monitoring.alerts;
+local prometheusRule =
+  kube._Object('monitoring.coreos.com/v1', 'PrometheusRule', 'rotating-bucket-backup-alerts') {
+    metadata+: {
+      namespace: params.namespace,
+    },
+    spec+: {
+      groups+: [
+        {
+          name: 'rotating-bucket-backup-alerts',
+          rules:
+            std.filterMap(
+              function(field) alerts[field].enabled == true,
+              function(field) alerts[field].rule {
+                alert: field,
+                labels+: alertlabels,
+              },
+              std.sort(std.objectFields(alerts)),
+            ),
+        },
+      ],
+    },
+  };
+
 // Define outputs below
 {
-  '00_namespace': kube.Namespace(params.namespace),
+  '00_namespace': kube.Namespace(params.namespace) {
+    metadata+: com.makeMergeable(params.namespace_metadata),
+  },
   '10_jobs': (import 'jobs.libsonnet'),
+  [if params.monitoring.enabled then '20_prometheus_rule']: prometheusRule,
 }

docs/modules/ROOT/pages/references/parameters.adoc

@@ -11,6 +11,21 @@ default:: `syn-rotating-bucket-backup`
 The namespace in which to deploy this component.
 
 
+== `namespace_metadata`
+
+[horizontal]
+type:: dict
+default::
++
+[source,yaml]
+----
+annotations: {}
+labels: {}
+----
+
+Allows adding additional annotations and labels to the namespace.
+
+
 == `images`
 
 [horizontal]
@@ -92,6 +107,60 @@ type:: dict
 The parameters for the target bucket. The keys and values depend on the type of the target bucket.
 
 
+== `monitoring.enabled`
+
+[horizontal]
+type:: boolean
+default:: `true`
+
+Controls if `PrometheusRule` objects are created.
+The alerts expect `kube-state-metrics` to be running in the cluster.
+
+
+== `monitoring.alerts`
+
+[horizontal]
+type:: dict
+default:: See https://github.com/vshn/component-rotating-bucket-backup/blob/master/class/defaults.yml[defaults.yml]
+example::
++
+[source,yaml]
+----
+alerts:
+  MyAlert:
+    enabled: true
+    rule:
+      expr: |
+        up{namespace="${rotating_bucket_backup:namespace}"} == 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Instance {{ $labels.instance }} down"
+        description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
+----
+
+The alerts to configure.
+The key is the name of the alert.
+The value is a dictionary with the following keys:
+
+=== `enabled`
+
+[horizontal]
+type:: boolean
+
+Controls if the alert is enabled.
+
+
+=== `rule`
+
+[horizontal]
+type:: dict
+
+The Prometheus rule for the alert.
+The keys and values depend on the alert.
+
+
 == Example
 
 [source,yaml]

tests/golden/defaults/rotating-bucket-backup/rotating-bucket-backup/20_prometheus_rule.yaml

@@ -0,0 +1,35 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  annotations: {}
+  labels:
+    name: rotating-bucket-backup-alerts
+  name: rotating-bucket-backup-alerts
+  namespace: syn-rotating-bucket-backup
+spec:
+  groups:
+    - name: rotating-bucket-backup-alerts
+      rules:
+        - alert: RotatingClusterBackup_JobCompletion_12h
+          annotations:
+            description: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking
+              more than 12 hours to complete. Job did not complete in time
+          expr: |
+            kube_job_spec_completions{namespace="syn-rotating-bucket-backup"} - kube_job_status_succeeded{namespace="syn-rotating-bucket-backup"} > 0
+          for: 12h
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: rotating-bucket-backup
+        - alert: RotatingClusterBackup_JobFailed
+          annotations:
+            description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed
+              to complete. Removing failed job after investigation should clear this
+              alert.
+          expr: |
+            kube_job_failed{namespace="syn-rotating-bucket-backup"} > 0
+          for: 15m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: rotating-bucket-backup