minio add podmonitor and update metrics path (#408)

1. update the metrics path from `/minio/prometheus/metrics` to `/minio/v2/metrics/cluster`
2. add podMonitor for minio
3. version bumped
4. add podmonitor's secret

Author: locustbaby

charts/minio/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 description: High Performance, Kubernetes Native Object Storage
 name: minio
-version: 8.0.14
+version: 8.0.15
 appVersion: master
 keywords:
 - storage

charts/minio/templates/podmonitor.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.metrics.podMonitor.enabled }}
+{{ $scheme := "http" }}
+{{- if .Values.tls.enabled }}
+{{ $scheme = "https" }}
+{{ end }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ template "minio.fullname" . }}
+  labels:
+    app: {{ template "minio.name" . }}
+    release: {{ .Release.Name }}
+spec:
+  podMetricsEndpoints:
+    - port: {{ $scheme }}
+      path: /minio/v2/metrics/cluster
+      {{- if .Values.metrics.podMonitor.interval }}
+      interval: {{ .Values.metrics.podMonitor.interval }}
+      {{- end }}
+      {{- if .Values.metrics.podMonitor.scrapeTimeout }}
+      scrapeTimeout: {{ .Values.metrics.podMonitor.scrapeTimeout }}
+      {{- end }}
+      bearerTokenSecret:
+        name: {{ template "minio.fullname" . }}-podmonitor-prometheus
+        key: token
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace | quote }}
+  selector:
+    matchLabels:
+      app: {{ include "minio.name" . }}
+      release: {{ .Release.Name }}
+{{- end }}

charts/minio/templates/post-install-prometheus-metrics-job.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.metrics.serviceMonitor.enabled }}
+{{- if or .Values.metrics.serviceMonitor.enabled .Values.metrics.podMonitor.enabled }}
 {{- $fullName := include "minio.fullname" . -}}
 {{ $scheme := "http" }}
 {{- if .Values.tls.enabled }}
@@ -101,18 +101,35 @@ spec:
         - name: kubectl-create
           image: "{{ .Values.helmKubectlJqImage.repository }}:{{ .Values.helmKubectlJqImage.tag }}"
           imagePullPolicy: {{ .Values.helmKubectlJqImage.pullPolicy }}
-          command:
-            - /bin/sh
-            - "-c"
+          command: ["/bin/sh", "-c"]
+          args:
             # The following script does:
             # - get the servicemonitor that need this secret and copy some metadata and create the ownerreference for the secret file
             # - create the secret
             # - merge both json
-            - >
-              kubectl -n {{ .Release.Namespace }} get servicemonitor {{ $fullName }} -o json |
-                jq -c '{metadata: {name: "{{ $fullName }}-prometheus", namespace: .metadata.namespace, labels: {app: .metadata.labels.app, release: .metadata.labels.release}, ownerReferences: [{apiVersion: .apiVersion, kind: .kind, blockOwnerDeletion: true, controller: true, uid: .metadata.uid, name: .metadata.name}]}}' > /workdir/metadata.json &&
-              kubectl create secret generic {{ $fullName }}-prometheus --from-file=token=/workdir/token --dry-run -o json > /workdir/secret.json &&
-              cat /workdir/secret.json /workdir/metadata.json | jq -s add > /workdir/object.json
+            {{- if and .Values.metrics.serviceMonitor.enabled .Values.metrics.podMonitor.enabled }}
+            - |
+              mkdir -p /workdir/secrets && kubectl -n {{ .Release.Namespace }} get servicemonitor {{ $fullName }} -o json |
+                jq -c '{metadata: {name: "{{ $fullName }}-servicemonitor-prometheus", namespace: .metadata.namespace, labels: {app: .metadata.labels.app, release: .metadata.labels.release}, ownerReferences: [{apiVersion: .apiVersion, kind: .kind, blockOwnerDeletion: true, controller: true, uid: .metadata.uid, name: .metadata.name}]}}' > /workdir/servicemonitormetadata.json &&
+              kubectl create secret generic {{ $fullName }}-servicemonitor-prometheus --from-file=token=/workdir/token --dry-run -o json > /workdir/servicemonitorsecret.json &&
+              cat /workdir/servicemonitorsecret.json /workdir/servicemonitormetadata.json | jq -s add > /workdir/secrets/servicemonitorobject.json;
+              mkdir -p /workdir/secrets && kubectl -n {{ .Release.Namespace }} get podmonitor {{ $fullName }} -o json |
+                jq -c '{metadata: {name: "{{ $fullName }}-podmonitor-prometheus", namespace: .metadata.namespace, labels: {app: .metadata.labels.app, release: .metadata.labels.release}, ownerReferences: [{apiVersion: .apiVersion, kind: .kind, blockOwnerDeletion: true, controller: true, uid: .metadata.uid, name: .metadata.name}]}}' > /workdir/podmonitormetadata.json &&
+              kubectl create secret generic {{ $fullName }}-podmonitor-prometheus --from-file=token=/workdir/token --dry-run -o json > /workdir/podmonitorsecret.json &&
+              cat /workdir/podmonitorsecret.json /workdir/podmonitormetadata.json | jq -s add > /workdir/secrets/podmonitorobject.json
+            {{- else if .Values.metrics.podMonitor.enabled }}
+            - |
+              mkdir -p /workdir/secrets && kubectl -n {{ .Release.Namespace }} get podmonitor {{ $fullName }} -o json |
+                jq -c '{metadata: {name: "{{ $fullName }}-podmonitor-prometheus", namespace: .metadata.namespace, labels: {app: .metadata.labels.app, release: .metadata.labels.release}, ownerReferences: [{apiVersion: .apiVersion, kind: .kind, blockOwnerDeletion: true, controller: true, uid: .metadata.uid, name: .metadata.name}]}}' > /workdir/podmonitormetadata.json &&
+              kubectl create secret generic {{ $fullName }}-podmonitor-prometheus --from-file=token=/workdir/token --dry-run -o json > /workdir/podmonitorsecret.json &&
+              cat /workdir/podmonitorsecret.json /workdir/podmonitormetadata.json | jq -s add > /workdir/secrets/podmonitorobject.json
+            {{- else if .Values.metrics.serviceMonitor.enabled }}
+            - |
+              mkdir -p /workdir/secrets && kubectl -n {{ .Release.Namespace }} get servicemonitor {{ $fullName }} -o json |
+                jq -c '{metadata: {name: "{{ $fullName }}-servicemonitor-prometheus", namespace: .metadata.namespace, labels: {app: .metadata.labels.app, release: .metadata.labels.release}, ownerReferences: [{apiVersion: .apiVersion, kind: .kind, blockOwnerDeletion: true, controller: true, uid: .metadata.uid, name: .metadata.name}]}}' > /workdir/servicemonitormetadata.json &&
+              kubectl create secret generic {{ $fullName }}-servicemonitor-prometheus --from-file=token=/workdir/token --dry-run -o json > /workdir/servicemonitorsecret.json &&
+              cat /workdir/servicemonitorsecret.json /workdir/servicemonitormetadata.json | jq -s add > /workdir/secrets/servicemonitorobject.json
+            {{- end }}
           volumeMounts:
             - name: workdir
               mountPath: /workdir
@@ -126,7 +143,7 @@ spec:
             - kubectl
             - apply
             - "-f"
-            - /workdir/object.json
+            - /workdir/secrets
           volumeMounts:
             - name: workdir
               mountPath: /workdir

charts/minio/templates/post-install-prometheus-metrics-role.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.serviceAccount.create -}}
+{{- if or .Values.metrics.serviceMonitor.enabled .Values.metrics.podMonitor.enabled }}
 {{- $fullName := include "minio.fullname" . -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -20,7 +20,12 @@ rules:
       - update
       - patch
     resourceNames:
-      - {{ $fullName }}-prometheus
+    {{- if .Values.metrics.serviceMonitor.enabled }}
+      - {{ $fullName }}-servicemonitor-prometheus
+    {{- end }}
+    {{- if .Values.metrics.podMonitor.enabled }}
+      - {{ $fullName }}-podmonitor-prometheus
+    {{- end}}
   - apiGroups:
       - ""
     resources:
@@ -30,7 +35,12 @@ rules:
   - apiGroups:
       - monitoring.coreos.com
     resources:
+    {{- if .Values.metrics.serviceMonitor.enabled }}
       - servicemonitors
+    {{- end}}
+    {{- if .Values.metrics.podMonitor.enabled }}
+      - podmonitors
+    {{- end}}
     verbs:
       - get
     resourceNames:

charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.serviceAccount.create -}}
+{{- if or .Values.metrics.serviceMonitor.enabled .Values.metrics.podMonitor.enabled }}
 {{- $fullName := include "minio.fullname" . -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

charts/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.serviceAccount.create -}}
+{{- if or .Values.metrics.serviceMonitor.enabled .Values.metrics.podMonitor.enabled }}
 {{- $fullName := include "minio.fullname" . -}}
 apiVersion: v1
 kind: ServiceAccount

charts/minio/templates/servicemonitor.yaml

@@ -21,7 +21,7 @@ metadata:
 spec:
   endpoints:
     - port: {{ $scheme }}
-      path: /minio/prometheus/metrics
+      path: /minio/v2/metrics/cluster
       {{- if .Values.metrics.serviceMonitor.interval }}
       interval: {{ .Values.metrics.serviceMonitor.interval }}
       {{- end }}
@@ -32,7 +32,7 @@ spec:
 {{ toYaml .Values.metrics.serviceMonitor.relabelConfigs | indent 6 }}
       {{- end }}
       bearerTokenSecret:
-        name: {{ template "minio.fullname" . }}-prometheus
+        name: {{ template "minio.fullname" . }}-servicemonitor-prometheus
         key: token
   namespaceSelector:
     matchNames:

charts/minio/values.yaml

@@ -344,6 +344,10 @@ metrics:
     # namespace: monitoring
     # interval: 30s
     # scrapeTimeout: 10s
+  podMonitor:
+    enabled: false
+    # interval: 30s
+    # scrapeTimeout: 10s
 
 ## ETCD settings: https://github.com/minio/minio/blob/master/docs/sts/etcd.md
 ## Define endpoints to enable this section.