monitoring-template.yaml

apiVersion: template.openshift.io/v1
kind: Template
metadata:
  name: prometheus
  annotations:
    "openshift.io/display-name": "ConSol Openshift Monitoring"
    description: |
      A monitoring solution for an OpenShift cluster - collect and gather metrics and alerts from nodes, services, and the infrastructure.
    iconClass: fa fa-cogs
    tags: "monitoring,prometheus,alertmanager,time-series"

parameters:
# General
- description: The namespace to instantiate the monitoring system under. Defaults to 'kube-system'.
  name: NAMESPACE
  value: kube-system

# Prometheus
- description: The location of the prometheus image.
  name: IMAGE_PROMETHEUS
  value: prom/prometheus:v2.3.2
- description: Retention days for Prometheus. Defaults to '10d'
  name: RETENTION_DAYS_PROMETHEUS
  value: 10d
- description: Size of the persistent volume for prometheus. Defaults to '10Gi'.
  name: VOLUMESIZE_PROMETHEUS
  value: 10Gi
- description: Memory limit for Prometheus. Defaults to '8Gi'.
  name: LIMITMEMORY_PROMETHEUS
  value: 8Gi
- description: Requested memory for Prometheus. Defaults to '1Gi'
  name: REQMEMORY_PROMETHEUS
  value: 1Gi

# kube-state-metrics
- description: The location of the kube-state-metrics image.
  name: IMAGE_KUBESTATEMETRICS
  # Use k-s-m >= v1.2.0 for Kubernetes versions >= 1.8
  #value: k8s.gcr.io/kube-state-metrics:v1.2.0
  # Use k-s-m v1.1.0 for Kubernetes versions < 1.8
  value: k8s.gcr.io/kube-state-metrics:v1.1.0

# Grafana
- description: The location of the grafana image.
  name: IMAGE_GRAFANA
  value: grafana/grafana:5.2.2
- description: Size of the persistent volume for prometheus
  name: VOLUMESIZE_GRAFANA
  value: 100Mi
- description: Memory limit for Grafana. Defaults to '1Gi'.
  name: LIMITMEMORY_GRAFANA
  value: 1Gi
- description: Requested memory for Grafana. Defaults to '512Mi'
  name: REQMEMORY_GRAFANA
  value: 512Mi
- description: Default admin user name for Grafana. Defaults to 'root'
  name: ADMIN_USERNAME_GRAFANA
  value: root
- description: Default admin password for Grafana. Defaults to 'secret'
  name: ADMIN_PASSWORD_GRAFANA
  value: secret

objects:
##############################################################################################
#
# Prometheus
#
##############################################################################################
# service account with role cluster-reader
- apiVersion: v1
  kind: ServiceAccount
  metadata:
    name: prometheus
    namespace: "${NAMESPACE}"
- apiVersion: authorization.openshift.io/v1
  kind: ClusterRoleBinding
  metadata:
    name: prometheus-cluster-reader
  roleRef:
    name: cluster-reader
  subjects:
  - kind: ServiceAccount
    name: prometheus
    namespace: "${NAMESPACE}"

- apiVersion: route.openshift.io/v1
  kind: Route
  metadata:
    name: prometheus
    namespace: "${NAMESPACE}"
  spec:
    port:
      targetPort: 9090
    to:
      kind: Service
      name: prometheus

- apiVersion: v1
  kind: Service
  metadata:
    name: prometheus
    namespace: "${NAMESPACE}"
  spec:
    ports:
    - name: prometheus
      port: 9090
      protocol: TCP
      targetPort: 9090
    selector:
      name: prometheus

- apiVersion: v1
  kind: PersistentVolumeClaim
  metadata:
    name: pvc-prometheus
    namespace: "${NAMESPACE}"
  spec:
    accessModes:
      - ReadWriteOnce
    resources:
      requests:
        storage: "${VOLUMESIZE_PROMETHEUS}"

- apiVersion: v1
  kind: DeploymentConfig
  metadata:
    name: prometheus
    namespace: "${NAMESPACE}"
  spec:
    replicas: 1
    selector:
      name: prometheus
    template:
      metadata:
        labels:
          name: prometheus
      spec:
        serviceAccount: prometheus
        serviceAccountName: prometheus
        containers:
        - name: prometheus
          image: "${IMAGE_PROMETHEUS}"
          imagePullPolicy: IfNotPresent
          args:
          - --config.file=/etc/prometheus/prometheus.yml
          - --storage.tsdb.path=/prometheus
          - --storage.tsdb.retention=${RETENTION_DAYS_PROMETHEUS}
          readinessProbe:
            httpGet:
              path: /-/ready
              port: 9090
            initialDelaySeconds: 5
            periodSeconds: 5
          ports:
          - containerPort: 9090
          resources:
            limits:
              memory: "${LIMITMEMORY_PROMETHEUS}"
            requests:
              memory: "${REQMEMORY_PROMETHEUS}"
          volumeMounts:
          - mountPath: /etc/prometheus
            name: config-prometheus
          - mountPath: '/prometheus'
            name: data-prometheus
        restartPolicy: Always
        volumes:
        - name: config-prometheus
          configMap:
            defaultMode: 420
            name: configmap-prometheus
        - name: data-prometheus
          persistentVolumeClaim:
            claimName: pvc-prometheus

- apiVersion: v1
  kind: ConfigMap
  metadata:
    name: configmap-prometheus
    namespace: "${NAMESPACE}"
  data:
    alerting.rules: |
      groups:
      - name: example-rules
        interval: 30s # defaults to global interval
        rules:
        - alert: Node Down
          expr: up{job="kubernetes-nodes"} == 0
          annotations:
            miqTarget: "ContainerNode"
            severity: "HIGH"
            message: "{{$labels.instance}} is down"

    recording.rules: |
      groups:
      - name: aggregate_container_resources
        rules:
        - record: container_cpu_usage_rate
          expr: sum without (cpu) (rate(container_cpu_usage_seconds_total[5m]))
        - record: container_memory_rss_by_type
          expr: container_memory_rss{id=~"/|/system.slice|/kubepods.slice"} > 0
        - record: container_cpu_usage_percent_by_host
          expr: sum by (hostname,type)(rate(container_cpu_usage_seconds_total{id="/"}[5m])) / on (hostname,type) machine_cpu_cores
        - record: apiserver_request_count_rate_by_resources
          expr: sum without (client,instance,contentType) (rate(apiserver_request_count[5m]))

    prometheus.yml: |
      global:
        scrape_interval: 15s
        evaluation_interval: 30s
        scrape_timeout: 10s
        # Attach these labels to any time series or alerts when communicating with
        # external systems (federation, remote storage, Alertmanager).
        external_labels:
            monitor: "prometheus_${NAMESPACE}"

      rule_files:
        - '*.rules'

      scrape_configs:

      - job_name: 'prometheus'
        metrics_path: '/metrics'
        scheme: http
        static_configs:
        - targets: ['localhost:9090']

      # ------------------------------------------------------------------------------
      # Scrape config for API servers.
      #
      # Kubernetes exposes API servers as endpoints to the default/kubernetes
      # service so this uses `endpoints` role and uses relabelling to only keep
      # the endpoints associated with the default/kubernetes service using the
      # default named port `https`. This works for single API server deployments as
      # well as HA API server deployments.
      - job_name: 'kubernetes-apiservers'
        kubernetes_sd_configs:
        - role: endpoints
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        # Keep only the default/kubernetes service endpoints for the https port. This
        # will add targets for each API server which Kubernetes adds an endpoint to
        # the default/kubernetes service.
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: default;kubernetes;https

      # ------------------------------------------------------------------------------
      # Scrape config for controllers.
      #
      # Each master node exposes a /metrics endpoint on :8444 that contains operational metrics for
      # the controllers.
      #
      # TODO: move this to a pure endpoints based metrics gatherer when controllers are exposed via
      #       endpoints.
      - job_name: 'kubernetes-controllers'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
        - role: endpoints
        # Keep only the default/kubernetes service endpoints for the https port, and then
        # set the port to 8444. This is the default configuration for the controllers on OpenShift
        # masters.
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: default;kubernetes;https
        - source_labels: [__address__]
          action: replace
          target_label: __address__
          regex: (.+)(?::\d+)
          replacement: $1:8444

      # ------------------------------------------------------------------------------
      # Scrape config for nodes.
      #
      # Each node exposes a /metrics endpoint that contains operational metrics for
      # the Kubelet and other components.
      - job_name: 'kubernetes-nodes'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
        - role: node
        # FIXME Drop a very high cardinality metric that is incorrect in 3.7. It will be
        # fixed in 3.9.
        metric_relabel_configs:
        - source_labels: [__name__]
          action: drop
          regex: 'openshift_sdn_pod_(setup|teardown)_latency(.*)'
        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)

      # ------------------------------------------------------------------------------
      # Scrape config for cAdvisor.
      #
      # Beginning in Kube 1.7, each node exposes a /metrics/cadvisor endpoint that
      # reports container metrics for each running pod. Scrape those by default.
      - job_name: 'kubernetes-cadvisor'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        metrics_path: /metrics/cadvisor
        kubernetes_sd_configs:
        - role: node
        # Exclude a set of high cardinality metrics that can contribute to significant
        # memory use in large clusters. These can be selectively enabled as necessary
        # for medium or small clusters.
        metric_relabel_configs:
        #TODO drop metrics with empy pod_name
        #- source_labels: [pod_name]
        #  action: drop
        #  regex: '^$'
        #TODO drop metrics with container_name POD
        #- source_labels: [container_name]
        #  action: drop
        #  regex: 'POD'
        - source_labels: [__name__]
          action: drop
          regex: 'container_(cpu_user_seconds_total|cpu_cfs_periods_total|memory_cache|last_seen|fs_(read_seconds_total|write_seconds_total|sector_(.*)|io_(.*)|reads_merged_total|writes_merged_total)|tasks_state|memory_failcnt|memory_failures_total|spec_memory_swap_limit_bytes|fs_(.*)_bytes_total)'
          # re-enabled some of the following potentially expensive metrics
          #regex: 'container_(cpu_user_seconds_total|cpu_cfs_periods_total|memory_usage_bytes|memory_swap|memory_working_set_bytes|memory_cache|last_seen|fs_(read_seconds_total|write_seconds_total|sector_(.*)|io_(.*)|reads_merged_total|writes_merged_total)|tasks_state|memory_failcnt|memory_failures_total|spec_memory_swap_limit_bytes|fs_(.*)_bytes_total|spec_(.*))'
        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)

      # ------------------------------------------------------------------------------
      # Scrape config for service endpoints.
      #
      # The relabeling allows the actual service scrape endpoint to be configured
      # via the following annotations:
      #
      # * `prometheus.io/scrape`: Only scrape services that have a value of `true`
      # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
      # to set this to `https` & most likely set the `tls_config` of the scrape config.
      # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
      # * `prometheus.io/port`: If the metrics are exposed on a different port to the
      # service then set this appropriately.
      - job_name: 'kubernetes-service-endpoints'
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          # TODO: this should be per target
          insecure_skip_verify: true
        kubernetes_sd_configs:
        - role: endpoints
        relabel_configs:
        #TODO only scrape infrastructure components
        #- source_labels: [__meta_kubernetes_namespace]
        #  action: keep
        #  regex: 'default|logging|metrics|kube-.+|openshift|openshift-.+'
        #FIXME router/haproxy metrics dropped for now due to authentication problems
        - source_labels: [__meta_kubernetes_endpoints_name]
          action: drop
          regex: (router.*)
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
          action: replace
          target_label: __scheme__
          regex: (https?)
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
          action: replace
          target_label: __metrics_path__
          regex: (.+)
        - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
          action: replace
          target_label: __address__
          regex: (.+)(?::\d+);(\d+)
          replacement: $1:$2
        - action: labelmap
          regex: __meta_kubernetes_service_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_service_name]
          action: replace
          target_label: kubernetes_name

      # ------------------------------------------------------------------------------
      # Scrape config for the template service broker
      - job_name: 'openshift-template-service-broker'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
          server_name: apiserver.openshift-template-service-broker.svc
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
        - role: endpoints
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: openshift-template-service-broker;apiserver;https

      # ------------------------------------------------------------------------------
      # Scrape config for piods
      #
      # The relabeling allows the actual pod scrape endpoint to be configured via the
      # following annotations:
      #
      # * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
      # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
      # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the
      # pod's declared ports (default is a port-free target if none are declared).
      - job_name: 'kubernetes-pods'
        kubernetes_sd_configs:
        - role: pod
        relabel_configs:
        #drop expensive series from node-exporter pods
        - source_labels: [__name__, device]
          action: drop
          regex: 'node_network_.+;veth.+'
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
          action: replace
          target_label: __metrics_path__
          regex: (.+)
        - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
          action: replace
          regex: ([^:]+)(?::\d+)?;(\d+)
          replacement: $1:$2
          target_label: __address__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: kubernetes_pod_name

##############################################################################################
#
# Grafana
#
##############################################################################################
# This is based upon the findings from http://widerin.net/blog/official-grafana-docker-image-on-openshift/
# - adding a configmap for grafana.ini
# - starting the docker image with custom "command" and "args"
- apiVersion: v1
  kind: ServiceAccount
  metadata:
    name: grafana
    namespace: "${NAMESPACE}"

- apiVersion: route.openshift.io/v1
  kind: Route
  metadata:
    name: grafana
    namespace: "${NAMESPACE}"
  spec:
    port:
      targetPort: 3000
    to:
      kind: Service
      name: grafana

- apiVersion: v1
  kind: Service
  metadata:
    name: grafana
    namespace: "${NAMESPACE}"
  spec:
    ports:
    - name: grafana
      port: 3000
      targetPort: 3000
    selector:
      name: grafana

- apiVersion: v1
  kind: PersistentVolumeClaim
  metadata:
    name: pvc-grafana
    namespace: "${NAMESPACE}"
  spec:
    accessModes:
      - ReadWriteOnce
    resources:
      requests:
        storage: "${VOLUMESIZE_GRAFANA}"

- apiVersion: v1
  kind: DeploymentConfig
  metadata:
    name: grafana
    namespace: "${NAMESPACE}"
  spec:
    replicas: 1
    selector:
      name: grafana
    template:
      metadata:
        labels:
          name: grafana
      spec:
        serviceAccount: grafana
        serviceAccountName: grafana
        containers:
        - name: grafana
          image: ${IMAGE_GRAFANA}
          imagePullPolicy: IfNotPresent
          env:
            - name: GF_AUTH_ANONYMOUS_ENABLED
              value: "false"
            - name: GF_AUTH_BASIC_ENABLED
              value: "true"
          command:
          - /usr/sbin/grafana-server
          args:
          - '--homepath=/usr/share/grafana'
          - '--config=/etc/grafana/grafana.ini'
          readinessProbe:
            httpGet:
              path: /api/health
              port: 3000
            initialDelaySeconds: 5
            periodSeconds: 5
          ports:
          - containerPort: 3000
          resources:
            limits:
              memory: "${LIMITMEMORY_GRAFANA}"
            requests:
              memory: "${REQMEMORY_GRAFANA}"
          volumeMounts:
          - mountPath: /etc/grafana
            name: config-grafana
          - mountPath: /var/lib/grafana
            name: data-grafana
          - mountPath: /dashboards
            name: dashboards-grafana
          #- mountPath: /var/log/grafana
          #  name: log-grafana
          restartPolicy: Always
        volumes:
        - name: config-grafana
          configMap:
            defaultMode: 420
            name: configmap-grafana
            items:
            - key: grafana.ini
              path: grafana.ini
            - key: datasource.yaml
              path: provisioning/datasources/datasource.yaml
            - key: dashboard.yaml
              path: provisioning/dashboards/dashboard.yaml
        #- name: log-grafana
        #  emptyDir: {}
        - name: data-grafana
          persistentVolumeClaim:
            claimName: pvc-grafana
        - name: dashboards-grafana
          configMap:
            name: dashboards-grafana

- apiVersion: v1
  kind: ConfigMap
  metadata:
    name: configmap-grafana
    namespace: "${NAMESPACE}"
  data:
    grafana.ini: |
      [paths]
      data = /var/lib/grafana
      plugins = /var/lib/grafana/plugins
      provisioning = /etc/grafana/provisioning
      #logs = /var/log/grafana

      [log]
      mode = console

      [security]
      admin_user = ${ADMIN_USERNAME_GRAFANA}
      admin_password = ${ADMIN_PASSWORD_GRAFANA}

    datasource.yaml: |
      apiVersion: 1
      datasources:
      - name: prometheus
        type: prometheus
        access: proxy
        org_id: 1
        url: http://prometheus.${NAMESPACE}.svc.cluster.local:9090/
        password:
        user:
        database:
        basic_auth:
        basic_auth_user:
        basic_auth_password:
        with_credentials:
        is_default: true
        version: 1
        editable: false

    dashboard.yaml: |
      apiVersion: 1
      providers:
      - name: 'default'
        org_id: 1
        folder: ''
        type: file
        options:
          path: /dashboards

##############################################################################################
#
# kube-state-metrics
#
##############################################################################################
# kube-state-metrics service account with role cluster-reader
- apiVersion: v1
  kind: ServiceAccount
  metadata:
    name: kube-state-metrics
    namespace: "${NAMESPACE}"
- apiVersion: authorization.openshift.io/v1
  kind: ClusterRoleBinding
  metadata:
    name: kube-state-metrics-cluster-reader
  roleRef:
    name: cluster-reader
  subjects:
  - kind: ServiceAccount
    name: kube-state-metrics
    namespace: "${NAMESPACE}"

- apiVersion: v1
  kind: Service
  metadata:
    name: kube-state-metrics
    namespace: "${NAMESPACE}"
    annotations:
      prometheus.io/scrape: 'true'
      prometheus.io/port: '8080'
  spec:
    ports:
    - name: kube-state-metrics
      port: 8080
      targetPort: 8080
    selector:
      name: kube-state-metrics

- apiVersion: v1
  kind: DeploymentConfig
  metadata:
    name: kube-state-metrics
    namespace: "${NAMESPACE}"
  spec:
    replicas: 1
    selector:
      name: kube-state-metrics
    template:
      metadata:
        labels:
          name: kube-state-metrics
      spec:
        serviceAccount: kube-state-metrics
        serviceAccountName: kube-state-metrics
        containers:
        - name: kube-state-metrics
          image: ${IMAGE_KUBESTATEMETRICS}
          imagePullPolicy: IfNotPresent
          # Flags are needed for k-s-m >= v1.2.0
          #args:
          #- --telemetry-host=127.0.0.1
          #- --telemetry-port=8181
          ports:
          - containerPort: 8080
          readinessProbe:
            httpGet:
              path: /healthz
              port: 8080
            initialDelaySeconds: 5
            timeoutSeconds: 5