setup consul with kubectl

jckling · Oct 17, 2023 · 9c5d68c · 9c5d68c
1 parent 833c4ad
commit 9c5d68c
Show file tree

Hide file tree

Showing 14 changed files with 373 additions and 16 deletions.
diff --git a/README.md b/README.md
@@ -1,8 +1,7 @@
 # observability-demo
 
 TODO：
-- kubectl/helm：promail + consul + demo
-- k8s: rolebinding, ingress authentication
+- helm
 - docker：通过 promtail 发送容器日志到 loki（缺点：不同系统路径可能不同，目前使用 loki docker plugin 发送容器日志数据）
 
 ## 参阅

diff --git a/k8s/README.md b/k8s/README.md
@@ -100,14 +100,15 @@ kubectl exec -it grafana-68768c78b7-4spss  -n monitoring -- /bin/bash
 localhost 访问需要手动指定端口转发
 
 部署地址
-- Prometheus: http://localhost:9090
+- Prometheus: http://monitoring.unity.com/prometheus
 - Pushgateway: http://monitoring.unity.com/pushgateway
 - Alertmanager: http://localhost:9093
 - Grafana: http://monitoring.unity.com/grafana
   - username: `admin`
   - password: `grafana`
+- Consul: http://monitoring.unity.com/consul
 
-### pushgateway
+### Pushgateway
 
 向 pushgateway 推送数据
 
@@ -130,7 +131,7 @@ EOF
 
 ![](images/grafana-metric.png)
 
-### loki
+### Loki
 
 向 loki 推送日志数据
 
@@ -144,7 +145,7 @@ curl -v -H "Content-Type: application/json" -XPOST -s "http://monitoring.unity.c
 ![](images/grafana-loki.png)
 
 
-### promtail
+### Promtail
 
 创建 DaemonSet（`default` 命名空间）
 
@@ -169,6 +170,30 @@ kubectl delete -f nginx.yaml
 kubectl delete -f daemonset.yaml
 ```
 
+### Consul
+
+进入 `observability-demo/k8s/kubectl/consul` 目录，调用脚本注册服务
+
+```bash
+# 注册服务
+./register-service.sh -r
+
+# 删除服务
+./register-service.sh -d
+```
+
+进入 `observability-demo/k8s/kubectl/pushgateway` 目录，调用脚本推送
+
+```bash
+# 推送指标
+./push-metrics.sh -p
+
+# 删除指标
+./push-metrics.sh -d
+```
+
+如果没有注册服务，prometheus 会读不到推给 pushgateway 的数据
+
 ## 参阅
 
 kubectl

diff --git a/k8s/kubectl/configmap.yaml b/k8s/kubectl/configmap.yaml
@@ -14,9 +14,13 @@ data:
       evaluation_interval: 15s
     alerting:
       alertmanagers:
-        - static_configs:
-          - targets:
-            - alertmanager:9093
+        - consul_sd_configs:
+          - server: "consul:8500"
+            services:
+              - alertmanager
+        # - static_configs:
+        #   - targets:
+        #     - alertmanager:9093
     scrape_configs:
       - job_name: 'prometheus'
         static_configs:
@@ -25,9 +29,13 @@ data:
         honor_timestamps: true
         metrics_path: /metrics
         scheme: http
-        static_configs:
-          - targets:
-              - pushgateway:9091
+        consul_sd_configs:
+          - server: "consul:8500"
+            services:
+              - pushgateway
+        # static_configs:
+        #   - targets:
+        #       - pushgateway:9091
 
 ---
 ## alertmanager
@@ -137,4 +145,30 @@ data:
             prefix: index_
             period: 24h
     ruler:
-      alertmanager_url: http://alertmanager:9093
+      alertmanager_url: http://alertmanager:9093
+
+---
+## consul
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: consul-conf
+  labels:
+    name: consul-conf
+  namespace: monitoring
+data:
+  server.json: |-
+    {
+      "datacenter":"dc1",
+      "node_name":"consul",
+      "server":true,
+      "bootstrap":true,
+      "log_level":"INFO",
+      "data_dir":"/consul/data",
+      "ui_config":{
+          "enabled":true
+      },
+      "addresses":{
+          "http":"0.0.0.0"
+      }
+    }
diff --git a/k8s/kubectl/consul/alertmanager.json b/k8s/kubectl/consul/alertmanager.json
@@ -0,0 +1,22 @@
+{
+    "ID":"alertmanager",
+    "Name":"alertmanager",
+    "Tags":[
+        "alertmanager",
+        "v1"
+    ],
+    "Address": "alertmanager",
+    "Port":9093,
+    "Meta":{
+        "os":"linux"
+    },
+    "EnableTagOverride":false,
+    "Check":{
+        "http": "http://monitoring.unity.com/alertmanager/-/healthy",
+        "interval": "10s"
+    },
+    "Weights":{
+        "Passing":10,
+        "Warning":1
+    }
+}
diff --git a/k8s/kubectl/consul/grafana.json b/k8s/kubectl/consul/grafana.json
@@ -0,0 +1,22 @@
+{
+    "ID":"grafana",
+    "Name":"grafana",
+    "Tags":[
+        "grafana",
+        "v1"
+    ],
+    "Address": "grafana",
+    "Port":3000,
+    "Meta":{
+        "os":"linux"
+    },
+    "EnableTagOverride":false,
+    "Check":{
+        "http": "http://monitoring.unity.com/consul/grafana/api/health",
+        "interval": "10s"
+    },
+    "Weights":{
+        "Passing":10,
+        "Warning":1
+    }
+}
diff --git a/k8s/kubectl/consul/loki.json b/k8s/kubectl/consul/loki.json
@@ -0,0 +1,22 @@
+{
+    "ID":"loki",
+    "Name":"loki",
+    "Tags":[
+        "loki",
+        "v1"
+    ],
+    "Address": "loki",
+    "Port":3100,
+    "Meta":{
+        "os":"linux"
+    },
+    "EnableTagOverride":false,
+    "Check":{
+        "http": "http://monitoring.unity.com/consul/loki/ready",
+        "interval": "10s"
+    },
+    "Weights":{
+        "Passing":10,
+        "Warning":1
+    }
+}
diff --git a/k8s/kubectl/consul/prometheus.json b/k8s/kubectl/consul/prometheus.json
@@ -0,0 +1,22 @@
+{
+    "ID":"prometheus",
+    "Name":"prometheus",
+    "Tags":[
+        "prometheus",
+        "v1"
+    ],
+    "Address": "prometheus",
+    "Port":9090,
+    "Meta":{
+        "os":"linux"
+    },
+    "EnableTagOverride":false,
+    "Check":{
+        "http": "http://monitoring.unity.com/consul/prometheus/-/healthy",
+        "interval": "10s"
+    },
+    "Weights":{
+        "Passing":10,
+        "Warning":1
+    }
+}
diff --git a/k8s/kubectl/consul/pushgateway.json b/k8s/kubectl/consul/pushgateway.json
@@ -0,0 +1,22 @@
+{
+    "ID":"pushgateway",
+    "Name":"pushgateway",
+    "Tags":[
+        "pushgateway",
+        "v1"
+    ],
+    "Address": "pushgateway",
+    "Port":9091,
+    "Meta":{
+        "os":"linux"
+    },
+    "EnableTagOverride":false,
+    "Check":{
+        "http": "http://monitoring.unity.com/consul/pushgateway/-/healthy",
+        "interval": "10s"
+    },
+    "Weights":{
+        "Passing":10,
+        "Warning":1
+    }
+}
diff --git a/k8s/kubectl/consul/register-service.sh b/k8s/kubectl/consul/register-service.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+services=("prometheus" "pushgateway" "alertmanager" "grafana" "loki")
+
+Register()
+{
+    for service in ${services[@]}; do
+        filename=$service".json"
+        curl -X PUT --data @$filename http://monitoring.unity.com/consul/v1/agent/service/register
+    done
+}
+
+Deregister()
+{
+    for service in ${services[@]}; do
+        curl -X PUT http://monitoring.unity.com/consul/v1/agent/service/deregister/$service
+    done
+}
+
+while getopts "rd" option; do
+   case $option in
+    r)
+        Register
+        exit;;
+    d)
+        Deregister
+        exit;;
+   esac
+done
diff --git a/k8s/kubectl/deployment.yaml b/k8s/kubectl/deployment.yaml
@@ -232,4 +232,50 @@ spec:
             name: loki-conf
         - name: loki-storage-volume
           persistentVolumeClaim:
-            claimName: loki-pvc
+            claimName: loki-pvc
+
+---
+## consul
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: consul
+  namespace: monitoring
+  labels:
+    app: consul
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: consul
+  template:
+    metadata:
+      labels:
+        app: consul
+    spec:
+      containers:
+        - name: consul
+          image: hashicorp/consul
+          ports:
+            - name: consul-port
+              containerPort: 8500
+          resources:
+            requests:
+              cpu: '1'
+              memory: 500M
+            limits:
+              cpu: '1'
+              memory: 1Gi
+          volumeMounts:
+            - name: consul-config-volume
+              mountPath: /consul/config/server.json
+            - name: consul-storage-volume
+              mountPath: /consul/data
+      volumes:
+        - name: consul-config-volume
+          configMap:
+            defaultMode: 420
+            name: consul-conf
+        - name: consul-storage-volume
+          persistentVolumeClaim:
+            claimName: consul-pvc