WIP loki clustering - BLOCKED

infrastructure/ansible/roles/docker/files/docker-daemon.json

@@ -2,6 +2,7 @@
   "log-driver": "json-file",
   "log-opts": {
     "max-size": "10m",
-    "max-file": "3"
+    "max-file": "3",
+    "labels-regex": "^.+"
   }
 }

monitoring/docker-compose.dev.yml

@@ -12,3 +12,27 @@ services:
       - target: 9090
         published: 9090
         mode: host
+
+  loki:
+    ports:
+      - target: 3100
+        published: 3100
+        mode: host
+
+  loki2:
+    ports:
+      - target: 3100
+        published: 3101
+        mode: host
+
+  loki3:
+    ports:
+      - target: 3100
+        published: 3102
+        mode: host
+
+  minio1:
+    ports:
+      - target: 9001
+        published: 9001
+        mode: host

monitoring/docker-compose.yml

@@ -49,6 +49,8 @@ services:
         source: kminion-groups_rev1.json
       - target: /etc/grafana/provisioning/dashboards/applications/kminion-topic_rev1.json
         source: kminion-topic_rev1.json
+      - target: /etc/grafana/provisioning/dashboards/containers/logging-universal-dashboard_rev1.json
+        source: logging-universal-dashboard_rev1.json
 
   prometheus:
     image: prom/prometheus:v2.38.0
@@ -94,6 +96,88 @@ services:
       - KAFKA_BROKER_LIST=kafka:9092
       - KAFKA_COMPRESSION=gzip
 
+  loki:
+    image: grafana/loki:2.7.1
+    configs:
+      - target: /etc/loki/loki-config.yml
+        source: loki-config.yml
+    command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true -print-config-stderr
+    environment:
+      - NODE_NAME=loki
+      - MEMBER1=loki2
+      - MEMBER2=loki3
+    deploy:
+      labels:
+        - prometheus-job-service=loki
+        - prometheus-address=loki:3100
+
+  loki2:
+    image: grafana/loki:2.7.1
+    configs:
+      - target: /etc/loki/loki-config.yml
+        source: loki-config.yml
+    command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true -print-config-stderr
+    environment:
+      - NODE_NAME=loki2
+      - MEMBER1=loki1
+      - MEMBER2=loki3
+    deploy:
+      labels:
+        - prometheus-job-service=loki2
+        - prometheus-address=loki2:3100
+
+  loki3:
+    image: grafana/loki:2.7.1
+    configs:
+      - target: /etc/loki/loki-config.yml
+        source: loki-config.yml
+    command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true -print-config-stderr
+    environment:
+      - NODE_NAME=loki3
+      - MEMBER1=loki2
+      - MEMBER2=loki1
+    deploy:
+      labels:
+        - prometheus-job-service=loki3
+        - prometheus-address=loki3:3100
+
+  promtail:
+    image: grafana/promtail:2.6.1
+    volumes:
+      - /var/lib/docker/containers:/host/containers
+      - /var/log:/var/log:ro
+    configs:
+      - target: /etc/promtail/promtail-config.yml
+        source: promtail-config.yml
+    command: -config.file=/etc/promtail/promtail-config.yml
+    deploy:
+      mode: global
+
+  minio1:
+    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio{1...1}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://localhost:9000/minio/health/live"
+        ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio1
+    volumes:
+      - minio_data1:/data1
+      - minio_data2:/data2
+    deploy:
+      replicas: 1
+
 configs:
   grafana.ini:
     file: ./grafana/grafana.ini
@@ -135,12 +219,29 @@ configs:
     name: kminion-topic_rev1.json-${kminion_topic_rev1_json_DIGEST:?err}
     labels:
       name: grafana
+  logging-universal-dashboard_rev1.json:
+    file: ./grafana/dashboards/containers/logging-universal-dashboard_rev1.json
+    name: logging-universal-dashboard_rev1.json-${logging_universal_dashboard_rev1_json_DIGEST:?err}
+    labels:
+      name: grafana
   prometheus.yml:
     file: ./prometheus/prometheus.yml
     name: prometheus.yml-${prometheus_yml_DIGEST:?err}
     labels:
       name: prometheus
+  loki-config.yml:
+    file: ./loki/loki-config.yml
+    name: loki-config.yml-${loki_config_yml_DIGEST:?err}
+    labels:
+      name: loki
+  promtail-config.yml:
+    file: ./promtail/promtail-config.yml
+    name: promtail-config.yml-${promtail_config_yml_DIGEST:?err}
+    labels:
+      name: promtail
 
 volumes:
   prometheus_data:
   grafana_data:
+  minio_data1:
+  minio_data2: