feat: add monitoring and logging support with Prometheus, Grafana, Lo…

…ki, and Fluentd

- Updated .pre-commit-config.yaml to exclude new Dockerfile path
- Added various PHONY targets to Makefile for clearer command definitions
- Updated compose.yml to include Prometheus, Fluentd, Loki, and Grafana services
- Set up Dockerfile for Fluentd and its configuration
- Created Grafana dashboards and provisioning files
- Added Loki and Prometheus configurations

.pre-commit-config.yaml

@@ -67,6 +67,7 @@ repos:
             LICENSE|
             Makefile|
             README.md|
+            deployments/resources/fluentd/Dockerfile|
             pkg/bliss/configfx/README.md|
             pkg/bliss/datafx/README.md|
             pkg/bliss/httpfx/README.md|

Makefile

@@ -1,8 +1,8 @@
-.PHONY: init dev build multiarch-build generate clean run test test-api test-cov test-ci dep lint container-start container-start-prod container-rebuild container-rebuild-prod container-restart container-restart-prod container-stop container-stop-prod container-destroy container-destroy-prod container-update container-update-prod container-dev container-ps container-ps-prod container-logs-all container-logs-all-prod container-logs container-logs-prod container-cli container-cli-prod container-push
 # .RECIPEPREFIX := $(.RECIPEPREFIX)<space>
 BINARY_NAME=service-cli
 TESTCOVERAGE_THRESHOLD=0
 
+.PHONY: init
 init:
 	brew install pre-commit
 	brew install make
@@ -11,12 +11,15 @@ init:
 	go install golang.org/x/vuln/cmd/govulncheck@latest
 	go install github.com/jandelgado/gcov2lcov@latest
 
+.PHONY: dev
 dev:
 	air; if [ $$? -ne 0 ]; then go run ./cmd/${BINARY_NAME}/; fi
 
+.PHONY: build
 build:
 	go build -o ./tmp/dist/${BINARY_NAME} ./cmd/${BINARY_NAME}/
 
+.PHONY: multiarch-build
 multiarch-build:
 	GOARCH=amd64 GOOS=darwin go build -o ./tmp/dist/${BINARY_NAME}-darwin-amd64 ./cmd/${BINARY_NAME}
 	GOARCH=amd64 GOOS=linux go build -o ./tmp/dist/${BINARY_NAME}-linux-amd64 ./cmd/${BINARY_NAME}
@@ -25,31 +28,39 @@ multiarch-build:
 	GOARCH=arm64 GOOS=linux go build -o ./tmp/dist/${BINARY_NAME}-linux-arm64 ./cmd/${BINARY_NAME}
 	GOARCH=arm64 GOOS=windows go build -o ./tmp/dist/${BINARY_NAME}-windows-arm64 ./cmd/${BINARY_NAME}
 
+.PHONY: generate
 generate:
 	go generate ./...
 
+.PHONY: clean
 clean:
 	go clean
 
+.PHONY: run
 run: build
 	./tmp/dist/${BINARY_NAME}
 
+.PHONY: test-api
 test-api:
 	cd ./deployments/api/ && \
 	bru run ./ --env development && \
 	cd ../../
 
+.PHONY: test
 test:
 	go test -failfast -count 1 ./...
 
+.PHONY: test-cov
 test-cov:
 	go test -failfast -count 1 -coverpkg=./... -coverprofile=${TMPDIR}cov_profile.out ./...
 	# `go env GOPATH`/bin/gcov2lcov -infile ${TMPDIR}cov_profile.out -outfile ./cov_profile.lcov
 
+.PHONY: test-view-html
 test-view-html:
 	go tool cover -html ${TMPDIR}cov_profile.out -o ${TMPDIR}cov_profile.html
 	open ${TMPDIR}cov_profile.html
 
+.PHONY: test-ci
 test-ci: test-cov
 	$(eval ACTUAL_COVERAGE := $(shell go tool cover -func=${TMPDIR}cov_profile.out | grep total | grep -Eo '[0-9]+\.[0-9]+'))
 
@@ -65,76 +76,100 @@ test-ci: test-cov
     echo "OK"; \
   fi
 
+.PHONY: dep
 dep:
 	go mod download
 	go mod tidy
 
+.PHONY: lint
 lint:
 	`go env GOPATH`/bin/golangci-lint run
 
+.PHONY: container-start
 container-start:
 	docker compose --file ./deployments/compose.yml up --detach
 
+.PHONY: container-start-prod
 container-start-prod:
 	docker compose --file ./deployments/compose.production.yml up --detach
 
+.PHONY: container-rebuild
 container-rebuild:
 	docker compose --file ./deployments/compose.yml up --detach --build
 
+.PHONY: container-rebuild-prod
 container-rebuild-prod:
 	docker compose --file ./deployments/compose.production.yml up --detach --build
 
+.PHONY: container-restart
 container-restart:
 	docker compose --file ./deployments/compose.yml restart
 
+.PHONY: container-restart-prod
 container-restart-prod:
 	docker compose --file ./deployments/compose.production.yml restart
 
+.PHONY: container-stop
 container-stop:
 	docker compose --file ./deployments/compose.yml stop
 
+.PHONY: container-stop-prod
 container-stop-prod:
 	docker compose --file ./deployments/compose.production.yml stop
 
+.PHONY: container-destroy
 container-destroy:
 	docker compose --file ./deployments/compose.yml down
 
+.PHONY: container-destroy-prod
 container-destroy-prod:
 	docker compose --file ./deployments/compose.production.yml down
 
+.PHONY: container-update
 container-update:
 	docker compose --file ./deployments/compose.yml pull
 
+.PHONY: container-update-prod
 container-update-prod:
 	docker compose --file ./deployments/compose.production.yml pull
 
+.PHONY: container-dev
 container-dev:
 	docker compose --file ./deployments/compose.yml watch
 
+.PHONY: container-ps
 container-ps:
 	docker compose --file ./deployments/compose.yml ps --all
 
+.PHONY: container-ps-prod
 container-ps-prod:
 	docker compose --file ./deployments/compose.production.yml ps --all
 
+.PHONY: container-logs-all
 container-logs-all:
 	docker compose --file ./deployments/compose.yml logs
 
+.PHONY: container-logs-all-prod
 container-logs-all-prod:
 	docker compose --file ./deployments/compose.production.yml logs
 
+.PHONY: container-logs
 container-logs:
 	docker compose --file ./deployments/compose.yml logs go-service
 
+.PHONY: container-logs-prod
 container-logs-prod:
 	docker compose --file ./deployments/compose.production.yml logs go-service
 
+.PHONY: container-cli
 container-cli:
 	docker compose --file ./deployments/compose.yml exec go-service bash
 
+.PHONY: container-cli-prod
 container-cli-prod:
 	docker compose --file ./deployments/compose.production.yml exec go-service bash
 
+.PHONY: container-push
 container-push:
 ifdef VERSION
 	docker build --platform=linux/amd64 -t acikyazilim.registry.cpln.io/golang-service-template:v$(VERSION) .

deployments/compose.yml

@@ -26,22 +26,125 @@ services:
     ports:
       - 8080:8080
     depends_on:
-      - postgres
+      fluentd:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: localhost:24224
+        tag: go-service
 
   postgres:
     image: postgres:16-bookworm
     restart: unless-stopped
     environment:
       POSTGRES_PASSWORD: s3cr3t
     healthcheck:
-      test: ["CMD-SHELL", "psql -U 'postgres' -c '\\q'"]
+      test: [ "CMD-SHELL", "psql -U 'postgres' -c '\\q'" ]
       interval: 10s
       timeout: 5s
       retries: 10
     volumes:
+      - postgres-data:/var/lib/postgresql/data
       - ./resources/init.sql:/docker-entrypoint-initdb.d/init.sql
+    networks:
+      - go-service-network
     ports:
       - 5435:5432
+    depends_on:
+      fluentd:
+        condition: service_healthy
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: localhost:24224
+        tag: postgres
+
+  prometheus:
+    image: prom/prometheus:latest
+    configs:
+      - source: prometheus-config
+        target: /etc/prometheus/prometheus.yml
+    networks:
+      - go-service-network
+    ports:
+      - 9090:9090
+
+  fluentd:
+    build:
+      context: ./resources/fluentd/
+    environment:
+      FLUENTD_CONF: "fluent.conf"
+    configs:
+      - source: fluentd-config
+        target: /fluentd/etc/fluent.conf
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "localhost", "24224" ]
+      interval: 10s
+      retries: 5
+      timeout: 5s
+    volumes:
+      # - host_logs:/var/log
+      # Needed for journald log ingestion:
+      - /etc/machine-id:/etc/machine-id
+      - /dev/log:/dev/log
+      - /var/run/systemd/journal/:/var/run/systemd/journal/
+    networks:
+      - go-service-network
+    ports:
+      - 24224:24224
+      - 24224:24224/udp
+    depends_on:
+      loki:
+        condition: service_started
+    logging:
+      options:
+        tag: infra.monitoring
+
+  loki:
+    image: grafana/loki:main
+    user: "0"
+    configs:
+      - source: loki-config
+        target: /etc/loki/local-config.yaml
+    volumes:
+      - loki-data:/var/loki
+    command: -config.file=/etc/loki/local-config.yaml
+    networks:
+      - go-service-network
+    ports:
+      - 3100:3100
+
+  grafana:
+    image: grafana/grafana:main
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: admin
+      GF_AUTH_ANONYMOUS_ENABLED: true
+      GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
+    volumes:
+      # - grafana-data:/var/lib/grafana
+      - ./resources/grafana/provisioning:/etc/grafana/provisioning
+      - ./resources/grafana/dashboards:/var/lib/grafana/dashboards
+    networks:
+      - go-service-network
+    ports:
+      - 3000:3000
+
+configs:
+  prometheus-config:
+    file: ./resources/prometheus/prometheus.yml
+  fluentd-config:
+    file: ./resources/fluentd/fluentd.conf
+  loki-config:
+    file: ./resources/loki/local-config.yaml
+
+volumes:
+  postgres-data:
+  # grafana-data:
+  loki-data:
 
 networks:
   go-service-network:

deployments/resources/fluentd/Dockerfile

@@ -0,0 +1,8 @@
+FROM fluent/fluentd:edge-debian
+
+USER root
+
+RUN apt-get update && apt-get install -y netcat-openbsd
+RUN gem install fluent-plugin-grafana-loki
+
+USER fluent

deployments/resources/fluentd/fluentd.conf

@@ -0,0 +1,32 @@
+<source>
+  @type forward
+  port 24224
+</source>
+
+<match **>
+  @type loki
+  url http://loki:3100
+  # url http://loki:3100/loki/api/v1/push
+  # insecure_tls true
+  # tenant ${$.kubernetes.labels.tenant}
+
+  extra_labels {"env":"dev"}
+  # extract_kubernetes_labels true
+  # remove_keys kubernetes
+  <label>
+    # worker fluentd_worker
+    # container $.kubernetes.container
+  </label>
+
+  <buffer>
+    chunk_limit_size 1m
+    flush_interval 10s
+    flush_at_shutdown true
+  </buffer>
+</match>
+
+<label @FLUENT_LOG>
+  <match **>
+    @type stdout
+  </match>
+</label>