[FEATURE] Prometheus/Grafana 환경 추가

.github/workflows/workflow.yml

@@ -44,7 +44,7 @@ jobs:
           username: ${{ secrets.REMOTE_USER }}
           key: ${{ secrets.SSH_PRIVATE_KEY }}
           source: ./deployment/
-          target: /home/ubuntu/deployment
+          target: /home/ubuntu/dearbelly
 
   deploy:
     name: Deploy

app/main.py

@@ -2,6 +2,7 @@
 from app.core.lifespan import lifespan
 from app.api.endpoints import predictions
 from app.core.logging_config import setup_logging
+from prometheus_fastapi_instrumentator import Instrumentator
 
 setup_logging()
 app = FastAPI(
@@ -11,6 +12,8 @@
     lifespan=lifespan
 )
 
+instrumentator = Instrumentator().instrument(app)
+instrumentator.expose(app, include_in_schema=False, endpoint="/actuator/prometheus")
 app.include_router(predictions.router, prefix="/api/v1", tags=["Prediction"])
 
 @app.get("/health")

deployment/deploy.sh

@@ -22,8 +22,8 @@ fi
 
 echo "Pulling new image"
 # docker pull
-docker compose pull app-${AFTER_COMPOSE_COLOR}
-docker compose up -d --no-deps --force-recreate app-${AFTER_COMPOSE_COLOR}
+docker compose -f docker-compose.yml pull app-${AFTER_COMPOSE_COLOR}
+docker compose -f docker-compose.yml up -d --no-deps --force-recreate app-${AFTER_COMPOSE_COLOR}
 
 
 # 새 컨테이너가 running 될 때까지 대기

deployment/docker-compose.monitoring.yml

@@ -0,0 +1,33 @@
+services:
+  prometheus:
+    user: ${PROMETHEUS_USER}
+    image: prom/prometheus:latest
+    container_name: prometheus
+    volumes:
+      - /home/ubuntu/dearbelly/deployment/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - /home/ubuntu/prometheus/volume:/prometheus/data
+    ports:
+      - 9090:9090
+    command:
+      - '--web.enable-lifecycle'
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.retention.time=15d'
+    restart: always
+    networks:
+      - mynetwork
+
+  grafana:
+    user: ${GRAFANA_USER}
+    image: grafana/grafana:latest
+    container_name: grafana
+    ports:
+      - 3000:3000
+    volumes:
+      - /home/ubuntu/grafana/volume:/var/lib/grafana
+    restart: always
+    networks:
+      - mynetwork
+
+networks:
+  mynetwork:
+    driver: bridge

deployment/prometheus-rule.yml

@@ -0,0 +1,17 @@
+groups:
+  - name: example
+    rules:
+      - alert: InstanceDown
+        expr: up == 0
+        for: 5m
+        labels:
+          severity: page
+        annotations:
+          summary: "Instance {{ $labels.instance }} down"
+          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
+      - alert: APIHighRequestLatency
+        expr: api_http_request_latencies_second{quantile="0.5"} > 1
+        for: 10m
+        annotations:
+          summary: "High request latency on {{ $labels.instance }}"
+          description: "{{ $labels.instance }} has a median request latency above 1s (current value: {{ $value }}s)"

deployment/prometheus.yml

@@ -0,0 +1,34 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 15s
+  evaluation_interval: 2m
+  external_labels:
+    monitor: 'codelab-monitor'
+  query_log_file: query_log_file.log
+rule_files:
+  - "prometheus-rule.yml"
+scrape_configs:
+  - job_name: 'prometheus'
+    scrape_interval: 10s
+    scrape_timeout: 10s
+    metrics_path: '/metrics'
+    honor_labels: false
+    honor_timestamps: false
+    scheme: 'http'
+    static_configs:
+      - targets: ['${REMOTE_HOST}:9090']
+        labels:
+          service: 'monitor-1'
+  - job_name: 'node'
+    static_configs:
+      - targets: ['${REMOTE_HOST}:9090']
+  - job_name: 'fastapi-actuator-blue'
+    metrics_path: '/actuator/prometheus'
+    scrape_interval: 1m
+    static_configs:
+      - targets: [ 'app-blue:8000' ]
+  - job_name: 'fastapi-actuator-green'
+    metrics_path: '/actuator/prometheus'
+    scrape_interval: 1m
+    static_configs:
+      - targets: [ 'app-green:8001' ]

requirements.txt

@@ -10,4 +10,6 @@ Pillow==11.3.0
 dotenv
 openai
 timm
-logging
+logging
+prometheus-client==0.19.0
+prometheus-fastapi-instrumentator==6.1.0