diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index d502246..d234b64 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -44,7 +44,7 @@ jobs: username: ${{ secrets.REMOTE_USER }} key: ${{ secrets.SSH_PRIVATE_KEY }} source: ./deployment/ - target: /home/ubuntu/deployment + target: /home/ubuntu/dearbelly deploy: name: Deploy diff --git a/app/main.py b/app/main.py index 3e852a0..19c2ffc 100644 --- a/app/main.py +++ b/app/main.py @@ -2,6 +2,7 @@ from app.core.lifespan import lifespan from app.api.endpoints import predictions from app.core.logging_config import setup_logging +from prometheus_fastapi_instrumentator import Instrumentator setup_logging() app = FastAPI( @@ -11,6 +12,8 @@ lifespan=lifespan ) +instrumentator = Instrumentator().instrument(app) +instrumentator.expose(app, include_in_schema=False, endpoint="/actuator/prometheus") app.include_router(predictions.router, prefix="/api/v1", tags=["Prediction"]) @app.get("/health") diff --git a/deployment/deploy.sh b/deployment/deploy.sh index 9f012d4..9ba080b 100644 --- a/deployment/deploy.sh +++ b/deployment/deploy.sh @@ -22,8 +22,8 @@ fi echo "Pulling new image" # docker pull -docker compose pull app-${AFTER_COMPOSE_COLOR} -docker compose up -d --no-deps --force-recreate app-${AFTER_COMPOSE_COLOR} +docker compose -f docker-compose.yml pull app-${AFTER_COMPOSE_COLOR} +docker compose -f docker-compose.yml up -d --no-deps --force-recreate app-${AFTER_COMPOSE_COLOR} # 새 컨테이너가 running 될 때까지 대기 diff --git a/deployment/docker-compose.monitoring.yml b/deployment/docker-compose.monitoring.yml new file mode 100644 index 0000000..a26403f --- /dev/null +++ b/deployment/docker-compose.monitoring.yml @@ -0,0 +1,33 @@ +services: + prometheus: + user: ${PROMETHEUS_USER} + image: prom/prometheus:latest + container_name: prometheus + volumes: + - /home/ubuntu/dearbelly/deployment/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - /home/ubuntu/prometheus/volume:/prometheus/data + ports: + - 9090:9090 + command: + - '--web.enable-lifecycle' + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.retention.time=15d' + restart: always + networks: + - mynetwork + + grafana: + user: ${GRAFANA_USER} + image: grafana/grafana:latest + container_name: grafana + ports: + - 3000:3000 + volumes: + - /home/ubuntu/grafana/volume:/var/lib/grafana + restart: always + networks: + - mynetwork + +networks: + mynetwork: + driver: bridge \ No newline at end of file diff --git a/deployment/prometheus-rule.yml b/deployment/prometheus-rule.yml new file mode 100644 index 0000000..2c72a59 --- /dev/null +++ b/deployment/prometheus-rule.yml @@ -0,0 +1,17 @@ +groups: + - name: example + rules: + - alert: InstanceDown + expr: up == 0 + for: 5m + labels: + severity: page + annotations: + summary: "Instance {{ $labels.instance }} down" + description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes." + - alert: APIHighRequestLatency + expr: api_http_request_latencies_second{quantile="0.5"} > 1 + for: 10m + annotations: + summary: "High request latency on {{ $labels.instance }}" + description: "{{ $labels.instance }} has a median request latency above 1s (current value: {{ $value }}s)" \ No newline at end of file diff --git a/deployment/prometheus.yml b/deployment/prometheus.yml new file mode 100644 index 0000000..13bda8f --- /dev/null +++ b/deployment/prometheus.yml @@ -0,0 +1,34 @@ +global: + scrape_interval: 15s + scrape_timeout: 15s + evaluation_interval: 2m + external_labels: + monitor: 'codelab-monitor' + query_log_file: query_log_file.log +rule_files: + - "prometheus-rule.yml" +scrape_configs: + - job_name: 'prometheus' + scrape_interval: 10s + scrape_timeout: 10s + metrics_path: '/metrics' + honor_labels: false + honor_timestamps: false + scheme: 'http' + static_configs: + - targets: ['${REMOTE_HOST}:9090'] + labels: + service: 'monitor-1' + - job_name: 'node' + static_configs: + - targets: ['${REMOTE_HOST}:9090'] + - job_name: 'fastapi-actuator-blue' + metrics_path: '/actuator/prometheus' + scrape_interval: 1m + static_configs: + - targets: [ 'app-blue:8000' ] + - job_name: 'fastapi-actuator-green' + metrics_path: '/actuator/prometheus' + scrape_interval: 1m + static_configs: + - targets: [ 'app-green:8001' ] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 339d3b7..e904539 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,6 @@ Pillow==11.3.0 dotenv openai timm -logging \ No newline at end of file +logging +prometheus-client==0.19.0 +prometheus-fastapi-instrumentator==6.1.0 \ No newline at end of file