From 967c0c9daaa25df3896d59c73017becde563322e Mon Sep 17 00:00:00 2001 From: JiHoon Date: Sat, 20 Sep 2025 15:43:02 +0900 Subject: [PATCH 1/2] =?UTF-8?q?feat/fix=20:=20loki=20=EA=B2=BD=EB=A1=9C=20?= =?UTF-8?q?=EC=88=98=EC=A0=95=20=EB=B0=8F=20grafana-agent=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy-fastapi.yml | 10 +++++ .../app/core/logging_config.py | 17 +++++--- apps/pre-processing-service/app/main.py | 6 +++ apps/pre-processing-service/poetry.lock | 33 ++++++++++++++- apps/pre-processing-service/pyproject.toml | 2 + docker/production-fastapi/agent-config.yml | 32 ++++++++++++++ docker/production-fastapi/docker-compose.yml | 42 +++++++++++++++++++ 7 files changed, 135 insertions(+), 7 deletions(-) create mode 100644 docker/production-fastapi/agent-config.yml diff --git a/.github/workflows/deploy-fastapi.yml b/.github/workflows/deploy-fastapi.yml index a87c064d..5e1fc206 100644 --- a/.github/workflows/deploy-fastapi.yml +++ b/.github/workflows/deploy-fastapi.yml @@ -64,6 +64,16 @@ jobs: target: "~/app" overwrite: true + - name: Copy agent-config to FastAPI EC2 + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.FASTAPI_SERVER_HOST }} + username: ubuntu + key: ${{ secrets.SERVER_SSH_KEY }} + source: "docker/production-fastapi/agent-config.yml" + target: "~/app" + overwrite: true + - name: Deploy on EC2 uses: appleboy/ssh-action@v1.0.3 with: diff --git a/apps/pre-processing-service/app/core/logging_config.py b/apps/pre-processing-service/app/core/logging_config.py index bffca718..a511aee2 100644 --- a/apps/pre-processing-service/app/core/logging_config.py +++ b/apps/pre-processing-service/app/core/logging_config.py @@ -14,17 +14,22 @@ def setup_file_logging(): """ PromTail을 통해 Loki로 전송하기 위한 파일 로깅 설정 + 환경변수 MODE에 따라 개발/운영 환경 분리 """ # 기존 loguru 핸들러 제거 (기본 콘솔 출력 제거) logger.remove() - log_dir = "../../docker/local/logs/develop" + mode = os.getenv("MODE", "dev").lower() - # 로그 디렉토리가 없으면 생성 - - # 로그 파일 경로 설정 - log_file_path = log_dir + "/pre-processing-app.log" - error_log_file_path = log_dir + "/pre-processing-app-error.log" + if mode == "prd": + log_dir = "/logs/production" + log_file_path = f"{log_dir}/app.log" + error_log_file_path = f"{log_dir}/error.log" + else: + # 개발환경: 기존 로컬 경로 + log_dir = "../../docker/local/logs/develop" + log_file_path = f"{log_dir}/pre-processing-app.log" + error_log_file_path = f"{log_dir}/pre-processing-app-error.log" # trace_id를 포함한 간단한 포맷 문자열 사용 def add_trace_id_filter(record): diff --git a/apps/pre-processing-service/app/main.py b/apps/pre-processing-service/app/main.py index 4bbf3ff1..1ea95694 100644 --- a/apps/pre-processing-service/app/main.py +++ b/apps/pre-processing-service/app/main.py @@ -4,6 +4,7 @@ from starlette.exceptions import HTTPException as StarletteHTTPException from fastapi.exceptions import RequestValidationError from app.middleware.ServiceLoggerMiddleware import ServiceLoggerMiddleware +from prometheus_fastapi_instrumentator import Instrumentator # 파일 로깅 설정 초기화 from app.core.logging_config import setup_file_logging @@ -19,6 +20,11 @@ # --- FastAPI 애플리케이션 인스턴스 생성 --- app = FastAPI(title="pre-processing-service", description="", version="1.0.0") +# --- Prometheus 메트릭스 설정 --- +instrumentator = Instrumentator() +instrumentator.instrument(app) +instrumentator.expose(app, endpoint="/actuator/prometheus") + # --- 예외 핸들러 등록 --- # 등록 순서가 중요합니다: 구체적인 예외부터 등록하고 가장 일반적인 예외(Exception)를 마지막에 등록합니다. app.add_exception_handler(CustomException, custom_exception_handler) diff --git a/apps/pre-processing-service/poetry.lock b/apps/pre-processing-service/poetry.lock index 1c39c240..ca5c20ab 100644 --- a/apps/pre-processing-service/poetry.lock +++ b/apps/pre-processing-service/poetry.lock @@ -1803,6 +1803,37 @@ files = [ {file = "poetry_core-2.2.0.tar.gz", hash = "sha256:b4033b71b99717a942030e074fec7e3082e5fde7a8ed10f02cd2413bdf940b1f"}, ] +[[package]] +name = "prometheus-client" +version = "0.23.1" +description = "Python client for the Prometheus monitoring system." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99"}, + {file = "prometheus_client-0.23.1.tar.gz", hash = "sha256:6ae8f9081eaaaf153a2e959d2e6c4f4fb57b12ef76c8c7980202f1e57b48b2ce"}, +] + +[package.extras] +twisted = ["twisted"] + +[[package]] +name = "prometheus-fastapi-instrumentator" +version = "7.1.0" +description = "Instrument your FastAPI app with Prometheus metrics" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "prometheus_fastapi_instrumentator-7.1.0-py3-none-any.whl", hash = "sha256:978130f3c0bb7b8ebcc90d35516a6fe13e02d2eb358c8f83887cdef7020c31e9"}, + {file = "prometheus_fastapi_instrumentator-7.1.0.tar.gz", hash = "sha256:be7cd61eeea4e5912aeccb4261c6631b3f227d8924542d79eaf5af3f439cbe5e"}, +] + +[package.dependencies] +prometheus-client = ">=0.8.0,<1.0.0" +starlette = ">=0.30.0,<1.0.0" + [[package]] name = "propcache" version = "0.3.2" @@ -3398,4 +3429,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.14" -content-hash = "6e10697924e89b5c0f7c3f6ecd79fb64ac412ac2240f75565d6ea5feb3a89a20" +content-hash = "42274fd00aedabf70dc419acd06e2f25b5c69b58b7bf76eef2ea7a9df6470b2c" diff --git a/apps/pre-processing-service/pyproject.toml b/apps/pre-processing-service/pyproject.toml index 672bf645..84a957b9 100644 --- a/apps/pre-processing-service/pyproject.toml +++ b/apps/pre-processing-service/pyproject.toml @@ -36,6 +36,8 @@ dbutils=">=3.1.2,<4.0.0" onnxruntime = "^1.22.1" openai = "^1.107.3" aiohttp = "^3.12.15" +prometheus-client = "^0.23.1" +prometheus-fastapi-instrumentator = "^7.1.0" [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] diff --git a/docker/production-fastapi/agent-config.yml b/docker/production-fastapi/agent-config.yml new file mode 100644 index 00000000..7119a4e3 --- /dev/null +++ b/docker/production-fastapi/agent-config.yml @@ -0,0 +1,32 @@ +server: + log_level: info + +prometheus: + wal_directory: /tmp/grafana-agent-wal + global: + scrape_interval: 15s + external_labels: + cluster: production + service: pre-processing-service + + configs: + - name: pre-processing-service-metrics + remote_write: + - url: ${GRAFANA_CLOUD_PROMETHEUS_URL} + basic_auth: + username: ${GRAFANA_CLOUD_PROMETHEUS_USER} + password: ${GRAFANA_CLOUD_API_KEY} + + scrape_configs: + - job_name: 'pre-processing-service' + static_configs: + - targets: ['pre-processing-service:8000'] + metrics_path: '/actuator/prometheus' + scrape_interval: 15s + params: + format: ['prometheus'] + + - job_name: 'node-exporter' + static_configs: + - targets: ['node-exporter:9100'] + scrape_interval: 15s \ No newline at end of file diff --git a/docker/production-fastapi/docker-compose.yml b/docker/production-fastapi/docker-compose.yml index 8dd5dbb8..c22a3833 100644 --- a/docker/production-fastapi/docker-compose.yml +++ b/docker/production-fastapi/docker-compose.yml @@ -14,6 +14,8 @@ services: - promtail env_file: - .env.prod + networks: + - app-network promtail: image: grafana/promtail:2.9.0 @@ -31,7 +33,47 @@ services: hard: 65535 env_file: - .env.prod + networks: + - app-network + + grafana-agent: + image: grafana/agent:latest + container_name: grafana-agent + restart: unless-stopped + volumes: + - ./agent-config.yml:/etc/agent/agent.yml:ro + networks: + - app-network + env_file: + - .env.prod + command: + - --config.file=/etc/agent/agent.yml + - --config.expand-env=true + + node-exporter: + image: prom/node-exporter:latest + container_name: node-exporter + restart: unless-stopped + ports: + - "127.0.0.1:9100:9100" # localhost에서만 접근 + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + networks: + - app-network + + volumes: logs_volume: driver: local + +networks: + app-network: + driver: bridge From d480f7169188fb30535ac31511905bb8c6c9bf09 Mon Sep 17 00:00:00 2001 From: JiHoon Date: Sat, 20 Sep 2025 15:50:22 +0900 Subject: [PATCH 2/2] =?UTF-8?q?fix=20:=20env=20=EB=B0=8F=20config=20?= =?UTF-8?q?=EC=88=98=EC=A0=95=20(=ED=99=98=EA=B2=BD=20=EB=B3=80=EC=88=98?= =?UTF-8?q?=20=EC=B6=94=EA=B0=80)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy-fastapi.yml | 3 +++ apps/pre-processing-service/app/core/config.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/deploy-fastapi.yml b/.github/workflows/deploy-fastapi.yml index 5e1fc206..7d68d7bd 100644 --- a/.github/workflows/deploy-fastapi.yml +++ b/.github/workflows/deploy-fastapi.yml @@ -31,6 +31,9 @@ jobs: echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> .env.prod echo "MODE=prd" >> .env.prod echo "APP_NAME=pre-processing" >> .env.prod + echo "GRAFANA_CLOUD_PROMETHEUS_URL=${{ secrets.GRAFANA_CLOUD_PROMETHEUS_URL }}" >> .env.prod + echo "GRAFANA_CLOUD_PROMETHEUS_USER=${{ secrets.GRAFANA_CLOUD_PROMETHEUS_USER }}" >> .env.prod + echo "GRAFANA_CLOUD_API_KEY=${{ secrets.GRAFANA_CLOUD_API_KEY }}" >> .env.prod - name: Set repo lowercase run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV diff --git a/apps/pre-processing-service/app/core/config.py b/apps/pre-processing-service/app/core/config.py index c6f31f49..9580934b 100644 --- a/apps/pre-processing-service/app/core/config.py +++ b/apps/pre-processing-service/app/core/config.py @@ -87,6 +87,10 @@ class BaseSettingsConfig(BaseSettings): loki_password: str loki_port: int = 3100 + grafana_cloud_prometheus_url: Optional[str] = None + grafana_cloud_prometheus_user: Optional[str] = None + grafana_cloud_api_key: Optional[str] = None + # 테스트/추가용 필드 openai_api_key: Optional[str] = None # << 이 부분 추가