diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml index 54d1ab1a..e958f8c3 100644 --- a/.github/workflows/ci-python.yml +++ b/.github/workflows/ci-python.yml @@ -2,8 +2,6 @@ name: CI (Python/FastAPI) on: push: - branches: - - feature/onnx tags: - 'pre-processing-v*' pull_request: @@ -60,17 +58,17 @@ jobs: - name: Run Formatter Check (Black) run: poetry run black --check . - test: - name: Run Tests - runs-on: ubuntu-latest - needs: lint - defaults: - run: - working-directory: apps/pre-processing-service - steps: - - name: Checkout repository - uses: actions/checkout@v4 - +# test: +# name: Run Tests +# runs-on: ubuntu-latest +# needs: lint +# defaults: +# run: +# working-directory: apps/pre-processing-service +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# # - name: Set up Python 3.11 # uses: actions/setup-python@v5 # with: @@ -123,7 +121,7 @@ jobs: name: Build Docker Image and push runs-on: ubuntu-latest needs: - - test +# - test - set-image-tag if: startsWith(github.ref, 'refs/tags/pre-processing-v') diff --git a/.github/workflows/deploy-fastapi.yml b/.github/workflows/deploy-fastapi.yml new file mode 100644 index 00000000..23ee6676 --- /dev/null +++ b/.github/workflows/deploy-fastapi.yml @@ -0,0 +1,148 @@ +name: Deploy FastAPI +on: + push: + branches: + - fix/** + workflow_run: + workflows: ["CI (Python/FastAPI)"] + types: [completed] + +jobs: + deploy: + name: Deploy to AWS FastAPI EC2 + runs-on: ubuntu-latest + if: | + github.event.workflow_run.conclusion == 'success' && + startsWith(github.event.workflow_run.head_branch, 'pre-processing-v') + + steps: + - uses: actions/checkout@v4 + - name: Create env file + run: | + echo "DB_HOST=${{ secrets.DB_HOST }}" > .env.prod + echo "DB_PORT=${{ secrets.DB_PORT }}" >> .env.prod + echo "DB_USER=${{ secrets.DB_USER }}" >> .env.prod + echo "DB_PASS=${{ secrets.DB_PASS }}" >> .env.prod + echo "DB_NAME=${{ secrets.DB_NAME }}" >> .env.prod + echo "LOKI_HOST=${{ secrets.LOKI_HOST }}" >> .env.prod + echo "LOKI_USERNAME=${{ secrets.LOKI_USERNAME }}" >> .env.prod + echo "LOKI_PASSWORD=${{ secrets.LOKI_PASSWORD }}" >> .env.prod + echo "ENV_NAME=${{ secrets.ENV_NAME }}" >> .env.prod + echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> .env.prod + + - name: Set repo lowercase + run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV + + - name: Copy docker compose files to FastAPI EC2 + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.FASTAPI_SERVER_HOST }} + username: ubuntu + key: ${{ secrets.SERVER_SSH_KEY }} + source: "docker/production-fastapi/docker-compose.yml" + target: "~/app" + + - name: Copy .env.prod file to FastAPI EC2 + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.FASTAPI_SERVER_HOST }} + username: ubuntu + key: ${{ secrets.SERVER_SSH_KEY }} + source: ".env.prod" + target: "~/app/docker/production/" + overwrite: true + + - name: Copy promtail-config to FastAPI EC2 + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.FASTAPI_SERVER_HOST }} + username: ubuntu + key: ${{ secrets.SERVER_SSH_KEY }} + source: "docker/production-fastapi/promtail-config.yml" + target: "~/app/docker/production/" + overwrite: true + + - name: Check ONNX model on FastAPI EC2 + id: check-onnx + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ secrets.FASTAPI_SERVER_HOST }} + username: ubuntu + key: ${{ secrets.SERVER_SSH_KEY }} + script: | + if [ -f "~/app/models/klue_bert.onnx" ]; then + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + + - name: Copy ONNX to FastAPI EC2 + if: steps.check-onnx.outputs.exists == 'false' + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.FASTAPI_SERVER_HOST }} + username: ubuntu + key: ${{ secrets.SERVER_SSH_KEY }} + source: "apps/pre-processing-service/klue_bert.onnx" + target: "~/app/models/" + overwrite: false + + - name: Deploy on EC2 + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{secrets.FASTAPI_SERVER_HOST}} + username: ubuntu + key: ${{secrets.SERVER_SSH_KEY}} + script: | + cd ~/app/docker/production + + echo "${{secrets.GITHUB_TOKEN}}" | docker login ghcr.io -u ${{github.actor}} --password-stdin + + docker-compose pull + + docker-compose down + + docker-compose up -d + + sleep 5 + docker-compose ps + + echo "Waiting for containers to become healthy..." + for i in {1..30}; do + unhealthy=$(docker ps --filter "health=unhealthy" --format "{{.Names}}") + starting=$(docker ps --filter "health=starting" --format "{{.Names}}") + if [ -z "$unhealthy" ] && [ -z "$starting" ]; then + echo "All containers are healthy!" + break + fi + echo "Waiting... ($i/30)" + sleep 2 + done + + docker image prune -f + + - name: Send Discord notification - Success + if: success() + uses: Ilshidur/action-discord@master + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK_URL }} + with: + args: | + **배포 성공** + **Repository:** ${{ env.REPO_LC }} + **Tag:** ${{ github.ref_name }} + **Server:** ${{ secrets.FASTAPI_SERVER_HOST }} + **Status:** Success! + + - name: Send Discord notification - Failure + if: failure() + uses: Ilshidur/action-discord@master + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK_URL }} + with: + args: | + **배포 실패** + **Repository:** ${{ env.REPO_LC }} + **Tag:** ${{ github.ref_name }} + **Error:** 배포 중 오류가 발생했습니다. + **Check:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \ No newline at end of file diff --git a/.github/workflows/deploy-java.yml b/.github/workflows/deploy-java.yml index d7526506..3a19d80b 100644 --- a/.github/workflows/deploy-java.yml +++ b/.github/workflows/deploy-java.yml @@ -5,7 +5,7 @@ on: branches: - fix/** workflow_run: - workflows: ["CI (Java)", "CI (Python/FastAPI)"] + workflows: ["CI (Java)"] types: [completed] @@ -15,9 +15,7 @@ jobs: runs-on: ubuntu-latest if: | github.event.workflow_run.conclusion == 'success' && - (startsWith(github.event.workflow_run.head_branch, 'user-service-v') || - startsWith(github.event.workflow_run.head_branch, 'pre-processing-v')) - + startsWith(github.event.workflow_run.head_branch, 'user-service-v') steps: - uses: actions/checkout@v4 @@ -28,10 +26,11 @@ jobs: echo "DB_USER=${{ secrets.DB_USER }}" >> .env.prod echo "DB_PASS=${{ secrets.DB_PASS }}" >> .env.prod echo "DB_NAME=${{ secrets.DB_NAME }}" >> .env.prod - echo "ENV_NAME=${{ secrets.LOKI_URL }}" >> .env.prod - echo "ENV_NAME=${{ secrets.LOKI_USERNAME }}" >> .env.prod - echo "ENV_NAME=${{ secrets.LOKI_PASSWORD }}" >> .env.prod + echo "LOKI_HOST=${{ secrets.LOKI_HOST }}" >> .env.prod + echo "LOKI_USERNAME=${{ secrets.LOKI_USERNAME }}" >> .env.prod + echo "LOKI_PASSWORD=${{ secrets.LOKI_PASSWORD }}" >> .env.prod echo "ENV_NAME=${{ secrets.ENV_NAME }}" >> .env.prod + echo "FASTAPI_HOST=${{ secrets.FASTAPI_SERVER_HOST }}" >> .env.prod - name: Set repo lowercase run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV @@ -75,7 +74,6 @@ jobs: target: "~/app/docker/production/" overwrite: true - - name: Deploy on EC2 uses: appleboy/ssh-action@v1.0.3 with: @@ -140,4 +138,4 @@ jobs: **Repository:** ${{ env.REPO_LC }} **Tag:** ${{ github.ref_name }} **Error:** 배포 중 오류가 발생했습니다. - **Check:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \ No newline at end of file + **Check:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.gitignore b/.gitignore index 06344943..8cf755dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .idea logs *.log -*.env* \ No newline at end of file +*.env* + +*.onnx \ No newline at end of file diff --git a/apps/pre-processing-service/app/core/logging_config.py b/apps/pre-processing-service/app/core/logging_config.py index b2c10d88..50edd1a6 100644 --- a/apps/pre-processing-service/app/core/logging_config.py +++ b/apps/pre-processing-service/app/core/logging_config.py @@ -3,11 +3,9 @@ import sys from contextvars import ContextVar -# trace_id context 변수 import try: from app.middleware.ServiceLoggerMiddleware import trace_id_context except ImportError: - # 모듈이 아직 로드되지 않은 경우를 위한 기본값 trace_id_context: ContextVar[str] = ContextVar("trace_id", default="") @@ -18,16 +16,14 @@ def setup_file_logging(): # 기존 loguru 핸들러 제거 (기본 콘솔 출력 제거) logger.remove() - # 환경변수로 로그 디렉토리 설정 (기본값: logs/develop) - log_dir = "../../docker/local/logs/develop" + log_dir = "/logs/production" - # 로그 디렉토리가 없으면 생성 + os.makedirs(log_dir, exist_ok=True) - # 로그 파일 경로 설정 - log_file_path = log_dir + "/pre-processing-app.log" - error_log_file_path = log_dir + "/pre-processing-app-error.log" + log_file_path = os.path.join(log_dir, "app.log") + error_log_file_path = os.path.join(log_dir, "error.log") - # trace_id를 포함한 간단한 포맷 문자열 사용 + # trace_id를 포함한 structured 로그 포맷 def add_trace_id_filter(record): try: current_trace_id = trace_id_context.get() @@ -46,10 +42,11 @@ def exclude_logging_middleware_filter(record): return False return add_trace_id_filter(record) - # 파일 로깅 핸들러 추가 - trace_id 포함, LoggingMiddleware 제외 + structured_format = "{time:YYYY-MM-DD HH:mm:ss.SSS} {level: <8} {thread.name: <15} {name}:{function}:{line} [{extra[trace_id]}] {message}" + logger.add( log_file_path, - format="[{extra[trace_id]}] {time:YYYY-MM-DD HH:mm:ss.SSS} | {level} | {name}:{function}:{line} | {message}", + format=structured_format, level="DEBUG", rotation="100 MB", # 100MB마다 로테이션 retention="7 days", # 7일간 보관 @@ -61,10 +58,10 @@ def exclude_logging_middleware_filter(record): filter=exclude_logging_middleware_filter, ) - # 에러 레벨 이상은 별도 파일에도 기록 - trace_id 포함, LoggingMiddleware 제외 + # 에러 레벨 이상은 별도 파일에도 기록 logger.add( error_log_file_path, - format="[{extra[trace_id]}] {time:YYYY-MM-DD HH:mm:ss.SSS} | {level} | {name}:{function}:{line} | {message}", + format=structured_format, level="ERROR", rotation="50 MB", retention="30 days", @@ -82,7 +79,7 @@ def exclude_logging_middleware_filter(record): sys.stdout, format="[{extra[trace_id]}] {time:YYYY-MM-DD HH:mm:ss} | {level: <8} | {name}:{function}:{line} | {message}", level="DEBUG", - colorize=False, # colorize 비활성화하여 태그 충돌 방지 + colorize=False, filter=add_trace_id_filter, ) diff --git a/docker/production/docker-compose.yml b/docker/production/docker-compose.yml index deff1ca4..544b4355 100644 --- a/docker/production/docker-compose.yml +++ b/docker/production/docker-compose.yml @@ -49,26 +49,11 @@ services: env_file: - .env.prod - pre-processing-service: - image: ghcr.io/kernel180-be12/final-4team-icebang/pre-processing-service:latest - container_name: pre-processing-service - restart: on-failure:3 - ports: - - "8000:8000" - networks: - - app-network - env_file: - - .env.prod - volumes: - - onnx_models:/app/models # ONNX 모델 저장용 볼륨 - volumes: caddy_data: caddy_config: logs_volume: driver: local - onnx_models: - driver: local networks: app-network: diff --git a/docker/produdction-fastapi/docker-compose.yml b/docker/produdction-fastapi/docker-compose.yml new file mode 100644 index 00000000..2092e3ca --- /dev/null +++ b/docker/produdction-fastapi/docker-compose.yml @@ -0,0 +1,37 @@ +version: "3.9" + +services: + pre-processing-service: + image: ghcr.io/kernel180-be12/final-4team-icebang/pre-processing-service:latest + container_name: pre-processing-service + restart: on-failure:3 + ports: + - "80:8000" + volumes: + - ./models:/app/models + - logs_volume:/logs + depends_on: + - promtail + env_file: + - .env.prod + + promtail: + image: grafana/promtail:2.9.0 + container_name: promtail + restart: unless-stopped + volumes: + - ./promtail-config.yml:/etc/promtail/config.yml:ro + - logs_volume:/logs + command: + - -config.file=/etc/promtail/config.yml + - -config.expand-env=true + ulimits: + nofile: + soft: 65535 + hard: 65535 + env_file: + - .env.prod + +volumes: + logs_volume: + driver: local diff --git a/docker/produdction-fastapi/promtail-config.yml b/docker/produdction-fastapi/promtail-config.yml new file mode 100644 index 00000000..b44fee00 --- /dev/null +++ b/docker/produdction-fastapi/promtail-config.yml @@ -0,0 +1,47 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: https://${LOKI_USERNAME}:${LOKI_PASSWORD}@${LOKI_HOST}/loki/api/v1/push + +scrape_configs: + - job_name: pre-processing-logs + static_configs: + - targets: + - localhost + labels: + job: pre-processing + app: pre-processing + env: production + __path__: /logs/production/app.log + pipeline_stages: + - regex: + expression: + - labels: + traceId: + level: + thread: + logger: + + - job_name: pre-processing-errors + static_configs: + - targets: + - localhost + labels: + job: pre-processing-errors + app: pre-processing + env: production + log_type: error + __path__: /logs/production/error.log + pipeline_stages: + - regex: + expression: + - labels: + traceId: + level: + thread: + logger: