Spring boot Promtail을 통해 Loki log 전송

.github/workflows/deploy-java.yml

@@ -28,6 +28,9 @@ jobs:
           echo "DB_USER=${{ secrets.DB_USER }}" >> .env.prod
           echo "DB_PASS=${{ secrets.DB_PASS }}" >> .env.prod
           echo "DB_NAME=${{ secrets.DB_NAME }}" >> .env.prod
+          echo "ENV_NAME=${{ secrets.LOKI_URL }}" >> .env.prod
+          echo "ENV_NAME=${{ secrets.LOKI_USERNAME }}" >> .env.prod
+          echo "ENV_NAME=${{ secrets.LOKI_PASSWORD }}" >> .env.prod
           echo "ENV_NAME=${{ secrets.ENV_NAME }}" >> .env.prod
 
       - name: Set repo lowercase
@@ -62,6 +65,17 @@ jobs:
           target: "~/app/docker/production/"
           overwrite: true
 
+      - name: Copy promtail-config to EC2
+        uses: appleboy/scp-action@v0.1.7
+        with:
+          host: ${{ secrets.SERVER_HOST }}
+          username: ubuntu
+          key: ${{ secrets.SERVER_SSH_KEY }}
+          source: "docker/production/promtail-config.yml"
+          target: "~/app/docker/production/"
+          overwrite: true
+
+
       - name: Deploy on EC2
         uses: appleboy/ssh-action@v1.0.3
         with:

apps/user-service/src/main/java/site/icebang/global/handler/exception/GlobalExceptionHandler.java

@@ -9,10 +9,13 @@
 import org.springframework.web.bind.annotation.RestControllerAdvice;
 import org.springframework.web.servlet.resource.NoResourceFoundException;
 
+import lombok.extern.slf4j.Slf4j;
+
 import site.icebang.common.dto.ApiResponse;
 import site.icebang.common.exception.DuplicateDataException;
 
 @RestControllerAdvice
+@Slf4j
 public class GlobalExceptionHandler {
   @ExceptionHandler(MethodArgumentNotValidException.class)
   @ResponseStatus(HttpStatus.BAD_REQUEST)
@@ -24,6 +27,7 @@ public ApiResponse<String> handleValidation(MethodArgumentNotValidException ex)
   @ExceptionHandler(Exception.class)
   @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
   public ApiResponse<String> handleGeneric(Exception ex) {
+    log.error(ex.getMessage(), ex);
     return ApiResponse.error("Internal error: ", HttpStatus.INTERNAL_SERVER_ERROR);
   }
 
@@ -48,6 +52,7 @@ public ApiResponse<String> handleAccessDenied(AccessDeniedException ex) {
   @ExceptionHandler(DuplicateDataException.class)
   @ResponseStatus(HttpStatus.CONFLICT)
   public ApiResponse<String> handleDuplicateData(DuplicateDataException ex) {
+    log.warn(ex.getMessage(), ex);
     return ApiResponse.error("Duplicate: " + ex.getMessage(), HttpStatus.CONFLICT);
   }
 }

apps/user-service/src/main/resources/log4j2-develop.yml

@@ -7,16 +7,16 @@ Configuration:
       - name: "app-name"
         value: "user-service"
       - name: "log-path"
-        value: "./logs"
+        value: "./docker/local/logs"
       - name: "charset-UTF-8"
         value: "UTF-8"
       # DEBUG 환경용 콘솔 패턴 - 더 간단하고 가독성 좋게
       - name: "console-layout-pattern"
         value: "%highlight{[%-5level]} [%X{traceId}] [%X{spanId}] %d{HH:mm:ss} [%t] %n   %logger{20} - %msg%n%n "
-      # 파일용 패턴
+      # 파일용 패턴 - Promtail이 파싱하기 쉽게 구조화 (UTC 시간 사용)
       - name: "file-layout-pattern"
-        value: "[%X{traceId}] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"
-      # 개발 환경용 로그 파일들
+        value: "[%X{traceId}] [%X{spanId}] %d{yyyy-MM-dd HH:mm:ss.SSS}{UTC} [%t] %-5level %logger{36} - %msg%n"
+      # 개발 환경용 로그 파일들 - 절대경로나 상대경로 설정
       - name: "dev-log"
         value: ${log-path}/develop/app.log
       - name: "error-log"
@@ -29,37 +29,8 @@ Configuration:
       target: SYSTEM_OUT
       PatternLayout:
         pattern: ${console-layout-pattern}
-#        disableAnsi: false
-
-    # Loki Appender - 개발환경 모니터링용
-    Loki:
-      name: loki-appender
-      host: localhost
-      port: 3100
-      JsonLayout:
-        compact: true
-        eventEol: true
-        includeStacktrace: true
-        KeyValuePair:
-          - key: "app"
-            value: "${app-name}"
-          - key: "env"
-            value: "develop"
-      Label:
-        - name: "app"
-          value: "${app-name}"
-        - name: "env"
-          value: "develop"
-        - name: "traceId"
-          value: "${ctx:traceId}"
-        - name: "spanId"
-          value: "${ctx:spanId}"
-        - name: "executionType"
-          value: "${ctx:executionType:-application}"
-        - name: "sourceId"
-          value: "${ctx:sourceId}"
-        - name: "runId"
-          value: "${ctx:runId}"
+
+    # JDBC Appender - 워크플로우 로그용
     JDBC:
       name: workflow-appender
       tableName: "execution_log"
@@ -80,20 +51,27 @@ Configuration:
         - name: "log_level"
           pattern: "%level"
         - name: "executed_at"
-          pattern: "%d{yyyy-MM-dd HH:mm:ss.SSS}"  # 패턴으로 시간 직접 지정
+          pattern: "%d{yyyy-MM-dd HH:mm:ss.SSS}"
         - name: "log_message"
           pattern: "%message"
         - name: "trace_id"
           pattern: "%X{traceId}"
         - name: "reserved1"
           pattern: "%X{spanId}"
-#        - name: "config_snapshot"
-#          pattern: "%X{configSnapshot}"
+
+    # 파일 Appenders - Promtail이 이 파일들을 읽음
     File:
       - name: file-dev-appender
         fileName: ${dev-log}
         PatternLayout:
           pattern: ${file-layout-pattern}
+        # 로그 로테이션 설정 (선택사항)
+        # Policies:
+        #   SizeBasedTriggeringPolicy:
+        #     size: 10MB
+        # DefaultRolloverStrategy:
+        #   max: 10
+
       - name: file-error-appender
         fileName: ${error-log}
         PatternLayout:
@@ -102,20 +80,19 @@ Configuration:
           level: ERROR
 
   Loggers:
-    # Root 로거 - 개발환경에서는 기본적으로 INFO 레벨
+    # Root 로거
     Root:
       level: INFO
       AppenderRef:
         - ref: console-appender
 
     Logger:
-      # 애플리케이션 로그 - 개발 시 모든 레벨 + Loki 전송
+      # 애플리케이션 로그 - 파일로만 저장 (Promtail이 읽어감)
       - name: site.icebang
         additivity: "false"
         level: DEBUG
         AppenderRef:
           - ref: console-appender
-          - ref: loki-appender
           - ref: file-dev-appender
           - ref: file-error-appender
 
@@ -124,58 +101,54 @@ Configuration:
         additivity: "false"
         AppenderRef:
           - ref: workflow-appender
-          - ref: loki-appender
           - ref: console-appender
           - ref: file-dev-appender
           - ref: file-error-appender
 
-      # Spring Framework - 개발 시 필요한 정보만
+      # Spring Framework
       - name: org.springframework
         additivity: "false"
         level: INFO
         AppenderRef:
           - ref: console-appender
           - ref: file-dev-appender
 
-      # Spring Security - 인증 디버깅용
+      # Spring Security
       - name: org.springframework.security
         level: DEBUG
         additivity: "false"
         AppenderRef:
           - ref: console-appender
           - ref: file-dev-appender
-          - ref: loki-appender
 
-      # 웹 요청 로그 - API 개발 시 유용
+      # 웹 요청 로그
       - name: org.springframework.web
         level: DEBUG
         additivity: "false"
         AppenderRef:
           - ref: console-appender
           - ref: file-dev-appender
-          - ref: loki-appender
 
-      # 트랜잭션 로그 - DB 작업 디버깅
+      # 트랜잭션 로그
       - name: org.springframework.transaction
         level: DEBUG
         additivity: "false"
         AppenderRef:
           - ref: console-appender
           - ref: file-dev-appender
-          - ref: loki-appender
 
       # HikariCP 로그 비활성화
       - name: com.zaxxer.hikari
         level: "OFF"
 
-      # SQL 로그 - 개발 시 쿼리 확인용 (필요시 활성화)
+      # SQL 로그
       - name: org.hibernate.SQL
         level: DEBUG
         additivity: "false"
         AppenderRef:
           - ref: console-appender
 
-      # 파라미터 바인딩 로그 (필요시 활성화)
+      # 파라미터 바인딩 로그
       - name: org.hibernate.type.descriptor.sql.BasicBinder
         level: TRACE
         additivity: "false"

apps/user-service/src/main/resources/log4j2-production.yml

@@ -12,10 +12,10 @@ Configuration:
         value: "UTF-8"
       # 프로덕션 환경용 콘솔 패턴 - 구조화된 로그
       - name: "console-layout-pattern"
-        value: "%highlight{[%-5level]} [%X{traceId}] [%X{spanId}] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %logger{36} - %msg%n"
-      # 파일용 패턴
+        value: "%highlight{[%-5level]} [%X{traceId}] [%X{spanId}] %d{HH:mm:ss}{UTC} [%t]   %logger{20} - %msg% "
+      # 파일용 패턴 - Promtail이 파싱하기 쉽게 구조화 (UTC 시간 사용)
       - name: "file-layout-pattern"
-        value: "[%X{traceId}] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"
+        value: "[%X{traceId}] [%X{spanId}] %d{yyyy-MM-dd HH:mm:ss.SSS}{UTC} [%t] %-5level %logger{36} - %msg%n"
       # 프로덕션 환경용 로그 파일들
       - name: "prod-log"
         value: ${log-path}/production/app.log
@@ -33,8 +33,9 @@ Configuration:
     # Loki Appender - 프로덕션 모니터링용
     Loki:
       name: loki-appender
-      host: localhost
-      port: 3100
+      url: ${LOKI_URL}                 # Grafana Cloud Loki URL
+      basicAuthUsername: ${LOKI_USERNAME}   # Grafana Cloud 사용자 이름
+      basicAuthPassword: ${LOKI_PASSWORD}   # Grafana Cloud API Key
       JsonLayout:
         compact: true
         eventEol: true

docker/local/docker-compose.yml

@@ -50,6 +50,31 @@ services:
     networks:
       - icebang-network
 
+  promtail:
+    image: grafana/promtail:2.9.0
+    container_name: promtail
+    restart: unless-stopped
+    ports:
+      - "9080:9080"
+    volumes:
+      - ./promtail-config.yml:/etc/promtail/config.yml:ro       # config 파일
+      - ./logs:/logs:cached
+      - promtail_positions:/var/lib/promtail                     # positions 파일용 writable volume
+    command:
+      - -config.file=/etc/promtail/config.yml
+      - -config.expand-env=true
+    ulimits:
+      nofile:
+        soft: 65535
+        hard: 65535
+    depends_on:
+      - loki
+    healthcheck:
+      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:9080/ready || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+
   grafana:
     image: grafana/grafana:10.1.0
     container_name: grafana
@@ -74,6 +99,7 @@ volumes:
   mariadb_data:
   loki_data:
   grafana_data:
+  promtail_positions: {}   # Promtail positions 파일용 named volume
 
 networks:
   icebang-network:

docker/local/promtail-config.yml

@@ -0,0 +1,49 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://localhost:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: user-service-logs
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: user-service
+          app: user-service
+          env: develop
+          __path__: /logs/develop/app.log
+    pipeline_stages:
+      - regex:
+          expression: '^\[(?P<traceId>[^\]]*)\] \[(?P<spanId>[^\]]*)\] (?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \[(?P<thread>[^\]]+)\] (?P<level>\w+)\s+(?P<logger>\S+) - (?P<message>.*)$'
+      - labels:
+          traceId:
+          level:
+          thread:
+          logger:
+          spanId:
+
+  - job_name: user-service-errors
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: user-service-errors
+          app: user-service
+          env: develop
+          log_type: error
+          __path__: /logs/develop/error.log
+    pipeline_stages:
+      - regex:
+          expression: '^\[(?P<traceId>[^\]]*)\] \[(?P<spanId>[^\]]*)\] (?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \[(?P<thread>[^\]]+)\] (?P<level>\w+)\s+(?P<logger>\S+) - (?P<message>.*)$'
+      - labels:
+          traceId:
+          level:
+          thread:
+          logger:
+          spanId: