Merge pull request #211 from boostcampwm-2024/be-feat#210

[BE] 프록시 서버 -> 클릭하우스 http_log 데이터 삽입 버퍼 구현
boostcampwm-2024 · Nov 28, 2024 · b6702c4 · b6702c4
2 parents bc259d7 + c0cff84
commit b6702c4
Show file tree

Hide file tree

Showing 6 changed files with 69 additions and 18 deletions.
diff --git a/backend/proxy-server/package.json b/backend/proxy-server/package.json
@@ -7,6 +7,7 @@
   "scripts": {
     "test": "jest --config jest.config.js -ts-config=tsconfig.test.json",
     "build": "tsc -p tsconfig.json && tsc-alias -p tsconfig.json",
+    "start": "NODE_ENV=production tsx src/index.ts",
     "dev": "NODE_ENV=development tsx src/index.ts",
     "lint": "eslint src/**/*.ts",
     "lint:fix": "eslint src/**/*.ts --fix"

diff --git a/backend/proxy-server/src/app.ts b/backend/proxy-server/src/app.ts
@@ -29,7 +29,10 @@ export class Application {
             const projectCacheRepository = new ProjectCacheRepositoryRedis();
             const projectService = new ProjectService(projectRepository, projectCacheRepository);
             const proxyService = new ProxyService(projectService);
-            const logRepository = new LogRepositoryClickhouse();
+            const logRepository = new LogRepositoryClickhouse({
+                maxSize: 1000,
+                flushIntervalSecond: 5,
+            });
             const logService = new LogService(logRepository);
 
             const errorLogRepository = new ErrorLogRepository();

diff --git a/backend/proxy-server/src/database/query/log.repository.clickhouse.ts b/backend/proxy-server/src/database/query/log.repository.clickhouse.ts
@@ -4,35 +4,80 @@ import { LogRepository } from '../../domain/log/log.repository';
 import { HttpLogEntity } from '../../domain/log/http-log.entity';
 import { ClickHouseClient } from '@clickhouse/client';
 import { formatDateTime } from '../../common/utils/date.util';
+import { LogBufferConfig } from 'domain/config/log-buffer.config';
+import { FastifyLogger } from 'common/logger/fastify.logger';
+
+type httpLogRecord = {
+    method: string;
+    path: string;
+    host: string;
+    status_code: number;
+    elapsed_time: number;
+    timestamp: string;
+};
 
 export class LogRepositoryClickhouse implements LogRepository {
     private readonly clickhouse: ClickHouseClient;
+    private readonly config: LogBufferConfig;
+    private logBuffer: httpLogRecord[] = [];
+    private flushTimer: NodeJS.Timeout | null = null;
+    private isProcessing: boolean = false;
 
-    constructor() {
+    constructor(config: LogBufferConfig) {
         this.clickhouse = ClickhouseDatabase.getInstance();
+        this.config = config;
+        this.startFlushTimer();
     }
 
-    public async insertHttpLog(log: HttpLogEntity): Promise<void> {
-        const values = [
-            {
-                method: log.method,
-                path: log.path || '',
-                host: log.host,
-                status_code: log.statusCode,
-                elapsed_time: Math.round(log.responseTime),
-                timestamp: formatDateTime(new Date()),
-            },
-        ];
+    private startFlushTimer(): void {
+        if (this.flushTimer) {
+            clearInterval(this.flushTimer);
+        }
+
+        this.flushTimer = setInterval(async () => {
+            if (this.logBuffer.length > 0 && !this.isProcessing) {
+                await this.flush();
+            }
+        }, this.config.flushIntervalSecond * 1000);
+    }
+
+    private async flush(): Promise<void> {
+        if (this.isProcessing || this.logBuffer.length === 0) {
+            return;
+        }
+        let batchToFlush = [...this.logBuffer];
 
+        this.logBuffer = [];
         try {
+            this.isProcessing = true;
+
             await this.clickhouse.insert({
                 table: 'http_log',
-                values: values,
+                values: batchToFlush,
                 format: 'JSONEachRow',
             });
         } catch (error) {
-            console.error('ClickHouse Error:', error);
+            this.logBuffer = [...this.logBuffer, ...batchToFlush];
             throw new DatabaseQueryError(error as Error);
+        } finally {
+            this.isProcessing = false;
+        }
+    }
+
+    public async insertHttpLog(log: HttpLogEntity): Promise<void> {
+        const httpLogRecord: httpLogRecord = {
+            method: log.method,
+            path: log.path || '',
+            host: log.host,
+            status_code: log.statusCode,
+            elapsed_time: Math.round(log.responseTime),
+            timestamp: formatDateTime(new Date()),
+        };
+
+        this.logBuffer.push(httpLogRecord);
+
+        if (this.logBuffer.length >= this.config.maxSize) {
+            this.flush();
         }
     }
 }
diff --git a/backend/proxy-server/src/domain/config/log-buffer.config.ts b/backend/proxy-server/src/domain/config/log-buffer.config.ts
@@ -0,0 +1,4 @@
+export type LogBufferConfig = {
+    maxSize: number;
+    flushIntervalSecond: number;
+};
diff --git a/backend/proxy-server/src/domain/proxy/proxy.service.ts b/backend/proxy-server/src/domain/proxy/proxy.service.ts
@@ -34,8 +34,6 @@ export class ProxyService {
 
             return ip;
         } catch (error) {
-            console.log('error: ', error);
-
             if (error instanceof ProxyError) {
                 throw error;
             }

diff --git a/backend/proxy-server/src/server/config/fastify.config.ts b/backend/proxy-server/src/server/config/fastify.config.ts
@@ -10,7 +10,7 @@ export const fastifyConfig = {
         process.env.NODE_ENV === 'development'
             ? true
             : {
-                  level: process.env.LOG_LEVEL || 'info',
+                  level: process.env.LOG_LEVEL || 'warning',
                   serializers: {
                       req(
                           request: FastifyRequest<