diff --git a/backend/console-server/src/log/rank/dto/get-elapsed-time-rank-response.dto.ts b/backend/console-server/src/log/rank/dto/get-elapsed-time-rank-response.dto.ts index 48203506..cd40c881 100644 --- a/backend/console-server/src/log/rank/dto/get-elapsed-time-rank-response.dto.ts +++ b/backend/console-server/src/log/rank/dto/get-elapsed-time-rank-response.dto.ts @@ -25,15 +25,15 @@ export class GetElapsedTimeRankResponseDto { example: [ { projectName: 'test059', - elapsedTime: 100, + value: 100, }, { projectName: 'test007', - elapsedTime: 110, + value: 110, }, { projectName: 'test079', - elapsedTime: 120, + value: 120, }, ], }) diff --git a/backend/name-server/src/database/query/dau-recorder.ts b/backend/name-server/src/database/query/dau-recorder.ts index f8291ffa..17003289 100644 --- a/backend/name-server/src/database/query/dau-recorder.ts +++ b/backend/name-server/src/database/query/dau-recorder.ts @@ -8,8 +8,9 @@ export class DAURecorder implements DAURecorderInterface { private clickhouseClient = ClickhouseDatabase.getInstance(); public async recordAccess(domain: string): Promise { - const dateString = new Date().toLocaleDateString(); - const values = [{ domain: domain.toLowerCase(), date: dateString, access: 1 }]; + const values = [ + { domain: domain.toLowerCase(), date: this.formatDate(new Date()), access: 1 }, + ]; try { await this.clickhouseClient.insert({ table: 'dau', @@ -20,4 +21,12 @@ export class DAURecorder implements DAURecorderInterface { console.error('ClickHouse Error:', error); } } + + private formatDate(date: Date): string { + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + + return `${year}-${month}-${day}`; + } } diff --git a/backend/proxy-server/package.json b/backend/proxy-server/package.json index c324ed32..2a700ae3 100644 --- a/backend/proxy-server/package.json +++ b/backend/proxy-server/package.json @@ -7,6 +7,7 @@ "scripts": { "test": "jest --config jest.config.js -ts-config=tsconfig.test.json", "build": "tsc -p tsconfig.json && tsc-alias -p tsconfig.json", + "start": "NODE_ENV=production tsx src/index.ts", "dev": "NODE_ENV=development tsx src/index.ts", "lint": "eslint src/**/*.ts", "lint:fix": "eslint src/**/*.ts --fix" diff --git a/backend/proxy-server/src/app.ts b/backend/proxy-server/src/app.ts index d58056b9..dd196648 100644 --- a/backend/proxy-server/src/app.ts +++ b/backend/proxy-server/src/app.ts @@ -29,7 +29,10 @@ export class Application { const projectCacheRepository = new ProjectCacheRepositoryRedis(); const projectService = new ProjectService(projectRepository, projectCacheRepository); const proxyService = new ProxyService(projectService); - const logRepository = new LogRepositoryClickhouse(); + const logRepository = new LogRepositoryClickhouse({ + maxSize: 1000, + flushIntervalSecond: 5, + }); const logService = new LogService(logRepository); const errorLogRepository = new ErrorLogRepository(); diff --git a/backend/proxy-server/src/database/query/log.repository.clickhouse.ts b/backend/proxy-server/src/database/query/log.repository.clickhouse.ts index 631efb43..7e5c045f 100644 --- a/backend/proxy-server/src/database/query/log.repository.clickhouse.ts +++ b/backend/proxy-server/src/database/query/log.repository.clickhouse.ts @@ -4,35 +4,80 @@ import { LogRepository } from '../../domain/log/log.repository'; import { HttpLogEntity } from '../../domain/log/http-log.entity'; import { ClickHouseClient } from '@clickhouse/client'; import { formatDateTime } from '../../common/utils/date.util'; +import { LogBufferConfig } from 'domain/config/log-buffer.config'; +import { FastifyLogger } from 'common/logger/fastify.logger'; + +type httpLogRecord = { + method: string; + path: string; + host: string; + status_code: number; + elapsed_time: number; + timestamp: string; +}; export class LogRepositoryClickhouse implements LogRepository { private readonly clickhouse: ClickHouseClient; + private readonly config: LogBufferConfig; + private logBuffer: httpLogRecord[] = []; + private flushTimer: NodeJS.Timeout | null = null; + private isProcessing: boolean = false; - constructor() { + constructor(config: LogBufferConfig) { this.clickhouse = ClickhouseDatabase.getInstance(); + this.config = config; + this.startFlushTimer(); } - public async insertHttpLog(log: HttpLogEntity): Promise { - const values = [ - { - method: log.method, - path: log.path || '', - host: log.host, - status_code: log.statusCode, - elapsed_time: Math.round(log.responseTime), - timestamp: formatDateTime(new Date()), - }, - ]; + private startFlushTimer(): void { + if (this.flushTimer) { + clearInterval(this.flushTimer); + } + + this.flushTimer = setInterval(async () => { + if (this.logBuffer.length > 0 && !this.isProcessing) { + await this.flush(); + } + }, this.config.flushIntervalSecond * 1000); + } + + private async flush(): Promise { + if (this.isProcessing || this.logBuffer.length === 0) { + return; + } + let batchToFlush = [...this.logBuffer]; + this.logBuffer = []; try { + this.isProcessing = true; + await this.clickhouse.insert({ table: 'http_log', - values: values, + values: batchToFlush, format: 'JSONEachRow', }); } catch (error) { - console.error('ClickHouse Error:', error); + this.logBuffer = [...this.logBuffer, ...batchToFlush]; throw new DatabaseQueryError(error as Error); + } finally { + this.isProcessing = false; + } + } + + public async insertHttpLog(log: HttpLogEntity): Promise { + const httpLogRecord: httpLogRecord = { + method: log.method, + path: log.path || '', + host: log.host, + status_code: log.statusCode, + elapsed_time: Math.round(log.responseTime), + timestamp: formatDateTime(new Date()), + }; + + this.logBuffer.push(httpLogRecord); + + if (this.logBuffer.length >= this.config.maxSize) { + this.flush(); } } } diff --git a/backend/proxy-server/src/domain/config/log-buffer.config.ts b/backend/proxy-server/src/domain/config/log-buffer.config.ts new file mode 100644 index 00000000..428f11bd --- /dev/null +++ b/backend/proxy-server/src/domain/config/log-buffer.config.ts @@ -0,0 +1,4 @@ +export type LogBufferConfig = { + maxSize: number; + flushIntervalSecond: number; +}; diff --git a/backend/proxy-server/src/domain/proxy/proxy.service.ts b/backend/proxy-server/src/domain/proxy/proxy.service.ts index c59eb58e..438c3f78 100644 --- a/backend/proxy-server/src/domain/proxy/proxy.service.ts +++ b/backend/proxy-server/src/domain/proxy/proxy.service.ts @@ -34,8 +34,6 @@ export class ProxyService { return ip; } catch (error) { - console.log('error: ', error); - if (error instanceof ProxyError) { throw error; } diff --git a/backend/proxy-server/src/server/config/fastify.config.ts b/backend/proxy-server/src/server/config/fastify.config.ts index 6377c8cb..31175083 100644 --- a/backend/proxy-server/src/server/config/fastify.config.ts +++ b/backend/proxy-server/src/server/config/fastify.config.ts @@ -10,7 +10,7 @@ export const fastifyConfig = { process.env.NODE_ENV === 'development' ? true : { - level: process.env.LOG_LEVEL || 'info', + level: process.env.LOG_LEVEL || 'warning', serializers: { req( request: FastifyRequest<