Merge branch 'dev-back' into be-feat#46

backend/console-server/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.8'
 services:
   nginx:
     image: nginx:latest
@@ -38,4 +38,4 @@ services:
       test: ["CMD", "nc", "-z", "localhost", "3002"]
       interval: 10s
       timeout: 2s
-      retries: 5
+      retries: 5

backend/console-server/package.json

@@ -28,6 +28,7 @@
     "@nestjs/core": "^10.0.0",
     "@nestjs/mapped-types": "^2.0.6",
     "@nestjs/platform-express": "^10.0.0",
+    "@nestjs/swagger": "^8.0.5",
     "@nestjs/typeorm": "^10.0.2",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
@@ -37,6 +38,7 @@
     "reflect-metadata": "^0.2.0",
     "rxjs": "^7.8.1",
     "sqlite3": "^5.1.7",
+    "swagger-ui-express": "^5.0.1",
     "typeorm": "^0.3.20"
   },
   "devDependencies": {

backend/console-server/src/app.module.ts

@@ -8,13 +8,16 @@ import { MailModule } from './mail/mail.module';
 import typeOrmConfig from './config/typeorm.config';
 import mailerConfig from './config/mailer.config';
 import { ClickhouseModule } from './clickhouse/clickhouse.module';
+import { LogModule } from './log/log.module';
+import clickhouseConfig from './config/clickhouse.config';
 
 @Module({
     imports: [
-        ConfigModule.forRoot({ isGlobal: true, load: [mailerConfig] }),
+        ConfigModule.forRoot({ isGlobal: true, load: [mailerConfig, clickhouseConfig] }),
         TypeOrmModule.forRootAsync(typeOrmConfig.asProvider()),
         ProjectModule,
         ClickhouseModule,
+        LogModule,
         MailModule,
     ],
     controllers: [AppController],

backend/console-server/src/clickhouse/clickhouse.ts

@@ -1,7 +1,7 @@
 import { createClient } from '@clickhouse/client';
 import { NodeClickHouseClient } from '@clickhouse/client/dist/client';
 import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
-import { clickhouseConfig } from './clickhouse.config';
+import { ConfigService } from '@nestjs/config';
 
 @Injectable()
 export class Clickhouse implements OnModuleInit, OnModuleDestroy {
@@ -10,6 +10,8 @@ export class Clickhouse implements OnModuleInit, OnModuleDestroy {
     private readonly MAX_RETRIES = 3;
     private readonly RETRY_DELAY = 5 * 1000; // 5s
 
+    constructor(private readonly configService: ConfigService) {}
+
     async onModuleDestroy() {
         this.cleanup();
     }
@@ -44,7 +46,8 @@ export class Clickhouse implements OnModuleInit, OnModuleDestroy {
 
     private async createConnection(): Promise<void> {
         try {
-            this.client = createClient(clickhouseConfig);
+            const config = this.configService.get('clickhouse');
+            this.client = createClient(config.clickhouse);
         } catch (error) {
             throw new Error(`Failed to initialize ClickHouse client: ${error.message}`);
         }
@@ -84,11 +87,12 @@ export class Clickhouse implements OnModuleInit, OnModuleDestroy {
         return new Promise((resolve) => setTimeout(resolve, ms));
     }
 
-    async query<T>(query: string): Promise<T[]> {
+    async query<T>(query: string, params?: Record<string, any>): Promise<T[]> {
         try {
             const resultSet = await this.client.query({
                 query,
                 format: 'JSONEachRow',
+                query_params: params,
             });
             return await resultSet.json<T>();
         } catch (error) {

backend/console-server/src/clickhouse/query-builder/time-series.query-builder.ts

@@ -0,0 +1,96 @@
+import { MetricAggregationType, metricExpressions } from '../util/metric-expressions';
+
+interface metric {
+    name: string;
+    aggregation?: MetricAggregationType;
+}
+
+export class TimeSeriesQueryBuilder {
+    private query: string;
+    private params: Record<string, any> = {};
+
+    constructor() {
+        this.query = `SELECT`;
+    }
+
+    /**
+     * @param interval Clickhouse interval format: '1 MINUTE', '5 MINUTE', '1 HOUR'
+     */
+    interval(interval: string): this {
+        this.query += `
+      toStartOf${interval}(timestamp) as timestamp`;
+
+        return this;
+    }
+
+    metrics(metrics: metric[]): this {
+        const metricsQuery = metrics.map((metric) => {
+            if (metric.aggregation) {
+                const expression = metricExpressions[metric.aggregation];
+
+                if (!expression) {
+                    throw new Error(`Unsupported aggregation: ${metric.aggregation}`);
+                }
+                return `${expression(metric.name)}`;
+            }
+            return metric.name;
+        });
+
+        this.query += ` ${metricsQuery.join(', ')}`;
+
+        return this;
+    }
+
+    from(table: string): this {
+        this.query += `
+    FROM ${table}`;
+
+        return this;
+    }
+
+    timeBetween(start: Date, end: Date) {
+        this.query += `
+    WHERE timestamp >= {startTime: DateTime64(3)}
+    AND timestamp < {endTime: DateTime64(3)}`;
+
+        this.params.startTime = start;
+        this.params.endTime = end;
+
+        return this;
+    }
+
+    filter(filters: Record<string, any>): this {
+        if (filters) {
+            Object.entries(filters).forEach(([key, value]) => {
+                this.query += ` AND ${key} = {${key}}`;
+                this.params[key] = value;
+            });
+        }
+
+        return this;
+    }
+
+    groupBy(group: string[]): this {
+        if (group.length > 0) {
+            this.query += ` GROUP BY ${group.join(', ')}`;
+        }
+
+        return this;
+    }
+
+    orderBy(fields: string[], desc: boolean) {
+        this.query += ` ORDER BY ${fields.join(', ')}`;
+
+        if (desc) {
+            this.query += ` DESC`;
+        }
+
+        return this;
+    }
+
+    build() {
+        console.log(this.query);
+
+        return { query: this.query, params: this.params };
+    }
+}

backend/console-server/src/clickhouse/util/date-range.ts

@@ -0,0 +1,9 @@
+export const getDateRange = (days: number = 7) => {
+    const end = new Date();
+    const start = new Date(end.getTime() - days * 24 * 60 * 60 * 1000);
+
+    return {
+        start,
+        end,
+    };
+};

backend/console-server/src/clickhouse/util/metric-expressions.ts

@@ -0,0 +1,13 @@
+type MetricFunction = (metric: string) => string;
+
+export const metricExpressions: Record<string, MetricFunction> = {
+    avg: (metric: string) => `avg(${metric}) as ${metric}`,
+    count: () => `count() as count`,
+    sum: (metric: string) => `sum(${metric}) as ${metric}`,
+    min: (metric: string) => `min(${metric}) as ${metric}`,
+    max: (metric: string) => `max(${metric}) as ${metric}`,
+    p95: (metric: string) => `quantile(0.95)(${metric}) as ${metric}`,
+    p99: (metric: string) => `quantile(0.99)(${metric}) as ${metric}`,
+};
+
+export type MetricAggregationType = keyof typeof metricExpressions;

backend/console-server/src/config/clickhouse.config.ts

@@ -0,0 +1,11 @@
+import { registerAs } from '@nestjs/config';
+
+export default registerAs('clickhouse', () => ({
+    clickhouse: {
+        url: process.env.CLICKHOUSE_URL ?? 'http://localhost:8123',
+        username: process.env.CLICKHOUSE_USER ?? 'default',
+        database: process.env.CLICKHOUSE_DATABASE ?? 'default',
+        password: process.env.CLICKHOUSE_PASSWORD ?? '',
+        request_timeout: 30 * 1000, // 30s
+    },
+}));