Merge pull request #63 from krystxf/refactor/be-logging

refactor(be): request logging

apps/backend/prisma/migrations/20241205005919_request_logs/migration.sql

@@ -0,0 +1,13 @@
+-- CreateTable
+CREATE TABLE "RequestLog" (
+    "id" BIGSERIAL NOT NULL,
+    "method" TEXT NOT NULL,
+    "path" TEXT NOT NULL,
+    "status" INTEGER NOT NULL,
+    "duration" INTEGER NOT NULL,
+    "response" TEXT,
+    "userAgent" TEXT,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "RequestLog_pkey" PRIMARY KEY ("id")
+);

apps/backend/prisma/schema.prisma

@@ -109,6 +109,18 @@ model Log {
   createdAt DateTime @default(now())
 }
 
+model RequestLog {
+  id        BigInt  @id @default(autoincrement())
+  method    String
+  path      String
+  status    Int
+  duration  Int
+  response  String?
+  userAgent String?
+
+  createdAt DateTime @default(now())
+}
+
 enum LogLevel {
   log
   error

apps/backend/src/app.module.ts

@@ -1,17 +1,19 @@
 import { ApolloDriver, type ApolloDriverConfig } from "@nestjs/apollo";
 import { CacheModule } from "@nestjs/cache-manager";
-import { Module } from "@nestjs/common";
+import { MiddlewareConsumer, Module, NestModule } from "@nestjs/common";
 import { ConfigModule } from "@nestjs/config";
 import { GraphQLModule } from "@nestjs/graphql";
 import { ScheduleModule } from "@nestjs/schedule";
 
 import { cacheModuleConfig } from "src/config/cache-module.config";
 import { configModuleConfig } from "src/config/config-module.config";
 import { GRAPHQL_PATH } from "src/constants/api";
+import { RequestLoggerMiddleware } from "src/middleware/request-logger-middleware";
 import { DepartureModule } from "src/modules/departure/departure.module";
 import { GtfsModule } from "src/modules/gtfs/gtfs.module";
 import { ImportModule } from "src/modules/import/import.module";
 import { LoggerModule } from "src/modules/logger/logger.module";
+import { LogsCleanupModule } from "src/modules/logs-cleanup/logs-cleanup.module";
 import { PlatformModule } from "src/modules/platform/platform.module";
 import { PrismaModule } from "src/modules/prisma/prisma.module";
 import { RouteModule } from "src/modules/route/route.module";
@@ -26,6 +28,7 @@ import { StopModule } from "src/modules/stop/stop.module";
         StopModule,
         PrismaModule,
         LoggerModule,
+        LogsCleanupModule,
         StatusModule,
         GtfsModule,
         RouteModule,
@@ -42,4 +45,8 @@ import { StopModule } from "src/modules/stop/stop.module";
     controllers: [],
     providers: [],
 })
-export class AppModule {}
+export class AppModule implements NestModule {
+    configure(consumer: MiddlewareConsumer) {
+        consumer.apply(RequestLoggerMiddleware).forRoutes("*");
+    }
+}

apps/backend/src/enums/log.enum.ts

@@ -10,6 +10,7 @@ export enum LogMessage {
     IMPORT_STOPS = "Import stops",
     REST = "REST",
     GRAPHQL = "GraphQL",
+    REQUEST_LOGS_CLEANUP = "Request logs cleanup",
 }
 
 export enum RestLogStatus {

apps/backend/src/middleware/request-logger-middleware.ts

@@ -0,0 +1,63 @@
+import { Injectable, NestMiddleware } from "@nestjs/common";
+import { NextFunction, Request, Response } from "express";
+
+import { PrismaService } from "src/modules/prisma/prisma.service";
+
+@Injectable()
+export class RequestLoggerMiddleware implements NestMiddleware {
+    constructor(private readonly prisma: PrismaService) {}
+
+    async use(req: Request, res: Response, next: NextFunction) {
+        const start = Date.now();
+
+        // Intercept the response to capture the body
+        const originalSend = res.send;
+        let responseBody: unknown;
+
+        res.send = (body): Response => {
+            responseBody = body; // Capture the response body
+            return originalSend.call(res, body); // Call the original `send` method
+        };
+
+        // Attach an event listener to log after the response is sent
+        res.on("finish", async () => {
+            const duration = Date.now() - start;
+            const {
+                method,
+                url: path,
+                headers: { "user-agent": userAgent = null },
+            } = req;
+            const { statusCode } = res;
+            const responseString =
+                typeof responseBody === "string"
+                    ? responseBody
+                    : JSON.stringify(responseBody);
+
+            const ignoreResponse = ["/v1/stop/all", "/v1/platform/"].some(
+                (item) => path.startsWith(item),
+            );
+
+            if (path.startsWith("/status")) {
+                return;
+            }
+
+            try {
+                // Log the request details to the database
+                await this.prisma.requestLog.create({
+                    data: {
+                        method,
+                        path,
+                        status: statusCode,
+                        duration,
+                        userAgent,
+                        response: ignoreResponse ? null : responseString,
+                    },
+                });
+            } catch (error) {
+                console.error("Failed to log request:", error);
+            }
+        });
+
+        next();
+    }
+}

apps/backend/src/modules/departure/departure.controller.ts

@@ -19,7 +19,6 @@ import {
     departureSchema,
     type DepartureSchema,
 } from "src/modules/departure/schema/departure.schema";
-import { LogInterceptor } from "src/modules/logger/log.interceptor";
 import {
     metroOnlySchema,
     vehicleTypeSchema,
@@ -28,7 +27,7 @@ import { metroOnlyQuery } from "src/swagger/query.swagger";
 
 @ApiTags("departure")
 @Controller("departure")
-@UseInterceptors(CacheInterceptor, LogInterceptor)
+@UseInterceptors(CacheInterceptor)
 @CacheTTL(4 * 1000)
 export class DepartureController {
     constructor(

apps/backend/src/modules/logs-cleanup/logs-cleanup.controller.ts

@@ -0,0 +1,18 @@
+import { Controller, OnModuleInit } from "@nestjs/common";
+import { Cron, CronExpression } from "@nestjs/schedule";
+
+import { LogsCleanupService } from "src/modules/logs-cleanup/logs-cleanup.service";
+
+@Controller("logs-cleanup")
+export class LogsCleanupController implements OnModuleInit {
+    constructor(private readonly logsCleanupService: LogsCleanupService) {}
+
+    async onModuleInit(): Promise<void> {
+        return this.logsCleanupService.cleanupLogs();
+    }
+
+    @Cron(CronExpression.EVERY_10_MINUTES)
+    async cronLogsCleanup(): Promise<void> {
+        return this.logsCleanupService.cleanupLogs();
+    }
+}

apps/backend/src/modules/logs-cleanup/logs-cleanup.module.ts

@@ -0,0 +1,11 @@
+import { Module } from "@nestjs/common";
+
+import { LogsCleanupController } from "src/modules/logs-cleanup/logs-cleanup.controller";
+import { LogsCleanupService } from "src/modules/logs-cleanup/logs-cleanup.service";
+
+@Module({
+    controllers: [LogsCleanupController],
+    providers: [LogsCleanupService],
+    imports: [],
+})
+export class LogsCleanupModule {}

apps/backend/src/modules/logs-cleanup/logs-cleanup.service.ts

@@ -0,0 +1,60 @@
+import { Injectable } from "@nestjs/common";
+
+import { LogLevel, LogMessage } from "src/enums/log.enum";
+import { LoggerService } from "src/modules/logger/logger.service";
+import { PrismaService } from "src/modules/prisma/prisma.service";
+
+const MAX_COUNT = 500_000;
+
+@Injectable()
+export class LogsCleanupService {
+    constructor(
+        private readonly prisma: PrismaService,
+        private readonly logger: LoggerService,
+    ) {}
+
+    async cleanupLogs(): Promise<void> {
+        try {
+            const recordCount = await this.prisma.requestLog.count();
+
+            if (recordCount < MAX_COUNT) {
+                return;
+            }
+
+            const last = await this.prisma.requestLog.findFirst({
+                orderBy: { createdAt: "desc" },
+                skip: MAX_COUNT,
+            });
+
+            if (!last) {
+                return;
+            }
+
+            const { count } = await this.prisma.requestLog.deleteMany({
+                where: {
+                    createdAt: {
+                        lte: last.createdAt,
+                    },
+                },
+            });
+
+            await this.logger.createLog(
+                LogLevel.log,
+                LogMessage.REQUEST_LOGS_CLEANUP,
+                {
+                    message: "Successfully removed old logs",
+                    count,
+                },
+            );
+        } catch (error) {
+            await this.logger.createLog(
+                LogLevel.error,
+                LogMessage.REQUEST_LOGS_CLEANUP,
+                {
+                    message: "Failed to cleanup logs",
+                    error: JSON.stringify(error),
+                },
+            );
+        }
+    }
+}

apps/backend/src/modules/platform/platform.controller.ts

@@ -13,7 +13,6 @@ import { z } from "zod";
 
 import { ApiDescription, ApiQueries } from "src/decorators/swagger.decorator";
 import { EndpointVersion } from "src/enums/endpoint-version";
-import { LogInterceptor } from "src/modules/logger/log.interceptor";
 import { PlatformService } from "src/modules/platform/platform.service";
 import {
     platformWithDistanceSchema,
@@ -34,7 +33,7 @@ import {
 
 @ApiTags("platform")
 @Controller("platform")
-@UseInterceptors(CacheInterceptor, LogInterceptor)
+@UseInterceptors(CacheInterceptor)
 export class PlatformController {
     constructor(private readonly platformService: PlatformService) {}
 

apps/backend/src/modules/stop/stop.controller.ts

@@ -12,13 +12,12 @@ import {
 import { ApiParam, ApiQuery, ApiTags } from "@nestjs/swagger";
 
 import { EndpointVersion } from "src/enums/endpoint-version";
-import { LogInterceptor } from "src/modules/logger/log.interceptor";
 import { StopService } from "src/modules/stop/stop.service";
 import { metroOnlyQuery } from "src/swagger/query.swagger";
 
 @ApiTags("stop")
 @Controller("stop")
-@UseInterceptors(CacheInterceptor, LogInterceptor)
+@UseInterceptors(CacheInterceptor)
 export class StopController {
     constructor(private readonly stopService: StopService) {}
 

apps/web/src/components/Footer/Footer.tsx

@@ -67,7 +67,7 @@ export const Footer = () => {
                         className="dark:p-2 dark:rounded-xl dark:bg-white transition-all ease-in-out dark:border dark:border-neutral-900 rounded dark:shadow-lg"
                         bgColor="transparent"
                         fgColor="black"
-                        level="L"
+                        level="Q"
                         imageSettings={{
                             src: "/appstore-icon.svg",
                             x: undefined,

apps/web/src/components/Navbar/NavbarDownloadLink.tsx

@@ -33,7 +33,7 @@ export const NavbarDownloadLink = () => {
                         value={APPSTORE_URL}
                         fgColor="#000000"
                         bgColor="transparent"
-                        level="L"
+                        level="Q"
                         imageSettings={{
                             src: "/appstore-icon.svg",
                             x: undefined,