Merge pull request #401 from vechr/feat/observability

Feat/observability

.env.example

@@ -9,8 +9,16 @@ EMAIL_PASS=Vechr
 EMAIL_HOST=mail-dev
 EMAIL_PORT=1025
 
-DB_URL="postgresql://Vechr:123@host.docker.internal:5433/notification_db?schema=public&connect_timeout=300"
+DB_URL="postgresql://Vechr:123@postgres-db:5432/notification_db?schema=public&connect_timeout=300"
 
 NATS_CA=/certificate/self-signed/rootCA.pem
 NATS_KEY=/certificate/self-signed/nats/nats.key
-NATS_CERT=/certificate/self-signed/nats/nats.crt
+NATS_CERT=/certificate/self-signed/nats/nats.crt
+
+APP_NAME=notification-service
+
+LOKI_HOST=http://loki:3100
+LOKI_USERNAME=
+LOKI_PASSWORD=
+
+OTLP_HTTP_URL=http://tempo:4318/v1/traces

package.json

@@ -37,7 +37,13 @@
     "@nestjs/passport": "^9.0.0",
     "@nestjs/platform-express": "^9.3.10",
     "@nestjs/swagger": "^6.2.1",
+    "@opentelemetry/exporter-prometheus": "^0.41.1",
+    "@opentelemetry/instrumentation-express": "^0.33.0",
+    "@opentelemetry/instrumentation-nestjs-core": "^0.33.0",
+    "@opentelemetry/instrumentation-pino": "^0.34.0",
+    "@opentelemetry/sdk-node": "^0.41.1",
     "@prisma/client": "^4.7.1",
+    "@prisma/instrumentation": "^5.0.0",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.0",
     "cookie-parser": "^1.4.6",
@@ -46,13 +52,15 @@
     "env-var": "^7.3.0",
     "jsonwebtoken": "^8.5.1",
     "nats": "^2.13.1",
+    "nestjs-otel": "^5.1.4",
     "nestjs-pino": "^3.1.1",
     "nodemailer": "^6.8.0",
     "passport": "^0.6.0",
     "passport-jwt": "^4.0.0",
     "passport-local": "^1.0.0",
     "pino": "^8.8.0",
     "pino-http": "^8.3.0",
+    "pino-loki": "^2.1.3",
     "pino-pretty": "^9.1.1",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",

src/app.module.ts

@@ -1,17 +1,45 @@
-import { Module } from '@nestjs/common';
+import { MiddlewareConsumer, Module, RequestMethod } from '@nestjs/common';
 import { LoggerModule } from 'nestjs-pino';
+import { OpenTelemetryModule } from 'nestjs-otel';
 import AuthModule from './core/auth.module';
 import AuditModule from './modules/audits/audit.module';
 import { MailModule } from './modules/mails/mail.module';
 import { NotificationEmailModule } from './modules/notification-email/notification-email.module';
 import { PrismaModule } from './prisma/prisma.module';
 import { logger } from './shared/utils/log.util';
+import { InstrumentMiddleware } from './shared/middlewares/instrument.middleware';
+
+const OpenTelemetryModuleConfig = OpenTelemetryModule.forRoot({
+  metrics: {
+    hostMetrics: true,
+    apiMetrics: {
+      enable: true,
+    },
+  },
+});
+
+const PinoLoggerModule = LoggerModule.forRoot({
+  pinoHttp: {
+    customLogLevel: function (_, res, err) {
+      if (res.statusCode >= 400 && res.statusCode < 500) {
+        return 'error';
+      } else if (res.statusCode >= 500 || err) {
+        return 'fatal';
+      } else if (res.statusCode >= 300 && res.statusCode < 400) {
+        return 'warn';
+      } else if (res.statusCode >= 200 && res.statusCode < 300) {
+        return 'info';
+      }
+      return 'debug';
+    },
+    logger,
+  },
+});
 
 @Module({
   imports: [
-    LoggerModule.forRoot({
-      pinoHttp: { logger },
-    }),
+    OpenTelemetryModuleConfig,
+    PinoLoggerModule,
     //Plugins
     PrismaModule,
     AuthModule,
@@ -21,4 +49,10 @@ import { logger } from './shared/utils/log.util';
     NotificationEmailModule,
   ],
 })
-export class AppModule {}
+export class AppModule {
+  configure(consumer: MiddlewareConsumer): void {
+    consumer
+      .apply(InstrumentMiddleware)
+      .forRoutes({ path: '*', method: RequestMethod.ALL });
+  }
+}

src/constants/app.constant.ts

@@ -22,4 +22,9 @@ export default Object.freeze({
   NATS_CA: env.get('NATS_CA').required().asString(),
   NATS_KEY: env.get('NATS_KEY').required().asString(),
   NATS_CERT: env.get('NATS_CERT').required().asString(),
+  APP_NAME: env.get('APP_NAME').default('notification-service').asString(),
+  LOKI_HOST: env.get('LOKI_HOST').asString(),
+  LOKI_USERNAME: env.get('LOKI_USERNAME').default('').asString(),
+  LOKI_PASSWORD: env.get('LOKI_PASSWORD').default('').asString(),
+  OTLP_HTTP_URL: env.get('OTLP_HTTP_URL').asString(),
 });

src/modules/audits/audit.service.ts

@@ -1,5 +1,6 @@
-import { Inject } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import { ClientProxy } from '@nestjs/microservices';
+import { OtelInstanceCounter, OtelMethodCounter } from 'nestjs-otel';
 import { TAuditCreatedPayload } from './types/audit-created.type';
 import { TAuditUpdatedPayload } from './types/audit-updated.type';
 import { TAuditDeletedPayload } from './types/audit-deleted.type';
@@ -9,9 +10,12 @@ import { publish } from '@/shared/utils/nats.util';
 import { TUserCustomInformation } from '@/shared/types/user.type';
 import appConstant from '@/constants/app.constant';
 
+@Injectable()
+@OtelInstanceCounter()
 export default class AuditService {
   constructor(@Inject('NATS_SERVICE') private readonly client: ClientProxy) {}
 
+  @OtelMethodCounter()
   public async sendAudit(
     ctx: IContext,
     action: AuditAction,

src/modules/mails/mail.controller.ts

@@ -1,15 +1,18 @@
 import { Controller, UseFilters } from '@nestjs/common';
 import { EventPattern, Payload } from '@nestjs/microservices';
+import { OtelInstanceCounter, OtelMethodCounter } from 'nestjs-otel';
 import { NotificationEmailRPC } from './dto/notification-email-rpc.dto';
 import { MailService } from './mail.service';
 import { ExceptionFilter } from '@/shared/filters/rpc-exception.filter';
 
 @Controller()
+@OtelInstanceCounter()
 export class MailController {
   constructor(private readonly mailService: MailService) {}
 
   @UseFilters(new ExceptionFilter())
   @EventPattern('notification.email')
+  @OtelMethodCounter()
   async queryDBTopic(@Payload() dto: NotificationEmailRPC): Promise<any> {
     await this.mailService.sendMailBulk(dto);
   }

src/modules/notification-email/notification-email.controller.ts

@@ -11,6 +11,7 @@ import {
   Version,
 } from '@nestjs/common';
 import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger';
+import { OtelInstanceCounter, OtelMethodCounter } from 'nestjs-otel';
 import { CreateNotificationEmailDto, UpdateNotificationEmailDto } from './dto';
 import { NotificationEmailService } from './notification-email.service';
 import ListNotificationEmailValidator, {
@@ -30,6 +31,7 @@ import Authorization from '@/shared/decorators/authorization.decorator';
 @ApiTags('Notification Email')
 @ApiBearerAuth('access-token')
 @Controller('notification/email')
+@OtelInstanceCounter()
 export class NotificationEmailController {
   constructor(
     private readonly notificationEmailService: NotificationEmailService,
@@ -44,6 +46,7 @@ export class NotificationEmailController {
   @Validator(ListNotificationEmailValidator)
   @Serializer(ListNotificationEmailResponse)
   @ApiFilterQuery('filters', ListNotificationEmailQueryValidator)
+  @OtelMethodCounter()
   public async list(@Context() ctx: IContext): Promise<SuccessResponse> {
     const { result, meta } = await this.notificationEmailService.list(ctx);
     return new SuccessResponse('Success get all records!', result, meta);
@@ -55,6 +58,7 @@ export class NotificationEmailController {
   @Post()
   @Authentication(true)
   @Authorization('email-notifications:create@auth')
+  @OtelMethodCounter()
   async createNotificationEmail(
     @Context() ctx: IContext,
     @Body() dto: CreateNotificationEmailDto,
@@ -75,6 +79,7 @@ export class NotificationEmailController {
   @Patch(':id')
   @Authentication(true)
   @Authorization('email-notifications:update@auth')
+  @OtelMethodCounter()
   async updateNotificationEmailById(
     @Context() ctx: IContext,
     @Param('id') notificationEmailId: string,
@@ -98,6 +103,7 @@ export class NotificationEmailController {
   @Delete(':id')
   @Authentication(true)
   @Authorization('email-notifications:delete@auth')
+  @OtelMethodCounter()
   async deleteNotificationEmailById(
     @Context() ctx: IContext,
     @Param('id') notificationEmailId: string,
@@ -119,6 +125,7 @@ export class NotificationEmailController {
   @Get()
   @Authentication(true)
   @Authorization('email-notifications:read@auth')
+  @OtelMethodCounter()
   async getNotificationEmails(): Promise<SuccessResponse> {
     const result = await this.notificationEmailService.getNotificationEmails();
     return new SuccessResponse('Success Get All Notification Email!', result);
@@ -130,6 +137,7 @@ export class NotificationEmailController {
   @Get(':id')
   @Authentication(true)
   @Authorization('email-notifications:read@auth')
+  @OtelMethodCounter()
   async getNotificationEmailById(
     @Param('id') notificationEmailId: string,
   ): Promise<SuccessResponse> {

src/server.ts

@@ -11,6 +11,7 @@ import HttpExceptionFilter from './shared/filters/http.filter';
 import ContextInterceptor from './shared/interceptors/context.interceptor';
 import appConstant from './constants/app.constant';
 import log from './shared/utils/log.util';
+import otelSDK from './tracing';
 
 const appServer = new Promise(async (resolve, reject) => {
   try {
@@ -109,5 +110,7 @@ const appServer = new Promise(async (resolve, reject) => {
 });
 
 (async function () {
+  if (appConstant.OTLP_HTTP_URL && appConstant.OTLP_HTTP_URL != '')
+    otelSDK.start();
   await Promise.all([appServer]);
 })();

src/shared/middlewares/instrument.middleware.ts

@@ -0,0 +1,11 @@
+import { Injectable, NestMiddleware } from '@nestjs/common';
+import { NextFunction } from 'express';
+import { Span } from 'nestjs-otel';
+
+@Injectable()
+export class InstrumentMiddleware implements NestMiddleware {
+  @Span('Instrument Middleware')
+  use(_req: Request, _res: Response, next: NextFunction) {
+    next();
+  }
+}

src/shared/utils/log.util.ts

@@ -1,5 +1,6 @@
-import pino from 'pino';
-import prettyLogger from 'pino-pretty';
+import pino, { StreamEntry } from 'pino';
+import { LokiOptions } from 'pino-loki/index';
+import appConstant from '@/constants/app.constant';
 
 type LogPayload = string | Record<string, any>;
 
@@ -49,13 +50,35 @@ const logFnErr = (message: LogPayload, error: any, fn: pino.LogFn): string => {
   return finalMessage;
 };
 
-export const logger = pino(
-  prettyLogger({
+const pinoPretty = pino.transport({
+  target: 'pino-pretty',
+  options: {
     colorize: true,
     translateTime: 'SYS:dd/mm/yyyy HH:MM:ss',
     ignore: 'pid,hostname',
-  }),
-);
+  },
+});
+
+const streams: StreamEntry[] = [{ level: 'debug', stream: pinoPretty }];
+
+if (appConstant.LOKI_HOST != '' && appConstant.LOKI_HOST) {
+  const pinoLoki = pino.transport<LokiOptions>({
+    target: 'pino-loki',
+    options: {
+      batching: false,
+      labels: { application: appConstant.APP_NAME },
+      host: appConstant.LOKI_HOST,
+      basicAuth: {
+        password: appConstant.LOKI_PASSWORD,
+        username: appConstant.LOKI_USERNAME,
+      },
+    },
+  });
+
+  streams.push({ level: 'debug', stream: pinoLoki });
+}
+
+export const logger = pino({ level: 'trace' }, pino.multistream(streams));
 
 export const log = {
   info: (message: LogPayload, ...args: LogPayload[]): void => {

src/tracing.ts

@@ -0,0 +1,49 @@
+import {
+  CompositePropagator,
+  W3CTraceContextPropagator,
+  W3CBaggagePropagator,
+} from '@opentelemetry/core';
+import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express';
+import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
+import { JaegerPropagator } from '@opentelemetry/propagator-jaeger';
+import { B3InjectEncoding, B3Propagator } from '@opentelemetry/propagator-b3';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
+import { PrismaInstrumentation } from '@prisma/instrumentation';
+import { PinoInstrumentation } from '@opentelemetry/instrumentation-pino';
+import appConstant from './constants/app.constant';
+
+const otelSDK = new NodeSDK({
+  serviceName: appConstant.APP_NAME,
+  metricReader: new PrometheusExporter({
+    port: 8081,
+  }),
+  spanProcessor: new BatchSpanProcessor(
+    new OTLPTraceExporter({
+      url: appConstant.OTLP_HTTP_URL,
+    }),
+  ),
+  contextManager: new AsyncLocalStorageContextManager(),
+  textMapPropagator: new CompositePropagator({
+    propagators: [
+      new JaegerPropagator(),
+      new W3CTraceContextPropagator(),
+      new W3CBaggagePropagator(),
+      new B3Propagator(),
+      new B3Propagator({
+        injectEncoding: B3InjectEncoding.MULTI_HEADER,
+      }),
+    ],
+  }),
+  instrumentations: [
+    new ExpressInstrumentation(),
+    new NestInstrumentation(),
+    new PrismaInstrumentation(),
+    new PinoInstrumentation(),
+  ],
+});
+
+export default otelSDK;