feat: add experimental logtail logger and improve general logger with…

… new options

.env.defaults

@@ -21,7 +21,9 @@ SKYHOOK="SKYHOOK"
 TMDB="TMDB"
 TRAKT="TRAKT"
 
-OPTIC_MIN_LEVEL=DEBUG
+MIN_LOG_LEVEL=DEBUG
+OPTIC_TRACING=true
+LOGTAIL_KEY="LOGTAIL_KEY"
 
 UPSTASH_REDIS_REST_URL="UPSTASH_REDIS_REST_URL" 
 UPSTASH_REDIS_REST_TOKEN="UPSTASH_REDIS_REST_TOKEN"

src/common/core/env.ts

@@ -5,7 +5,7 @@ try {
     export: true,
   });
 } catch (_e) {
-  // do nothing
+  throw _e;
 }
 
 export class MissingKeyError extends Error {

src/common/core/factory.ts

@@ -29,12 +29,12 @@ export default (opts: FactoryOptions): Application => {
 
   app.addEventListener('close', (event) => {
     _localSourceFactory.disconnect();
-    logger.info('Request application stop by user', event.type);
+    logger.info('common:core:factory:close: Request application stop by user', event.type);
   });
 
   app.addEventListener('error', (event) => {
     _localSourceFactory.disconnect();
-    logger.critical('Uncaught application exception', event.error);
+    logger.critical('common.core.factory:error: Uncaught application exception', event.error);
   });
 
   app.use(router.routes());

src/common/core/logger.ts

@@ -1,24 +1,48 @@
 import { SummaryMeasureFormatter } from 'x/optic/profiler';
 import { ConsoleStream, Level, Logger } from 'x/optic';
 import { TokenReplacer } from 'x/optic/formatters';
-//import { RegExpFilter } from 'x/optic/regex-filter';
+import { LogtailStream } from '../logger/logtail.ts';
+import { env } from './env.ts';
+import { MinLogLevel } from '../logger/types.d.ts';
 
-//const regex = RegExp(/[&?]+/);
-//const regExpFilter = new RegExpFilter(regex);
-export const logger = new Logger(); //.addFilter(regExpFilter);
+const consoleLogger = new ConsoleStream()
+  .withFormat(
+    new TokenReplacer()
+      .withFormat('{msg} {metadata}')
+      .withColor(),
+  );
+
+const betterStackLogger = new LogtailStream(
+  env<string>('LOGTAIL_KEY')
+);
+
+const logLevel = (level: MinLogLevel): Level => {
+  switch (level) {
+    case 'DEBUG':
+      return Level.Debug;
+    case 'INFO':
+      return Level.Info;
+    case 'WARN':
+      return Level.Warn;
+    case 'ERROR':
+      return Level.Error;
+    default:
+      throw new Error('Unkown log level', { cause: level });
+  }
+};
+
+const logger = new Logger()
+  .withMinLogLevel(
+    logLevel(env<MinLogLevel>('MIN_LOG_LEVEL')),
+  )
+  .addStream(consoleLogger)
+  .addStream(betterStackLogger);
 
 logger.profilingConfig()
-  .enabled(true)
+  .enabled(env<boolean>('OPTIC_TRACING'))
   .captureMemory(true)
   .captureOps(true)
   .withLogLevel(Level.Info)
   .withFormatter(new SummaryMeasureFormatter());
 
-logger.addStream(
-  new ConsoleStream()
-    .withFormat(
-      new TokenReplacer()
-        .withFormat('{msg} {metadata}')
-        .withColor(),
-    ),
-);
+export { logger };

src/common/core/setup.ts

@@ -23,13 +23,13 @@ const applicationState: State = {
     },
     trackingCallback: (experiment, result) => {
       // substitute with segment or something else for exp tracking
-      logger.info('Experiemnt tracked', {
+      logger.debug('Experiemnt tracked', {
         experimentId: experiment.key,
         variationId: result.key,
       });
     },
     onFeatureUsage: (featureKey, result) => {
-      logger.info('Feature used', { key: featureKey, value: result.value });
+      logger.debug('Feature used', { key: featureKey, value: result.value });
     },
   }),
   contextHeader: {

src/common/logger/logtail.ts

@@ -0,0 +1,43 @@
+import { Logtail } from "esm/logtail";
+import { TokenReplacer } from 'x/optic/formatters';
+import { BaseStream, Level, LogRecord } from 'x/optic';
+
+export class LogtailStream extends BaseStream {
+  private logtail: Logtail;
+
+  constructor(clientKey: string) {
+    super(new TokenReplacer());
+    this.logtail = new Logtail(clientKey);
+  }
+
+  log(msg: string): void {
+    this.logtail.log(msg);
+    this.logtail.flush();
+  }
+
+  override handle(logRecord: LogRecord): boolean {
+    const { level, metadata } = logRecord;
+    if (this.minLevel > level) return false;
+    const msg = this.format(logRecord);
+
+    switch (level) { 
+      case Level.Info:
+        this.logtail.info(msg, { ...metadata });
+        break; 
+      case Level.Warn:
+        this.logtail.warn(msg, { ...metadata });
+        break; 
+      case Level.Error:
+        this.logtail.error(msg, { ...metadata });
+        break; 
+      case Level.Critical:
+        this.logtail.log(msg, 'fatal', { ...metadata });
+        break;
+      default:
+        return false;
+    }
+
+    this.logtail.flush();
+    return true;
+  }
+}

src/common/logger/types.d.ts

@@ -0,0 +1 @@
+export type MinLogLevel = 'DEBUG' | 'INFO' | 'WARN' | 'ERROR';

src/common/middleware/growth.ts

@@ -13,15 +13,15 @@ export default async (
   })
     .then((data) => {
       if (data.error) {
-        logger.error('GrowthBook init error', data.error);
+        logger.error('common.middleware.growth: GrowthBook init error', data.error);
       } else {
-        logger.info('GrowthBook init complete', data.source);
+        logger.info('common.middleware.growth: GrowthBook init complete', data.source);
       }
       logger.mark('load-features-end');
       logger.measure(between('load-features-start', 'load-features-end'));
     })
     .catch((e) => {
-      logger.error('Failed to load features from GrowthBook', e);
+      logger.error('common.middleware.growth: Failed to load features from GrowthBook', e);
     })
     .finally(async () => {
       await next();
@@ -33,6 +33,6 @@ export default async (
       logger.measure(between('destory-growth-start', 'destory-growth-end'));
     })
     .catch((e) => {
-      logger.error('Failed to destory GrowthBook', e);
+      logger.error('common.middleware.growth: Failed to destory GrowthBook', e);
     });
 };

src/common/middleware/header.ts

@@ -22,7 +22,6 @@ const enforced: string[] = [
   'host',
   'accept',
   'accept-encoding',
-  'accept-language',
   'user-agent',
 ];
 
@@ -33,7 +32,7 @@ const fail = (header: string, ctx: AppContext) => {
   response.body = <Error> {
     message: 'Missing required header',
   };
-  logger.error(`Required header is missing from request: ${header}`);
+  logger.error(`common.middleware.header:fail: Required header is missing from request: ${header}`);
 };
 
 const pass = async (ctx: AppContext, next: () => Promise<unknown>) => {
@@ -46,7 +45,6 @@ const pass = async (ctx: AppContext, next: () => Promise<unknown>) => {
     agent: headers.get('user-agent')!,
     contentType: headers.get('content-type'),
     acceptEncoding: headers.get('accept-encoding')!,
-    language: headers.get('accept-language')!,
     application: {
       locale: headers.get('x-app-locale'),
       version: headers.get('x-app-version'),
@@ -56,6 +54,7 @@ const pass = async (ctx: AppContext, next: () => Promise<unknown>) => {
       buildType: headers.get('x-app-build-type'),
     },
   };
+  logger.debug('common.middleware.header:pass: ->', state.contextHeader);
   await next();
 };
 

src/common/middleware/timing.ts

@@ -6,6 +6,6 @@ export default async (
 ) => {
   const start = Date.now();
   await next();
-  const ms = Date.now() - start;
-  response.headers.set('x-response-time', `${ms}ms`);
+  const duration = Date.now() - start;
+  response.headers.set('x-response-time', `${duration}ms`);
 };

src/common/mongo/factory.ts

@@ -7,7 +7,7 @@ import { Local } from '../types/core.d.ts';
 class LocalSourceFactory {
   constructor(
     private readonly options: string,
-    private readonly client = new MongoClient(),
+    private readonly client: MongoClient,
   ) {
     logger.mark('mongo_connection_start');
   }
@@ -21,19 +21,26 @@ class LocalSourceFactory {
       );
       return db;
     } catch (e) {
-      logger.error(e);
+      logger.error('common.mongo.factory:connect:', e);
       return undefined;
     }
   };
 
   disconnect = () => {
     logger.mark('mongo_close_start');
-    this.client.close();
+    try {
+      this.client.close();
+    } catch (e) {
+      logger.error('common.mongo.factory:disconnect:', e);
+    }
     logger.mark('mongo_close_end');
     logger.measure(between('mongo_close_start', 'mongo_close_end'));
   };
 }
 
-const _localSourceFactory = new LocalSourceFactory(env<string>('MONGO_URL'));
+const _localSourceFactory = new LocalSourceFactory(
+  env<string>('MONGO_URL'),
+  new MongoClient(),
+);
 
 export default _localSourceFactory;

src/common/types/options.d.ts

@@ -1,7 +1,6 @@
 import { Context, Middleware, Router, State } from 'x/oak';
 
-// deno-lint-ignore no-explicit-any
-type AS = Record<string, any>;
+type AS = Record<string, unknown>;
 
 export interface FactoryOptions<S extends State = AS> {
   router?: Router;

src/common/types/state.d.ts

@@ -6,7 +6,6 @@ export type ContextHeader = {
   authorization: string | null;
   contentType: string | null;
   acceptEncoding: string;
-  language: string;
   application?: {
     locale: string | null;
     version: string | null;

src/config/local/source.ts

@@ -10,7 +10,7 @@ export class LocalSource {
   getConfig = async (): Promise<ConfigDocument | undefined> => {
     const config = await this.collection?.findOne()
       ?.catch((e) => {
-        logger.error(`Unable to find config in collection`, e);
+        logger.error(`config.local.source.LocalSource:getConfig: Unable to find config in collection`, e);
         return undefined;
       });
 

src/import_map.json

@@ -18,6 +18,7 @@
     "x/xml": "https://deno.land/x/xml@4.0.0/mod.ts",
     "x/deepmerge": "https://deno.land/x/deepmergets@v5.1.0/dist/deno/index.ts",
     "esm/ua_parser": "https://esm.sh/ua-parser-js@2.0.0-beta.2",
-    "esm/growthbook": "https://esm.sh/@growthbook/growthbook@1.0.0"
+    "esm/growthbook": "https://esm.sh/@growthbook/growthbook@1.0.0",
+    "esm/logtail": "https://esm.sh/@logtail/node@0.4.21"
   }
 }

src/news/transformer.ts

@@ -13,8 +13,8 @@ export const transform = (
 ): News[] => {
   const items = document.rss.channel.item;
   if (!Array.isArray(items)) {
-    logger.critical('Unexpected output in news.transformers', document);
-    throw Error('Expected array for `document.rss.channel.item`');
+    logger.critical('news.transformer:transform: Unexpected output in news.transformers', document);
+    throw Error('news.transformer:transform: Expected array for `document.rss.channel.item`');
   }
   return items.map((item) => {
     return {

src/series/local/index.ts

@@ -1,4 +1,4 @@
-import { Collection, Document } from 'x/mongo';
+import { Collection, Document, Filter, FindAndModifyOptions } from 'x/mongo';
 import { logger } from '../../common/core/logger.ts';
 import { IResponse } from '../../common/types/response.d.ts';
 import { MediaWithSeason } from '../types.d.ts';
@@ -14,7 +14,10 @@ export default class LocalSource {
       ?.findOne({ 'mediaId.anilist': id })
       ?.then((document) => transform(document))
       ?.catch((e) => {
-        logger.error(`Unable to find '${id}' in collection`, e);
+        logger.error(
+          `seriese.local.index.LocalSource:get: Unable to find '${id}' in collection`,
+          e,
+        );
         return undefined;
       });
 
@@ -24,19 +27,20 @@ export default class LocalSource {
   };
 
   save = async (media: MediaWithSeason) => {
-    await this.collection?.findAndModify(
-      {
-        'mediaId.anilist': media.mediaId.anilist,
-      },
-      {
-        upsert: true,
-        update: media,
-      },
-    ).then((result) => {
-      logger.debug('ObjectId', result);
+    const filter: Filter<Document> = {
+      'mediaId.anilist': media.mediaId.anilist,
+    };
+    const options: FindAndModifyOptions<Document> = {
+      upsert: true,
+      update: media,
+    };
+    await this.collection?.findAndModify(filter, options).then((result) => {
+      logger.debug('seriese.local.index.LocalSource:save: ObjectId', result);
     }).catch((e) => {
-      logger.error('Unable to save media to collection', e);
-      return undefined;
+      logger.error(
+        'seriese.local.index.LocalSource:save: Unable to save media to collection',
+        e,
+      );
     });
   };
 }