feat: mongodb driver migration and graceful process shutdown (#124)

* chore(deps): add esm.sh import for mongodb

* refactor: update  deps for assertions and handle process singals

- `testing/asserts.ts` has been deprecated and `assert/mod.ts` has been used as a replacement
- Adds `SIGTERM` and `SIGINT` process handlers to close mongodb connection and gracefully exit the app process

* feat: update mongo drive and remove ua_parser in favour of deno std

* chore: auto format

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

---------

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Dockerfile

@@ -10,10 +10,15 @@ RUN apt-get install unzip
 FROM scaffold AS cache
 RUN deno cache src/index.ts
 
-# Complilation broken in the latest version of deno
-# FROM cache AS final
-# RUN deno compile --allow-net --allow-env --allow-read --output=server src/index.ts 
+# Fallback and compilation has broken in some instances
+#ENTRYPOINT ["deno", "run", "--allow-net", "--allow-env", "--allow-read", "--allow-sys", "src/index.ts"]
 
-#ENTRYPOINT ["/usr/app/server"]
+FROM cache AS build
+RUN deno check src/index.ts
+RUN deno compile --allow-net --allow-env --allow-read --allow-sys --output=/usr/on-the-edge src/index.ts 
 
-ENTRYPOINT ["deno", "run", "--allow-net", "--allow-env", "--allow-read", "src/index.ts"]
+FROM build AS final
+RUN rm -r /usr/app
+WORKDIR /usr
+
+ENTRYPOINT ["/usr/on-the-edge"]

deno.json

@@ -2,29 +2,23 @@
   "$schema": "http://json-schema.org/draft-07/schema",
   "importMap": "./src/import_map.json",
   "lint": {
-    "files": {
-      "include": ["src"],
-      "exclude": [".github", "README.md"]
-    },
+    "include": ["src"],
+    "exclude": [".github", "README.md"],
     "rules": {
       "tags": ["recommended"]
     }
   },
   "fmt": {
-    "files": {
-      "include": ["src"],
-      "exclude": [".github", "README.md"]
-    },
-    "options": {
-      "useTabs": false,
-      "lineWidth": 80,
-      "indentWidth": 2,
-      "singleQuote": true,
-      "proseWrap": "preserve"
-    }
+    "include": ["src"],
+    "exclude": [".github", "README.md"],
+    "useTabs": false,
+    "lineWidth": 80,
+    "indentWidth": 2,
+    "singleQuote": true,
+    "proseWrap": "preserve"
   },
   "tasks": {
     "test": "deno test --allow-read --allow-env --allow-net",
-    "server": "deno run --allow-read --allow-env --allow-net ./src/index.ts"
+    "server": "deno run --allow-read --allow-env --allow-net --allow-sys ./src/index.ts"
   }
 }

src/common/core/factory.ts

@@ -8,40 +8,30 @@ import header from '../middleware/header.ts';
 import targeting from '../middleware/targeting.ts';
 import { logger } from './logger.ts';
 import { between } from 'x/optic';
-import _localSourceFactory from '../mongo/factory.ts';
 
 const app = new Application({
   state,
   contextState: 'prototype',
 });
 
-export default (opts: FactoryOptions): Application => {
-  logger.mark('factory-start');
-  const router = opts.router ?? new Router();
+app.use(
+  timing,
+  header,
+  growth,
+  targeting,
+  error,
+);
 
-  app.use(
-    timing,
-    header,
-    growth,
-    targeting,
-    error,
+app.addEventListener('error', (event) => {
+  logger.critical(
+    'common.core.factory:error: Uncaught application exception',
+    event.error,
   );
+});
 
-  app.addEventListener('close', (event) => {
-    _localSourceFactory.disconnect();
-    logger.info(
-      'common:core:factory:close: Request application stop by user',
-      event.type,
-    );
-  });
-
-  app.addEventListener('error', (event) => {
-    _localSourceFactory.disconnect();
-    logger.critical(
-      'common.core.factory:error: Uncaught application exception',
-      event.error,
-    );
-  });
+export default (opts: FactoryOptions): Application => {
+  logger.mark('factory-start');
+  const router = opts.router ?? new Router();
 
   app.use(router.routes());
   app.use(router.allowedMethods());

src/common/core/request.ts

@@ -1,17 +1,17 @@
 import { between } from 'x/optic';
 import { logger } from './logger.ts';
 
-const sanitize = (url: string): string => {
-  const urlObject = new URL(url);
+const sanitize = (uri: string): { safeUrl: string; host: string } => {
+  const url = new URL(uri);
 
-  const queryParams = urlObject.searchParams;
+  const queryParams = url.searchParams;
   for (const key of queryParams.keys()) {
     if (key.includes('api_key') || key.includes('api_secret')) {
       queryParams.set(key, '********');
     }
   }
 
-  return urlObject.toString();
+  return { safeUrl: url.toString(), host: url.host };
 };
 
 export const defaults: RequestInit = {
@@ -20,23 +20,30 @@ export const defaults: RequestInit = {
   headers: {
     'accept': 'application/json, application/xml, text/plain, */*',
     'accept-encoding': 'gzip, deflate, br',
+    'connection': 'keep-alive',
+    'user-agent': `Deno/${Deno.version.deno}`,
   },
 };
 
 export const request = async <T>(
   url: string,
   options: RequestInit = defaults,
 ): Promise<T> => {
-  const sanitizedUrl = sanitize(url);
-  logger.debug(`----> ${options.method}: ${sanitizedUrl}`);
   logger.mark('request-start');
-  return await fetch(url, options).then((response) => {
-    logger.debug(`<---- ${response.status} ${options.method}: ${sanitizedUrl}`);
+  const { safeUrl, host } = sanitize(url);
+  logger.debug(`----> HTTP ${options.method}: ${safeUrl}`);
+  return await fetch(url, {
+    headers: {
+      ...options.headers,
+      host: host,
+    },
+  }).then((response) => {
+    logger.debug(`<---- HTTP/${response.status} ${options.method}: ${safeUrl}`);
     logger.mark('request-end');
-    logger.measure(between('request-start', 'request-end'), sanitizedUrl);
+    logger.measure(between('request-start', 'request-end'), host);
     if (!response.ok) {
       throw new Error(
-        `<---- HTTP/${response.status} ${options.method}: ${sanitizedUrl}`,
+        `<---- HTTP/${response.status} ${options.method}: ${safeUrl}`,
       );
     }
     const contentType = response.headers.get('Content-Type');

src/common/core/setup.ts

@@ -38,11 +38,34 @@ const applicationState: State = {
     authorization: null,
     contentType: null,
     acceptEncoding: '',
-    language: '',
   },
   local: await _localSourceFactory.connect(),
 };
 
+const onDispose = (token: number) => {
+  setTimeout(() => {
+    Deno.removeSignalListener('SIGINT', onTerminationRequest);
+    Deno.removeSignalListener('SIGTERM', onTerminationRequest);
+    clearTimeout(token);
+    Deno.exit();
+  }, 500);
+};
+
+const onTerminationRequest = (): void => {
+  logger.debug(
+    'common.core.setup:onTerminationRequest: OS dispatched signal',
+  );
+  const token = setTimeout(async () => await _localSourceFactory.disconnect());
+  logger.debug(
+    'common.core.setup:onTerminationRequest: Attempting to exit Deno process',
+  );
+
+  onDispose(token);
+};
+
+Deno.addSignalListener('SIGINT', onTerminationRequest);
+Deno.addSignalListener('SIGTERM', onTerminationRequest);
+
 logger.mark('setup-end');
 logger.measure(between('setup-start', 'setup-end'));
 

src/common/core/utils.ts

@@ -35,4 +35,7 @@ export const pagination = (page: number, count: number) => {
   return { from, to };
 };
 
+export const isNullOrUndefined = (obj: unknown) =>
+  obj == null || obj == undefined;
+
 export const port = env<number>('PORT');

src/common/middleware/targeting.ts

@@ -1,6 +1,6 @@
 import { between } from 'x/optic';
 import { State } from '../types/state.d.ts';
-import { UAParser } from 'esm/ua_parser';
+import { UserAgent } from 'std/http';
 import { logger } from '../core/logger.ts';
 import type { AppContext } from '../types/core.d.ts';
 
@@ -22,8 +22,7 @@ const contextAttributes = (
     };
   }
 
-  const parser = new UAParser(agent);
-  const { browser, cpu, device, engine, os } = parser.getResult();
+  const { browser, cpu, device, engine, os } = new UserAgent(agent);
 
   return Promise.resolve({
     'browser_name': browser.name,

src/common/mongo/collection.ts

@@ -1,7 +1,7 @@
-import { Document } from 'x/mongo';
+import { Document } from 'npm/mongodb';
 import { Local } from '../types/core.d.ts';
 
-export const getCollection = <T extends Document>(
-  collection: string,
+export const collection = <T extends Document>(
+  name: string,
   database: Local,
-) => database?.collection<T>(collection);
+) => database?.collection<T>(name);

src/common/mongo/factory.ts

@@ -1,46 +1,52 @@
-import { MongoClient } from 'x/mongo';
+import { MongoClient } from 'npm/mongodb';
 import { logger } from '../core/logger.ts';
 import { between } from 'x/optic';
 import { env } from '../core/env.ts';
 import { Local } from '../types/core.d.ts';
 
 class LocalSourceFactory {
-  constructor(
-    private readonly options: string,
-    private readonly client: MongoClient,
-  ) {
+  constructor(private readonly client: MongoClient) {
     logger.mark('mongo_connection_start');
+    client.on('timeout', () => {
+      logger.warn(
+        'common.mongo.factory:LocalSourceFactory: Connection timed out',
+      );
+    });
   }
 
   connect = async (): Promise<Local> => {
-    try {
-      const db = await this.client.connect(this.options);
-      logger.mark('mongo_connection_end');
-      logger.measure(
-        between('mongo_connection_start', 'mongo_connection_end'),
-      );
-      return db;
-    } catch (e) {
-      logger.error('common.mongo.factory:connect:', e);
-      return undefined;
-    }
+    return await this.client.connect()
+      .then((client) => {
+        logger.mark('mongo_connection_end');
+        logger.measure(
+          between('mongo_connection_start', 'mongo_connection_end'),
+        );
+        return client.db();
+      })
+      .catch((e) => {
+        logger.error('common.mongo.factory:connect:', e);
+        return undefined;
+      });
   };
 
-  disconnect = () => {
+  disconnect = async () => {
     logger.mark('mongo_close_start');
-    try {
-      this.client.close();
-    } catch (e) {
-      logger.error('common.mongo.factory:disconnect:', e);
-    }
-    logger.mark('mongo_close_end');
-    logger.measure(between('mongo_close_start', 'mongo_close_end'));
+    await this.client.close(true)
+      .then(() => {
+      }).catch((e) => {
+        logger.error('common.mongo.factory:disconnect:', e);
+      }).finally(() => {
+        logger.mark('mongo_close_end');
+        logger.measure(between('mongo_close_start', 'mongo_close_end'));
+      });
   };
 }
 
 const _localSourceFactory = new LocalSourceFactory(
-  env<string>('MONGO_URL'),
-  new MongoClient(),
+  new MongoClient(env<string>('MONGO_URL'), {
+    connectTimeoutMS: 1000,
+    monitorCommands: true,
+  }),
 );
 
 export default _localSourceFactory;

src/common/mongo/index.ts

@@ -1 +1,3 @@
 export * from './collection.ts';
+export * from './utils.ts';
+export * from './types.d.ts';

src/common/mongo/types.d.ts

@@ -0,0 +1,15 @@
+import { Document, OptionalId, SortDirection } from 'npm/mongodb';
+
+export type Optional<T extends Document> = T | undefined | null;
+
+export type ProjectionOption<T extends Document> = {
+  [K in keyof OptionalId<T>]?: 0 | 1;
+};
+
+export type SortOption<T extends Document> = {
+  [K in keyof OptionalId<T>]?: SortDirection;
+};
+
+export interface EntityCursor {
+  cursor: string;
+}

src/common/mongo/utils.ts

@@ -0,0 +1,11 @@
+import { Document, ObjectId, Sort } from 'npm/mongodb';
+import { ProjectionOption, SortOption } from './types.d.ts';
+
+export const projectionOf = <T extends Document>(
+  projection: ProjectionOption<T>,
+) => projection;
+
+export const sortOf = <T extends Document>(option: SortOption<T>): Sort =>
+  option as Sort;
+
+export const idOf = (id: ObjectId): string => id.toHexString();

src/common/types/core.d.ts

@@ -1,7 +1,7 @@
 import { Context } from 'x/oak';
 import { State } from './state.d.ts';
 import { GrowthBook } from 'esm/growthbook';
-import { Database } from 'x/mongo';
+import { Db } from 'npm/mongodb';
 import { AppFeatures } from '../experiment/types.d.ts';
 
 export type RCF822Date = string;
@@ -14,4 +14,4 @@ export type AppContext = Context<State>;
 
 export type Features = GrowthBook<AppFeatures>;
 
-export type Local = Database | undefined;
+export type Local = Db | undefined;

src/common/types/paging.d.ts

@@ -1,7 +1,7 @@
 import { IResponse } from './response.d.ts';
 
 export interface IPaging<T> extends IResponse<T[]> {
-  page: number;
+  first?: string;
+  last?: string;
   count: number;
-  total: number;
 }

src/config/index.ts

@@ -1,10 +1,10 @@
 import { AppContext } from '../common/types/core.d.ts';
 import { Repository } from './repository/index.ts';
 import { LocalSource } from './local/index.ts';
-import { getCollection } from '../common/mongo/index.ts';
+import { collection } from '../common/mongo/index.ts';
 
 export const config = async ({ state, response }: AppContext) => {
-  const localSource = new LocalSource(getCollection('config', state.local));
+  const localSource = new LocalSource(collection('config', state.local));
   const repository = new Repository(
     state.features,
     localSource,