Skip to content

Commit

Permalink
Merge pull request #238 from beckn/release
Browse files Browse the repository at this point in the history 
Protocol Server new release - October
  • Loading branch information
@em-abee
em-abee authored Oct 16, 2024
2 parents fca86b9 + 2215481 commit 589a537
Show file tree
Hide file tree
Showing 8 changed files with 3,331 additions and 3,355 deletions.
6,227 changes: 3,085 additions & 3,142 deletions package-lock.json
View file

Large diffs are not rendered by default.

4 changes: 3 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,21 @@
"typescript": "^4.7.3"
},
"dependencies": {
"@apidevtools/json-schema-ref-parser": "^11.7.2",
"@exodus/schemasafe": "^1.3.0",
"amqplib": "^0.10.0",
"axios": "^0.26.1",
"axios-retry": "^4.0.0",
"config": "^3.3.7",
"cors": "^2.8.5",
"dotenv": "^16.4.1",
"express": "^4.18.1",
"express-openapi-validator": "^5.1.6",
"express-status-monitor": "^1.3.4",
"ioredis": "^5.0.6",
"libsodium-wrappers": "^0.7.9",
"mongodb": "^4.7.0",
"node-mocks-http": "^1.15.0",
"openapi-types": "^12.1.3",
"request-ip": "^3.3.0",
"uuid": "^8.3.2",
"winston": "^3.7.2",
Expand Down
4 changes: 2 additions & 2 deletions src/app.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { ClientUtils } from "./utils/client.utils";
import { getConfig } from "./utils/config.utils";
import { GatewayUtils } from "./utils/gateway.utils";
import logger from "./utils/logger.utils";
import { OpenApiValidatorMiddleware } from "./middlewares/schemaValidator.middleware";
import { Validator } from "./middlewares/validator";

const app = Express();

Expand Down Expand Up @@ -134,7 +134,7 @@ const main = async () => {
getConfig().app.gateway.mode.toLocaleUpperCase().substring(0, 1) +
getConfig().app.gateway.mode.toLocaleUpperCase().substring(1)
);
await OpenApiValidatorMiddleware.getInstance().initOpenApiMiddleware();
await Validator.getInstance().initialize();
logger.info('Initialized openapi validator middleware');
} catch (err) {
if (err instanceof Exception) {
Expand Down
227 changes: 18 additions & 209 deletions src/middlewares/schemaValidator.middleware.ts
View file
Original file line number Diff line number Diff line change
@@ -1,198 +1,17 @@
import express, { NextFunction, Request, Response } from "express";
import * as OpenApiValidator from "express-openapi-validator";
import { NextFunction, Request, Response } from "express";
import fs from "fs";
import path from "path";
import YAML from "yaml";
import * as httpMocks from "node-mocks-http";
import { v4 as uuid_v4 } from "uuid";
import { Exception, ExceptionType } from "../models/exception.model";
import { Locals } from "../interfaces/locals.interface";
import { getConfig } from "../utils/config.utils";
import { OpenAPIV3 } from "express-openapi-validator/dist/framework/types";
import logger from "../utils/logger.utils";
import { AppMode } from "../schemas/configs/app.config.schema";
import { GatewayMode } from "../schemas/configs/gateway.app.config.schema";
import {
RequestActions,
ResponseActions
} from "../schemas/configs/actions.app.config.schema";
import { Validator } from "./validator";
import { validationFailHandler } from "../utils/validations.utils";

const specFolder = 'schemas';

export class OpenApiValidatorMiddleware {
private static instance: OpenApiValidatorMiddleware;
private static cachedOpenApiValidator: {
[filename: string]: {
count: number,
requestHandler: express.RequestHandler[],
apiSpec: OpenAPIV3.Document
}
} = {};
private static cachedFileLimit: number;

private constructor() {
OpenApiValidatorMiddleware.cachedFileLimit = getConfig().app.openAPIValidator?.cachedFileLimit || 20;
}

public static getInstance(): OpenApiValidatorMiddleware {
if (!OpenApiValidatorMiddleware.instance) {
OpenApiValidatorMiddleware.instance = new OpenApiValidatorMiddleware();
}
return OpenApiValidatorMiddleware.instance;
}

private getApiSpec(specFile: string): OpenAPIV3.Document {
const apiSpecYAML = fs.readFileSync(specFile, "utf8");
const apiSpec = YAML.parse(apiSpecYAML);
return apiSpec;
};

public async initOpenApiMiddleware(): Promise<void> {
try {
let fileToCache = getConfig().app?.openAPIValidator?.initialFilesToCache;
let fileNames, noOfFileToCache = 0;
const cachedFileLimit: number = OpenApiValidatorMiddleware.cachedFileLimit;
logger.info(`OpenAPIValidator Total Cache capacity ${cachedFileLimit}`);
if (fileToCache) {
fileNames = fileToCache.split(/\s*,\s*/).map(item => item.trim());
logger.info(`OpenAPIValidator Init no of files to cache: ${fileNames?.length}`);
noOfFileToCache = fileNames.length;
} else {
const files = fs.readdirSync(specFolder);
fileNames = files.filter(file => fs.lstatSync(path.join(specFolder, file)).isFile() && (file.endsWith('.yaml') || file.endsWith('.yml')));
noOfFileToCache = Math.min(fileNames.length, 3); //If files to cache is not found in env then we will cache just three file
}
noOfFileToCache = Math.min(noOfFileToCache, cachedFileLimit);
console.log('Cache total files: ', noOfFileToCache);

for (let i = 0; i < noOfFileToCache; i++) {
const file = `${specFolder}/${fileNames[i]}`;
if (!OpenApiValidatorMiddleware.cachedOpenApiValidator[file]) {
logger.info(`Intially cache Not found loadApiSpec file. Loading.... ${file}`);
const apiSpec = this.getApiSpec(file);
const requestHandler = OpenApiValidator.middleware({
apiSpec,
validateRequests: true,
validateResponses: false,
$refParser: {
mode: "dereference"
}
})
OpenApiValidatorMiddleware.cachedOpenApiValidator[file] = {
apiSpec,
count: 0,
requestHandler
}
await initializeOpenApiValidatorCache(requestHandler);
}
}
} catch (err) {
logger.error('Error in initializing open API middleware', err);
}
}

public getOpenApiMiddleware(specFile: string): express.RequestHandler[] {
try {
let requestHandler: express.RequestHandler[];
if (OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile]) {
const cachedValidator = OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile];
cachedValidator.count = cachedValidator.count > 1000 ? cachedValidator.count : cachedValidator.count + 1;
logger.info(`Cache found for spec ${specFile}`);
requestHandler = cachedValidator.requestHandler;
} else {
const cashedSpec = Object.entries(OpenApiValidatorMiddleware.cachedOpenApiValidator);
const cachedFileLimit: number = OpenApiValidatorMiddleware.cachedFileLimit;
if (cashedSpec.length >= cachedFileLimit) {
const specWithLeastCount = cashedSpec.reduce((minEntry, currentEntry) => {
return currentEntry[1].count < minEntry[1].count ? currentEntry : minEntry;
}) || cashedSpec[0];
logger.info(`Cache count reached limit. Deleting from cache.... ${specWithLeastCount[0]}`);
delete OpenApiValidatorMiddleware.cachedOpenApiValidator[specWithLeastCount[0]];
}
logger.info(`Cache Not found loadApiSpec file. Loading.... ${specFile}`);
const apiSpec = this.getApiSpec(specFile);
OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile] = {
apiSpec,
count: 1,
requestHandler: OpenApiValidator.middleware({
apiSpec,
validateRequests: true,
validateResponses: false,
$refParser: {
mode: "dereference"
}
})
}
requestHandler = OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile].requestHandler;
}
const cacheStats = Object.entries(OpenApiValidatorMiddleware.cachedOpenApiValidator).map((cache) => {
return {
count: cache[1].count,
specFile: cache[0]
}
});
console.table(cacheStats);
return requestHandler;
} catch (err) {
logger.error('Error in getOpenApiMiddleware', err);
return []
}
};
}

const initializeOpenApiValidatorCache = async (stack: any) => {
try {
let actions: string[] = [];
if (
(getConfig().app.mode === AppMode.bap &&
getConfig().app.gateway.mode === GatewayMode.client) ||
(getConfig().app.mode === AppMode.bpp &&
getConfig().app.gateway.mode === GatewayMode.network)
) {
actions = Object.keys(RequestActions);
} else {
actions = Object.keys(ResponseActions);
}

actions.forEach((action) => {
const mockRequest = (body: any) => {
const req = httpMocks.createRequest({
method: "POST",
url: `/${action}`,
headers: {
"Content-Type": "application/json",
Authorization: uuid_v4()
},
body: body
});

req.app = {
enabled: (setting: any) => {
if (
setting === "strict routing" ||
setting === "case sensitive routing"
) {
return true;
}
return false;
}
} as any;
return req;
};

const reqObj = mockRequest({
context: { action: `${action}` },
message: {}
});

walkSubstack(stack, reqObj, {}, () => {
return;
}, false);
});
} catch (error: any) { }
};

export const schemaErrorHandler = (
err: any,
req: Request,
Expand Down Expand Up @@ -234,28 +53,6 @@ export const schemaErrorHandler = (
}
};

const walkSubstack = function (
stack: any,
req: any,
res: any,
next: NextFunction,
reportError = true
) {
if (typeof stack === "function") {
stack = [stack];
}
const walkStack = function (i: any, err?: any) {
if (err && reportError) {
return schemaErrorHandler(err, req, res, next);
}
if (i >= stack.length) {
return next();
}
stack[i](req, res, walkStack.bind(null, i + 1));
};
walkStack(0);
};

export const openApiValidatorMiddleware = async (
req: Request,
res: Response<{}, Locals>,
Expand All @@ -265,7 +62,7 @@ export const openApiValidatorMiddleware = async (
? req?.body?.context?.core_version
: req?.body?.context?.version;
let specFile = `${specFolder}/core_${version}.yaml`;

let specFileName = `core_${version}.yaml`;
if (getConfig().app.useLayer2Config) {
let doesLayer2ConfigExist = false;
let layer2ConfigFilename = `${req?.body?.context?.domain}_${version}.yaml`;
Expand All @@ -280,7 +77,10 @@ export const openApiValidatorMiddleware = async (
} catch (error) {
doesLayer2ConfigExist = false;
}
if (doesLayer2ConfigExist) specFile = `${specFolder}/${layer2ConfigFilename}`;
if (doesLayer2ConfigExist) {
specFile = `${specFolder}/${layer2ConfigFilename}`;
specFileName = layer2ConfigFilename;
}
else {
if (getConfig().app.mandateLayer2Config) {
const message = `Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration`
Expand All @@ -295,6 +95,15 @@ export const openApiValidatorMiddleware = async (
}
}
}
const openApiValidator = OpenApiValidatorMiddleware.getInstance().getOpenApiMiddleware(specFile);
walkSubstack([...openApiValidator], req, res, next);
const validatorInstance = Validator.getInstance();
const openApiValidator = await validatorInstance.getValidationMiddleware(specFile, specFileName);
// Call the openApiValidator and handle the response
await openApiValidator(req, res, (err: any) => {
if (err) {
schemaErrorHandler(err, req, res, next);
} else {
logger.info('Validation Success');
next();
}
});
};
Loading

0 comments on commit 589a537

Please sign in to comment.