From a3679a6433f9adc11b6734453cec9c60c6edf3a4 Mon Sep 17 00:00:00 2001 From: mohit-s96 Date: Wed, 20 Sep 2023 01:11:50 +0530 Subject: [PATCH] #40: misc improvements bugfix, minor refactor, combine validate and generate, add support for @odata field in payload --- .gitignore | 3 +- index.js | 29 ++--- lib/schema/index.js | 238 +++++++++++++++++++++++++++-------------- lib/schema/utils.js | 47 ++++++++ lib/schema/validate.js | 52 +++++++-- 5 files changed, 262 insertions(+), 107 deletions(-) create mode 100644 lib/schema/utils.js diff --git a/.gitignore b/.gitignore index 73dcfab..b622ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -104,6 +104,5 @@ dist .tern-port config.json -output -errors +reso-schema-validation-temp .DS_Store diff --git a/index.js b/index.js index ecfed8b..0720b57 100755 --- a/index.js +++ b/index.js @@ -1,6 +1,6 @@ #! /usr/bin/env node -const { generate, validate } = require('./lib/schema'); +const { schema } = require('./lib/schema'); const { restore } = require('./lib/restore-utils'); const { runTests } = require('./lib/batch-test-runner'); const { findVariations, computeVariations } = require('./lib/find-variations/index.js'); @@ -11,24 +11,17 @@ if (require?.main === module) { program.name('reso-certification-utils').description('Command line batch-testing and restore utils').version('0.0.3'); program - .command('generate') + .command('schema') + .option('-g, --generate', 'Generate a schema for payload validation') + .option('-v, --validate', 'Validate one or multiple payloads with a schema') .option('-m, --metadataPath ', 'Path to the metadata report JSON file') - .option('-o, --outputPath ', 'Path tho the directory to store the generated schema') - .option('-a, --additionalProperties', 'Pass this flag to allow additional properties in the schema') - .description('Generate schema from a given metadata report') - .action(generate); - - program - .command('validate') - .option('-m, --metadataPath ', 'Path to the metadata report JSON file') - .option('-p, --payloadPath ', 'Path to the payload that needs to be validated') - .option('-s, --schemaPath ', 'Path to the generated JSON schema') - .option('-e, --errorPath ', 'Path to save error reports in case of failed validation. Defaults to "./errors"') - .option('-a, --additionalProperties', 'Pass this flag to allow additional properties in the schema') - .option('-z, --zipFilePath ', 'Path to a zip file containing JSON payloads') - .option('-dv, --version ', 'The data dictionary version of the metadata report. Defaults to 1.7') - .description('Validate a payload against a schema') - .action(validate); + .option('-o, --outputPath ', 'Path tho the directory to store the generated schema. Defaults to "./"') + .option('-a, --additionalProperties', 'Pass this flag to allow additional properties in the schema. False by default') + .option('-dv, --ddVersion ', 'The DD version of the metadata report') + .option('-p, --payloadPath ', 'Path to the payload file OR directory/zip containing files that need to be validated') + .option('-r, --resourceName ', 'Resource name to validate against. Required if --version is passed when validating.') + .description('Generate a schema or validate a payload against a schema') + .action(schema); program .command('restore') diff --git a/lib/schema/index.js b/lib/schema/index.js index 03c5ca6..c0d6ba0 100644 --- a/lib/schema/index.js +++ b/lib/schema/index.js @@ -4,12 +4,14 @@ const chalk = require('chalk'); const { promises: fs } = require('fs'); const { generateSchema } = require('./generate'); const path = require('path'); -const { validatePayload } = require('./validate'); -const { extractFilesFromZip } = require('../../common'); +const { validatePayload, isValidDdVersion } = require('./validate'); +const { CURRENT_DATA_DICTIONARY_VERSION } = require('../../common'); const { readDirectory } = require('../restore-utils'); const { getReferenceMetadata } = require('reso-certification-etl'); +const { processFiles } = require('./utils'); -const OUTPUT_DIR = 'output'; +const OUTPUT_DIR = 'reso-schema-validation-temp'; +const ERROR_REPORT = 'schema-validation-report.json'; const readFile = async filePath => { try { @@ -43,98 +45,172 @@ const writeFile = async (path, data) => { } }; -const generate = async ({ metadataPath = '', outputPath = '', additionalProperties = false }) => { +const schema = async ({ + metadataPath = '', + outputPath = '.', + additionalProperties = false, + generate, + validate, + ddVersion, + payloadPath, + resourceName +}) => { try { - const metadataReportJson = JSON.parse((await readFile(metadataPath)) || null); - if (!metadataReportJson) { - console.log(chalk.redBright.bold('Invalid metadata file')); + if ((!generate && !validate) || (generate && validate)) { + console.log(chalk.redBright('Only one of --generate (-g) or --validate (-v) should be passed')); return; } - const schema = generateSchema(metadataReportJson, additionalProperties); - if (!schema) { - console.log(chalk.redBright.bold('Error generating JSON schema from the given metadata report')); + + if (metadataPath && ddVersion) { + console.log(chalk.redBright('Only one of --metadataPath (-m) or --ddVersion (-dv) should be present')); return; } - // write schema to the output path - const fileName = 'schema-' + (metadataPath?.split('/')?.at(-1) || ''); - const success = await writeFile(path.join(outputPath, fileName), JSON.stringify(schema)); - if (!success) { - console.log(chalk.redBright.bold('Error writing generated schema to the given location')); + + const version = ddVersion ?? CURRENT_DATA_DICTIONARY_VERSION; + + if (!isValidDdVersion(version)) { + console.log(chalk.redBright(`Invalid DD Version ${version}`)); return; } - console.log(chalk.greenBright.bold(`Schema successfully generated and saved in ${outputPath}/${fileName}`)); + + if (generate) { + let metadataReport = null; + if (!metadataPath) { + metadataReport = getReferenceMetadata(version); + if (!metadataReport) { + console.log(chalk.redBright(`Invalid version ${version}`)); + return; + } + } + const result = await generateSchemaFromMetadata({ + metadataPath, + metadataReport, + additionalProperties + }); + if (result?.schema) { + const { schema } = result; + + // write schema to the output path + const fileName = 'schema-' + (metadataPath?.split('/')?.at(-1) || 'metadata.json'); + const success = await writeFile(path.join(outputPath, fileName), JSON.stringify(schema)); + if (!success) { + console.log(chalk.redBright.bold('Error writing generated schema to the given location')); + return; + } + console.log(chalk.greenBright.bold(`Schema successfully generated and saved in ${outputPath}/${fileName}`)); + } + + return; + } + + if (validate) { + if (!payloadPath) { + console.log(chalk.redBright('Invalid path to payloads')); + return; + } + + if ((ddVersion && !resourceName) || (resourceName && !ddVersion)) { + console.log(chalk.redBright('Resource name (-r, --resourceName) and version (-dv, ddVersion) should be passed together')); + return; + } + + try { + await fs.rm(OUTPUT_DIR, { recursive: true, force: true }); + } catch { + /**ignore */ + } + + const outputPathExists = await createDirectoryIfNotPresent(OUTPUT_DIR); + if (!outputPathExists) { + throw new Error('Unable to create output directory for extracted files'); + } + + const { error } = (await processFiles({ inputPath: payloadPath, outputPath: OUTPUT_DIR })) || {}; + if (error) { + console.log(chalk.redBright('Invalid payload path')); + return; + } + + await validatePayloads({ + metadataPath, + payloadPath: OUTPUT_DIR, + additionalProperties, + version: ddVersion, + resourceName + }); + } } catch (error) { console.log(error); - console.log(chalk.redBright.bold('Something went wrong while generating the schema')); + console.log(chalk.redBright('SOmething wen wrong while processing')); } }; -const validate = async ({ - metadataPath = '', - payloadPath = '', - schemaPath = '', - errorPath = 'errors', +const generateSchemaFromMetadata = async ({ + metadataPath = '', // This can be ignored when calling in a lib. `metadataReport` can be passed instead. additionalProperties = false, - zipFilePath = '', - version = '1.7' + metadataReport }) => { - if (zipFilePath) { - try { - await fs.rm(OUTPUT_DIR, { recursive: true, force: true }); - } catch { - /**ignore */ + try { + const metadataReportJson = metadataPath ? JSON.parse((await readFile(metadataPath)) || null) : metadataReport; + if (!metadataReportJson) { + console.log(chalk.redBright.bold('Invalid metadata file')); + return; } - const outputPathExists = await createDirectoryIfNotPresent(OUTPUT_DIR); - if (!outputPathExists) { - throw new Error('Unable to create output directory for extracted files'); + const schema = generateSchema(metadataReportJson, additionalProperties); + if (!schema) { + console.log(chalk.redBright.bold('Error generating JSON schema from the given metadata report')); + return; } - //TODO: make this be independent of CLI usage. - await extractFilesFromZip({ outputPath: OUTPUT_DIR, zipPath: zipFilePath }); - const files = await readDirectory(OUTPUT_DIR); - - if (!files.length) throw new Error(`No JSON files found in the archive at ${zipFilePath}`); - - await validatePayloadAndGenerateResults({ - errorPath, - schemaPath, - payloadPaths: files.map(f => path.join(OUTPUT_DIR, f)), - version, - metadataPath, - additionalProperties - }); - } else { - await validatePayloadAndGenerateResults({ - errorPath, - payloadPaths: [payloadPath], - schemaPath, - version, - metadataPath, - additionalProperties - }); + return { schema }; + } catch (error) { + console.log(error); + console.log(chalk.redBright.bold('Something went wrong while generating the schema')); } }; -async function validatePayloadAndGenerateResults({ schemaPath, payloadPaths, errorPath, version, metadataPath, additionalProperties }) { +const validatePayloads = async ({ metadataPath = '', payloadPath = '', additionalProperties = false, version, resourceName }) => { + //TODO: make this be independent of CLI usage. + const files = await readDirectory(payloadPath); + + if (!files.length) throw new Error(`No JSON files found at ${payloadPath}`); + + await validatePayloadAndGenerateResults({ + errorPath: '.', + payloadPaths: files.map(f => path.join(OUTPUT_DIR, f)), + version, + metadataPath, + additionalProperties, + resourceName + }); +}; + +const validatePayloadAndGenerateResults = async ({ + payloadPaths, + errorPath, + version, + metadataPath, + additionalProperties, + resourceName +}) => { try { - let schemaJson = schemaPath ? JSON.parse((await readFile(schemaPath)) || null) : null; + let schemaJson = null; + + let metadataJson = metadataPath ? JSON.parse((await readFile(metadataPath)) || null) : null; + if (!metadataJson) { + // use RESO metadata report instead + metadataJson = getReferenceMetadata(version); + schemaJson = generateSchema(metadataJson, additionalProperties); + } else { + schemaJson = generateSchema(metadataJson, additionalProperties); + } + if (!schemaJson) { - let metadataJson = metadataPath ? JSON.parse((await readFile(metadataPath)) || null) : null; - if (!metadataJson) { - // use RESO metadata report instead - metadataJson = getReferenceMetadata(version); - schemaJson = generateSchema(metadataJson, true); - } else { - schemaJson = generateSchema(metadataJson, additionalProperties); - } - if (!schemaJson) { - console.log( - chalk.bgRed.bold( - 'Unable to generate a schema file. Pass the schema/metadata file in the options or check for invalid DD version.' - ) - ); - return; - } + console.log( + chalk.bgRed.bold('Unable to generate a schema file. Pass the schema/metadata file in the options or check for invalid DD version.') + ); + return; } + const payloadsJson = {}; for (const payloadPath of payloadPaths) { const payloadJson = JSON.parse((await readFile(payloadPath)) || null); @@ -149,13 +225,18 @@ async function validatePayloadAndGenerateResults({ schemaPath, payloadPaths, err console.log(chalk.redBright.bold('No payloads could be found')); return; } - const result = validatePayload(payloadsJson, schemaJson); - if (result.errors) { + const result = validatePayload({ + payloads: payloadsJson, + schema: schemaJson, + resourceNameFromArgs: resourceName, + versionFromArgs: version + }); + if (result?.errors) { const errorDirectoryExists = await createDirectoryIfNotPresent(errorPath); if (!errorDirectoryExists) throw new Error('Unable to create error directory'); const success = await writeFile( - path.join(errorPath, 'error-report.json'), + path.join(errorPath, ERROR_REPORT), JSON.stringify({ ...(result.errors || {}) }) @@ -173,9 +254,8 @@ async function validatePayloadAndGenerateResults({ schemaPath, payloadPaths, err console.log(error); console.log(chalk.redBright.bold('Something went wrong while validating the payload')); } -} +}; module.exports = { - validate, - generate + schema }; diff --git a/lib/schema/utils.js b/lib/schema/utils.js new file mode 100644 index 0000000..803cedc --- /dev/null +++ b/lib/schema/utils.js @@ -0,0 +1,47 @@ +const fs = require('fs'); +const fsPromises = fs.promises; +const path = require('path'); +const { extractFilesFromZip } = require('../../common'); + +const processFiles = async ({ inputPath, outputPath }) => { + try { + const stats = await fsPromises.stat(inputPath); + + if (stats.isFile()) { + await processFile({ filePath: inputPath, outputPath }); + } else if (stats.isDirectory()) { + const files = await fsPromises.readdir(inputPath); + + for (const file of files) { + await processFile({ + filePath: path.join(inputPath, file), + outputPath + }); + } + } else { + console.error(`Unsupported file type: ${inputPath}`); + } + } catch (error) { + console.log(error); + return { error }; + } +}; + +const processFile = async ({ filePath, outputPath }) => { + const ext = path.extname(filePath); + + if (ext === '.json') { + await fsPromises.copyFile(filePath, path.join(outputPath, path.basename(filePath))); + } else if (ext === '.zip') { + await extractFilesFromZip({ + zipPath: filePath, + outputPath + }); + } else { + console.error(`Unsupported file type: ${filePath}`); + } +}; + +module.exports = { + processFiles +}; diff --git a/lib/schema/validate.js b/lib/schema/validate.js index 79da7e5..429b76f 100644 --- a/lib/schema/validate.js +++ b/lib/schema/validate.js @@ -37,13 +37,14 @@ const DEFAULT_DD_VERSION = SUPPORTED_DD_VERSIONS.DD_1_7; const getValidDdVersions = () => Object.values(SUPPORTED_DD_VERSIONS ?? {}); const isValidDdVersion = (version = '') => getValidDdVersions()?.includes(version); -function validatePayload(payloads = {}, schema) { +function validatePayload({ payloads = {}, schema, resourceNameFromArgs = '', versionFromArgs }) { /** * Step 1 - Analyze the payload and parse out the relevant data like the version, resourceName, etc. * * Using this additional info we can improve our generated schema by providing a resource against which * the payload should be validated. */ + /** */ const payloadErrors = {}; const errorReport = []; const cache = {}; @@ -57,15 +58,30 @@ function validatePayload(payloads = {}, schema) { const origSchema = _.cloneDeep(schema); Object.entries(payloads).forEach(([fileName, payload]) => { - let resourceName, ddVersion; + let resourceName = resourceNameFromArgs, + ddVersion; + let validPayload = true; try { //TODO: 1. Not all payloads will contain this property - it will either by @reso.context or have at least one top-level @odata. property... // but in the OData case the version needs to be passed in from the command line and the resource name passed as well // 2. We also need to be able to take a passed-in version + if (!payload['@reso.context']) { - throw new Error('The required field "@reso.context" was not present in the payload'); + if (!versionFromArgs) { + validPayload = false; + throw new Error('Version is required for payloads without "@reso.context" property'); + } + if (!Object.keys(payload).some(k => k.startsWith('@odata.'))) { + validPayload = false; + throw new Error('No properties were found on the payload that match "@reso.context|@odata."'); + } } - const { resource, version } = parseResoUrn(payload['@reso.context']); + const { resource, version } = getResourceAndVersion({ + payload, + resource: resourceNameFromArgs, + version: versionFromArgs + }); + resourceName = resource; if (!isValidDdVersion(version)) { @@ -99,9 +115,11 @@ function validatePayload(payloads = {}, schema) { } } catch (error) { console.error(chalk.redBright.bold('ERROR: ' + error.message)); - addPayloadError(resourceName.toLowerCase(), fileName, error.message, payloadErrors); + addPayloadError(resourceName, fileName, error.message, payloadErrors); } + if (!validPayload) return; + // Step 2 - Validate with AJV and generate error report const [selectedSchema] = schema.oneOf; const additionalPropertiesAllowed = selectedSchema.properties.value @@ -215,7 +233,6 @@ function validatePayload(payloads = {}, schema) { } }; } - return true; } function generateErrorReport({ @@ -313,7 +330,10 @@ function parseResoUrn(urn = '') { const parts = urn?.split?.(':') || ''; if (parts.length < 6 || parts[0] !== 'urn' || parts[1] !== 'reso' || parts[2] !== 'metadata') { - throw new Error('Invalid URN'); + return { + version: '', + resource: '' + }; } return { @@ -322,6 +342,22 @@ function parseResoUrn(urn = '') { }; } +const getResourceAndVersion = ({ payload, resource, version }) => { + if (payload['@reso.context']) { + const { resource: parsedResource, version: parsedVersion } = parseResoUrn(payload['@reso.context']); + return { + resource: parsedResource, + version: parsedVersion + }; + } else { + return { + resource, + version + }; + } +}; + module.exports = { - validatePayload + validatePayload, + isValidDdVersion };