From 45af53b2f5cd7066abbffee3d2b03e2ff062dc3e Mon Sep 17 00:00:00 2001 From: mohit-s96 Date: Wed, 25 Sep 2024 02:02:39 +0530 Subject: [PATCH] #102: initial changes for rcf report generation works via cli when -c is passed still needs to work with the lib --- common.js | 7 +- index.js | 1 + lib/schema/create-report.js | 294 ++++++++++++++++++++++++++++++++++++ lib/schema/index.js | 69 ++++----- lib/schema/validate.js | 42 +++++- 5 files changed, 364 insertions(+), 49 deletions(-) create mode 100644 lib/schema/create-report.js diff --git a/common.js b/common.js index 289d6af..5c3487c 100644 --- a/common.js +++ b/common.js @@ -358,7 +358,7 @@ const buildMetadataMap = ({ fields = [], lookups = [] } = {}) => { ...fields.reduce( ( acc, - { resourceName, fieldName, type, isExpansion = false, isComplexType = false, annotations, typeName = '', nullable = true, isCollection = false } + { resourceName, fieldName, type, isExpansion = false, isComplexType = false, annotations, typeName = '', nullable = true, ...rest } ) => { if (!acc[resourceName]) { acc[resourceName] = {}; @@ -381,10 +381,11 @@ const buildMetadataMap = ({ fields = [], lookups = [] } = {}) => { typeName, nullable, isExpansion, - isCollection, isLookupField, isComplexType: isComplexType || (!isExpansion && !type?.startsWith('Edm.') && !isLookupField), - ddWikiUrl + annotations, + ddWikiUrl, + ...rest }; if (isLookupField && lookupMap?.[type]) { diff --git a/index.js b/index.js index 6b016ef..f78081c 100755 --- a/index.js +++ b/index.js @@ -137,6 +137,7 @@ if (require?.main === module) { .option('-a, --additionalProperties', 'Pass this flag to allow additional properties in the schema. False by default') .option('-v, --version ', 'The DD version of the metadata report') .option('-p, --payloadPath ', 'Path to the payload file OR directory/zip containing files that need to be validated') + .option('-c, --createReports', 'Option to generate metadata and availability reports for RCF testing') .option('-r, --resourceName ', 'Resource name to validate against. Required if --version is passed when validating.') .description('Generate a schema or validate a payload against a schema') .action(options => schema({ ...options, fromCli: FROM_CLI })); diff --git a/lib/schema/create-report.js b/lib/schema/create-report.js new file mode 100644 index 0000000..cfc8cc7 --- /dev/null +++ b/lib/schema/create-report.js @@ -0,0 +1,294 @@ +const { getMetadata } = require('@reso/reso-certification-etl/lib/common'); +const { createReplicationStateServiceInstance } = require('../../common'); +const { generateJsonSchema } = require('./generate'); +// const payload = require('/Users/mohit/Downloads/reso-replication-output/Property/2023-11-15T08-21-39.464Z/page-1.json'); +// const payload = require('/Users/mohit/Downloads/Property/2024-03-31T20-33-25.511Z/page-8.json'); +const replicationInstance = createReplicationStateServiceInstance(); + +const analyzeNumber = num => { + const result = {}; + + if (Number.isInteger(num)) { + if (num >= -32768 && num <= 32767) { + result.type = 'Edm.Int16'; + } else if (num >= -2147483648 && num <= 2147483647) { + result.type = 'Edm.Int32'; + } else { + result.type = 'Edm.Int64'; + } + } else { + result.type = 'Edm.Decimal'; + + const [, decimal] = num.toString().split('.'); + result.scale = decimal ? decimal.length : 0; + result.precision = num.toString().replace('.', '').length; + } + + return result; +}; +const inferType = value => { + if (Array.isArray(value)) { + const types = []; + value.forEach(v => types.push(inferType(v))); + const isExpansion = types.some(t => t.isExpansion); + return { types, isCollection: true, isExpansion }; + } + + if (typeof value === 'boolean') { + return { type: 'Edm.Boolean' }; + } + + if (typeof value === 'number') { + return analyzeNumber(value); + } + + if (typeof value === 'string') { + return { type: 'Edm.String' }; + } + + if (value === null) { + return { type: 'null', nullable: true }; + } + + if (typeof value === 'object') { + return { type: 'object', isExpansion: true }; + } + + // unreachable + throw Error('Unreachable code: Invalid Type'); +}; + +const buildPayloadCache = (payload, cache, resourceName, metadataMap) => { + payload = Array.isArray(payload.value) ? payload.value : [payload]; + payload.forEach(v => { + Object.entries(v).forEach(([key, value]) => { + const metadata = metadataMap?.[resourceName]?.[key]; + const { isExpansion: isLocalExpansion } = inferType(value); + if (metadata?.isExpansion) { + buildPayloadCache({ value }, cache, metadata?.typeName, metadataMap); + } else if (isLocalExpansion) { + buildPayloadCache({ value }, cache, key, metadataMap); + } else { + if (!cache[resourceName]) { + cache[resourceName] = {}; + } + if (key.startsWith('@')) return; + if (!cache[resourceName][key]) { + cache[resourceName][key] = []; + } + cache[resourceName][key].push(value); + } + }); + }); +}; + +const generateDDReport = ({ daReport, schema, payload, resourceName }) => { + const { MetadataMap } = schema.definitions || {}; + const { fields = [], lookupValues } = daReport || {}; + const ddFields = [], + ddLookups = []; + const localFields = [], + localLookups = []; + const cache = {}; + + buildPayloadCache(payload, cache, resourceName, MetadataMap); + lookupValues.forEach(l => { + const { resourceName, fieldName, lookupValue } = l; + const lookup = + MetadataMap?.[resourceName]?.[fieldName]?.lookupValues?.[lookupValue] ?? + MetadataMap?.[resourceName]?.[fieldName]?.legacyODataValues?.[lookupValue] ?? + {}; + const { type: localLookupName } = MetadataMap?.[resourceName]?.[fieldName] ?? {}; + const isReso = Object.keys(lookup).length > 0; + const { ddWikiUrl, type } = lookup; + if (isReso) { + const lookupObj = { + lookupName: type, + lookupValue, + type: 'Edm.Int32', + annotations: [ + { + term: 'RESO.DDWikiUrl', + value: ddWikiUrl + } + ] + }; + ddLookups.push(lookupObj); + } else { + localLookups.push({ + lookupName: localLookupName, + lookupValue, + type: 'Edm.Int32' + }); + } + }); + fields.forEach(f => { + const { resourceName, fieldName } = f; + const isReso = !!MetadataMap?.[resourceName]?.[fieldName]; + const { ddWikiUrl, legacyODataValues, isLookupField, lookupValues, ...fieldMetadata } = MetadataMap?.[resourceName]?.[fieldName] ?? {}; + if (isReso) { + const fieldObject = { + fieldName, + resourceName, + ...fieldMetadata + }; + ddFields.push(fieldObject); + } else { + localFields.push({ fieldName, resourceName }); + } + }); + localFields.forEach(({ fieldName, resourceName }) => { + const inferredMetadata = { + resourceName, + fieldName + }; + cache[resourceName][fieldName].forEach(v => { + const { type, types, isCollection, nullable, scale, precision, isExpansion } = inferType(v); + if (isCollection) { + inferredMetadata.isCollection = true; + const typeNames = [...new Set(types.map(t => t.type))]; + if (typeNames.length > 2) { + throw new Error('Impossible condition found'); + } + if (typeNames.includes('null') || nullable) { + inferredMetadata.nullable = true; + } + const nonNullTypes = typeNames.filter(x => x !== 'null'); + if (nonNullTypes.includes('object')) { + inferredMetadata.isExpansion = true; + } + // eslint-disable-next-line prefer-destructuring + inferredMetadata.type = nonNullTypes[0]; + } else { + if (type === 'null') return; + if (type?.startsWith('Edm.Int')) { + if (!inferredMetadata.type || inferredMetadata.type < type) { + inferredMetadata.type = type; + } + } else { + inferredMetadata.type = type; + } + if (isExpansion) { + inferredMetadata.isExpansion = true; + inferredMetadata.type = 'Custom Type'; + } + if (nullable) { + inferredMetadata.nullable = nullable; + } + if (scale) { + if (!inferredMetadata.scale || inferredMetadata.scale < scale) { + inferredMetadata.scale = scale; + } + } + if (precision) { + if (!inferredMetadata.precision || inferredMetadata.precision < precision) { + inferredMetadata.precision = precision; + } + } + if (type === 'Edm.String') { + const { length } = v; + if (!inferredMetadata.maxLength || inferredMetadata.maxLength < length) { + inferredMetadata.maxLength = length; + } + } + } + }); + ddFields.push(inferredMetadata); + }); + return { + description: 'RESO Data Dictionary Metadata Report', + generatedOn: new Date().toISOString(), + version: daReport.version, + fields: ddFields, + lookups: [...ddLookups, ...localLookups] + }; +}; + +const expansionInfoFromPayload = (payload, resourceName, metadataMap) => { + const expansionInfoMap = {}; + payload = Array.isArray(payload.value) ? payload.value : [payload]; + payload.forEach(p => { + Object.entries(p).forEach(([fieldName, value]) => { + const metadata = metadataMap?.[resourceName]?.[fieldName]; + const { isExpansion: isLocalExpansion, isCollection } = inferType(value); + if (metadata?.isExpansion) { + const modelName = metadata?.typeName; + if (!expansionInfoMap[modelName]) { + expansionInfoMap[modelName] = {}; + } + if (!expansionInfoMap[modelName][fieldName]) { + expansionInfoMap[modelName][fieldName] = { isCollection: metadata?.isCollection, type: metadata?.type }; + } + } else if (isLocalExpansion) { + const modelName = fieldName; + if (!expansionInfoMap[modelName]) { + expansionInfoMap[modelName] = {}; + } + if (!expansionInfoMap[modelName][fieldName]) { + expansionInfoMap[modelName][fieldName] = { isCollection: isCollection, type: 'Custom Type' }; + } + } + }); + }); + return Object.entries(expansionInfoMap).flatMap(([modelName, value]) => + Object.entries(value).map(([fieldName, { isCollection, type }]) => ({ fieldName, modelName, isCollection, type })) + ); +}; + +const generateRcfReports = async ({ payload, version, resourceName }) => { + const { scorePayload, consolidateResults } = require('../replication/utils'); + + const metadataReport = getMetadata(version); + const schema = await generateJsonSchema({ + metadataReportJson: metadataReport + }); + + const expansionInfo = expansionInfoFromPayload(payload, 'Property', schema.definitions.MetadataMap); + + replicationInstance.setMetadataMap(metadataReport); + scorePayload({ + expansionInfo: expansionInfo, + jsonData: payload, + replicationStateServiceInstance: replicationInstance, + resourceName + }); + + const daReport = { + description: 'RESO Data Availability Report', + version, + generatedOn: new Date().toISOString(), + ...consolidateResults({ + resourceAvailabilityMap: replicationInstance.getResourceAvailabilityMap(), + responses: replicationInstance.getResponses(), + topLevelResourceCounts: replicationInstance.getTopLevelResourceCounts() + }) + }; + + const ddReport = generateDDReport({ daReport, schema, payload, resourceName }); + + expansionInfo.forEach(({ fieldName, isCollection, modelName, type }) => { + ddReport.fields.push({ + resourceName, + fieldName, + typeName: modelName, + isCollection, + isExpansion: true, + type + }); + }); + + return { ddReport, daReport }; +}; + +const combineDDReports = (reports = []) => { + return reports.slice(1).reduce((acc, curr) => { + acc.fields = acc?.fields?.concat(curr?.fields); + acc.lookups = acc?.lookups?.concat(curr?.lookups); + return acc; + }, reports[0]); +}; + +module.exports = { + generateRcfReports, + combineDDReports +}; diff --git a/lib/schema/index.js b/lib/schema/index.js index a44eb57..073fea7 100644 --- a/lib/schema/index.js +++ b/lib/schema/index.js @@ -32,6 +32,7 @@ const schema = async ({ version, payloadPath, resourceName, + createReports = false, fromCli = false }) => { const handleError = getErrorHandler(fromCli); @@ -87,13 +88,18 @@ const schema = async ({ const fileContentsMap = {}; await processFiles({ inputPath: payloadPath, fileContentsMap }); - await validatePayloads({ - metadataPath, + if (!Object.keys(fileContentsMap).length) throw new Error('No JSON files found'); + + await validatePayloadAndGenerateResults({ + errorPath: '.', fileContentsMap, - additionalProperties, version, + metadataPath, + additionalProperties, resourceName, - validationConfig + validationConfig, + createReports, + outputPath }); } catch (err) { handleError(err); @@ -130,49 +136,18 @@ const generateSchemaFromMetadata = async ({ metadataPath = '', additionalPropert } }; -/** - * - * @param {Object} obj - * @param {boolean} obj.additionalProperties - * @param {string} obj.metadataPath - * @param {{}} obj.fileContentsMap - * @param {string} obj.version - * @param {string} obj.resourceName - * @param {Object} obj.validationConfig - * - * @description Reads the directory with the flattened JSON files and sends them for validation. - */ -const validatePayloads = async ({ - metadataPath = '', - fileContentsMap = {}, - additionalProperties = false, - version, - resourceName, - validationConfig = {} -}) => { - if (!Object.keys(fileContentsMap).length) throw new Error('No JSON files found'); - - await validatePayloadAndGenerateResults({ - errorPath: '.', - fileContentsMap, - version, - metadataPath, - additionalProperties, - resourceName, - validationConfig - }); -}; - /** * * @param {Object} obj * @param {boolean} obj.additionalProperties * @param {string} obj.errorPath + * @param {string} obj.outputPath * @param {string} obj.version * @param {string} obj.metadataPath * @param {string} obj.resourceName * @param {Object} obj.fileContentsMap * @param {Object} obj.validationConfig + * @param {boolean} obj.createReports * * @description Processes the input from the CLI and writes the final error report into the given path. */ @@ -183,7 +158,9 @@ const validatePayloadAndGenerateResults = async ({ metadataPath, additionalProperties, resourceName, - validationConfig = {} + validationConfig = {}, + createReports, + outputPath }) => { try { let schemaJson = null; @@ -202,14 +179,26 @@ const validatePayloadAndGenerateResults = async ({ throw new Error('Unable to generate a schema file. Pass the schema/metadata file in the options or check for invalid DD version.'); } - const result = validatePayload({ + const result = await validatePayload({ payloads: fileContentsMap, schema: schemaJson, resourceNameFromArgs: resourceName, versionFromArgs: version, - validationConfig + validationConfig, + createReports }); + if (createReports) { + const ddPath = path.join(outputPath, 'metadata-report.json'); + const daPath = path.join(outputPath, 'data-availability-report.json'); + + const { daReport, ddReport } = result; + await writeFile(ddPath, JSON.stringify(ddReport)); + console.log(`Metadata report written to: ${ddPath}`); + await writeFile(daPath, JSON.stringify(daReport)); + console.log(`Data Availability report written to: ${daPath}`); + } + if (result?.errors) { const errorDirectoryExists = await createDirectoryIfNotPresent(errorPath); if (!errorDirectoryExists) throw new Error('Unable to create error directory'); diff --git a/lib/schema/validate.js b/lib/schema/validate.js index c1ac1f2..dd11244 100644 --- a/lib/schema/validate.js +++ b/lib/schema/validate.js @@ -21,6 +21,7 @@ const { SCHEMA_ERROR_KEYWORDS } = require('./utils'); const { validationContext } = require('./context'); +const { generateRcfReports } = require('./create-report'); /** * @typedef {import('ajv').ValidateFunction} ValidateFunction @@ -191,6 +192,8 @@ const validate = ({ }; }; +const normalizePayload = payload => (Array.isArray(payload.value) ? payload : { value: [payload] }); + /** * TODO * @param {Object} obj @@ -199,10 +202,18 @@ const validate = ({ * @param {string} obj.resourceNameFromArgs * @param {string} obj.versionFromArgs * @param {Object} obj.validationConfig + * @param {boolean} obj.createReports * * @returns The processed error report */ -const validatePayload = ({ payloads = {}, schema, resourceNameFromArgs = '', versionFromArgs, validationConfig = {} }) => { +const validatePayload = async ({ + payloads = {}, + schema, + resourceNameFromArgs = '', + versionFromArgs, + validationConfig = {}, + createReports +}) => { const payloadErrors = {}; const errorCache = {}; const warningsCache = {}; @@ -220,8 +231,14 @@ const validatePayload = ({ payloads = {}, schema, resourceNameFromArgs = '', ver const isResoDataDictionarySchema = schema instanceof Map; + let payloadAccumulator = null; + Object.entries(payloads).forEach(([fileName, payload]) => { const { version } = getResourceAndVersion({ payload, version: DEFAULT_DD_VERSION }); + if (createReports) { + if (!payloadAccumulator) payloadAccumulator = payload; + else payloadAccumulator.value = payloadAccumulator.value.concat(normalizePayload(payload).value); + } errorMap = validate({ version: versionFromArgs, jsonPayload: payload, @@ -235,12 +252,25 @@ const validatePayload = ({ payloads = {}, schema, resourceNameFromArgs = '', ver }); const errorReport = combineErrors(errorMap); - - if (errorReport.items?.length) { - return { - errors: errorReport - }; + let daReport = null, + ddReport = null; + if (createReports) { + const reports = await generateRcfReports({ + payload: payloadAccumulator, + version: versionFromArgs, + resourceName: resourceNameFromArgs + }); + // eslint-disable-next-line prefer-destructuring + daReport = reports.daReport; + // eslint-disable-next-line prefer-destructuring + ddReport = reports.ddReport; } + + return { + ddReport, + daReport, + errors: errorReport.items?.length ? errorReport : null + }; }; /**