#202: add chunk option to the validate function to toggle splitting…

… the payload off by default
RESOStandards · Nov 22, 2024 · f24bef1 · f24bef1
1 parent e37dac7
commit f24bef1
Showing 1 changed file with 32 additions and 26 deletions.
diff --git a/lib/schema/validate.js b/lib/schema/validate.js
@@ -21,7 +21,7 @@ const {
   SCHEMA_ERROR_KEYWORDS,
   getKeyFieldForResource,
   getValueForKeyField,
-  // chunkPayload,
+  chunkPayload,
   createHeapUsageLogger
 } = require('./utils');
 const { validationContext } = require('./context');
@@ -59,6 +59,7 @@ const ORIGINAL_SCHEMA = 'original_schema';
  * @param {Boolean?} obj.isResoDataDictionarySchema
  * @param {Object} obj.validationConfig
  * @param {boolean} obj.disableKeys
+ * @param {boolean} obj.chunk
  *
  * @returns Intermediate error and warning caches along with stats. Can be combined later using `combineErrors`
  */
@@ -71,7 +72,8 @@ const validate = ({
   version,
   isResoDataDictionarySchema = false /*CLI only */,
   validationConfig = {},
-  disableKeys
+  disableKeys,
+  chunk = true
 } = {}) => {
   const { stats = { totalErrors: 0, totalWarnings: 0 }, errorCache = {}, warningsCache = {}, payloadErrors = {} } = errorMap;
 
@@ -176,33 +178,37 @@ const validate = ({
   // console.time('Chunk payload');
   // const chunked = chunkPayload(payload);
   // console.timeEnd('Chunk payload');
+  let chunkedPayload = [payload];
+  if (chunk) {
+    chunkedPayload = chunkPayload(payload);
+  }
 
   console.time('Validate with AJV');
-  // const timings = [];
-  // for (const p of chunked) {
-  const valid = validate(payload);
-  console.timeEnd('Validate with AJV');
-  if (!valid) {
-    const start = Date.now();
-    generateErrorReport({
-      validate,
-      json: payload,
-      additionalPropertiesAllowed,
-      resourceName,
-      errorCache,
-      warningsCache,
-      stats,
-      fileName,
-      isResoDataDictionarySchema,
-      version: ddVersion,
-      isRCF: validationContext.isRCF(),
-      metadataMap: schema?.definitions?.MetadataMap || {}
-    });
-    const stop = Date.now();
-    console.log(`Time spent on generating reports: ${stop - start}ms`);
+  const timings = [];
+  for (const p of chunkedPayload) {
+    const valid = validate(p);
+    if (!valid) {
+      const start = Date.now();
+      generateErrorReport({
+        validate,
+        json: p,
+        additionalPropertiesAllowed,
+        resourceName,
+        errorCache,
+        warningsCache,
+        stats,
+        fileName,
+        isResoDataDictionarySchema,
+        version: ddVersion,
+        isRCF: validationContext.isRCF(),
+        metadataMap: schema?.definitions?.MetadataMap || {}
+      });
+      const stop = Date.now();
+      timings.push(stop - start);
+    }
   }
-  // }
-
+  console.timeEnd('Validate with AJV');
+  console.log(`Time spent on generating reports: ${timings.reduce((acc, curr) => (acc += curr), 0)}ms`);
 
   schema.oneOf = oldOneOf;
   validationContext.reset();