Merge pull request #576 from vlocityinc/beta

Beta to Master: Batch Processing for Large Inputs
vlocityinc · Mar 6, 2023 · 227bb1b · 227bb1b
2 parents a4acd8c + 0db0b26
commit 227bb1b
Showing 1 changed file with 31 additions and 5 deletions.
diff --git a/lib/datapacksjob.js b/lib/datapacksjob.js
@@ -2301,12 +2301,38 @@ DataPacksJob.prototype.deployBulkRecords = async function (dataPack, BulkRecords
         BulkRecords = JSON.parse(JSON.stringify(BulkRecords).replace(/%vlocity_namespace%__/g, ''));
     }
 
-    var result = await self.vlocity.datapacksutils.createBulkJob(objectName, 'insert', BulkRecords, randomNum);
+    const BATCH_SIZE_FOR_BULK_JOB = 1000;
 
-    if (result == 'Batch Upload Error') {
-        jobInfo.currentStatus[dataPack.VlocityDataPackKey] = 'Error';
-        jobInfo.hasError = true;
-        jobInfo.errors.push('Bulk Job Error ' + dataPack.VlocityDataPackKey);
+    let result = [],
+        startIndexAfterEachBatchDeployment = 0,
+        index = BATCH_SIZE_FOR_BULK_JOB,
+        isBatchDeploymentInProgress = true;
+
+    while (isBatchDeploymentInProgress) {
+        result.push(await self.vlocity.datapacksutils.createBulkJob(objectName, 'insert', BulkRecords.slice(startIndexAfterEachBatchDeployment, index), randomNum));
+        if (index === BulkRecords.length) {
+            isBatchDeploymentInProgress = false;
+        }
+        startIndexAfterEachBatchDeployment = index;
+        index = index + BATCH_SIZE_FOR_BULK_JOB;
+        if (index > BulkRecords.length) {
+            index = BulkRecords.length;
+        }
+    }
+
+    let error = '';
+    if (Array.isArray(result) && result.length > 0) {
+        result.forEach(errorMessage => {
+            if (errorMessage) {
+                error += errorMessage + '; '
+            }
+        });
+        if (error) {
+            VlocityUtils.error('Bulk Job Error', error);
+            jobInfo.currentStatus[dataPack.VlocityDataPackKey] = 'Error';
+            jobInfo.hasError = true;
+            jobInfo.errors.push('Bulk Job Error ' + dataPack.VlocityDataPackKey);
+        }
     }
 }