multi select dropdown (#573)

apps/api/package.json

@@ -46,7 +46,7 @@
     "compression": "^1.7.4",
     "cookie-parser": "^1.4.6",
     "date-fns": "^2.30.0",
-    "dayjs": "^1.11.10",
+    "dayjs": "^1.11.11",
     "dotenv": "^16.0.2",
     "envalid": "^7.3.1",
     "exceljs": "^4.3.0",
@@ -62,7 +62,8 @@
     "socket.io": "^4.7.2",
     "source-map-support": "^0.5.21",
     "uuid": "^9.0.0",
-    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.1/xlsx-0.20.1.tgz"
+    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.1/xlsx-0.20.1.tgz",
+    "xlsx-populate": "^1.21.0"
   },
   "devDependencies": {
     "@nestjs/cli": "^9.1.5",

apps/api/src/app/column/commands/add-column.command.ts

@@ -58,4 +58,8 @@ export class AddColumnCommand extends BaseCommand {
   @IsOptional()
   @Validate(IsNumberOrString)
   defaultValue?: string | number;
+
+  @IsBoolean()
+  @IsOptional()
+  allowMultiSelect?: boolean;
 }

apps/api/src/app/column/commands/update-column.command.ts

@@ -54,4 +54,7 @@ export class UpdateColumnCommand extends BaseCommand {
   @IsOptional()
   @Validate(IsNumberOrString)
   defaultValue?: string | number;
+
+  @IsOptional()
+  allowMultiSelect?: boolean;
 }

apps/api/src/app/column/dtos/column-request.dto.ts

@@ -104,4 +104,11 @@ export class ColumnRequestDto {
   @IsOptional()
   @Validate(IsNumberOrString)
   defaultValue?: string | number;
+
+  @ApiPropertyOptional({
+    description: 'If true, column can have multiple values',
+  })
+  @IsBoolean()
+  @IsOptional()
+  allowMultiSelect?: boolean;
 }

apps/api/src/app/column/dtos/column-response.dto.ts

@@ -59,4 +59,9 @@ export class ColumnResponseDto {
     description: 'Sequence of column',
   })
   sequence?: number;
+
+  @ApiProperty({
+    description: 'If true, the column can have multiple values',
+  })
+  allowMultiSelect?: boolean;
 }

apps/api/src/app/column/usecases/update-column/update-column.usecase.ts

@@ -20,7 +20,7 @@ export class UpdateColumn {
     }
 
     command.dateFormats = command.dateFormats?.map((format) => format.toUpperCase()) || [];
-    const isKeyUpdated = command.key !== column.key;
+    const isKeyUpdated = command.key !== column.key || command.allowMultiSelect !== column.allowMultiSelect;
     const isTypeUpdated = command.type !== column.type;
     const isFieldConditionUpdated =
       JSON.stringify(column.selectValues) !== JSON.stringify(command.selectValues) ||

apps/api/src/app/mapping/usecases/rename-file-headings/rename-file-headings.usecase.ts

@@ -25,6 +25,8 @@ export class ReanameFileHeadings {
                 newHeadings[columnHeadingIndex],
                 newHeadings[keyHeadingIndex],
               ];
+            } else if (columnHeadingIndex > -1) {
+              newHeadings[columnHeadingIndex] = mapping.key;
             }
           }
         });

apps/api/src/app/review/usecases/do-review/base-review.usecase.ts

@@ -4,12 +4,15 @@ import * as Papa from 'papaparse';
 import { Writable } from 'stream';
 import addFormats from 'ajv-formats';
 import addKeywords from 'ajv-keywords';
+import * as customParseFormat from 'dayjs/plugin/customParseFormat';
 import Ajv, { AnySchemaObject, ErrorObject, ValidateFunction } from 'ajv';
 
 import { ColumnTypesEnum, Defaults, ITemplateSchemaItem } from '@impler/shared';
 
 import { SManager, BATCH_LIMIT, MAIN_CODE } from '@shared/services/sandbox';
 
+dayjs.extend(customParseFormat);
+
 interface IDataItem {
   index: number;
   errors?: Record<string, string>;
@@ -31,6 +34,7 @@ interface IRunData {
   dataStream: Writable;
   validator: ValidateFunction;
   extra: any;
+  multiSelectColumnHeadings: string[];
   dateFormats: Record<string, string[]>;
 }
 
@@ -115,11 +119,20 @@ export class BaseReview {
           Array.isArray(column.selectValues) && column.selectValues.length > 0
             ? [...column.selectValues, ...(column.isRequired ? [] : [''])]
             : [''];
-        property = {
-          type: 'string',
-          enum: Array.from(new Set(selectValues)), // handle duplicate
-          ...(!column.isRequired && { default: '' }),
-        };
+        if (column.allowMultiSelect)
+          property = {
+            type: 'array',
+            items: {
+              type: 'string',
+              enum: selectValues,
+            },
+          };
+        else
+          property = {
+            type: 'string',
+            enum: Array.from(new Set(selectValues)), // handle duplicate
+            ...(!column.isRequired && { default: '' }),
+          };
         break;
       case ColumnTypesEnum.REGEX:
         const [full, pattern, flags] = column.regex.match(/\/(.*)\/(.*)|(.*)/);
@@ -152,6 +165,7 @@ export class BaseReview {
     let field: string;
 
     return errors.reduce((obj, error) => {
+      console.log(error);
       if (error.keyword === 'required') field = error.params.missingProperty;
       else [, field] = error.instancePath.split('/');
 
@@ -250,6 +264,7 @@ export class BaseReview {
     headings,
     dateFormats,
     dataStream,
+    multiSelectColumnHeadings,
   }: IRunData): Promise<ISaveResults> {
     return new Promise(async (resolve, reject) => {
       let totalRecords = -1,
@@ -265,16 +280,34 @@ export class BaseReview {
           const record = results.data;
 
           if (totalRecords >= 1) {
-            const recordObj: Record<string, unknown> = headings.reduce((acc, heading, index) => {
-              if (heading === '_') return acc;
-
-              acc[heading] = typeof record[index] === 'string' ? record[index].trim() : record[index];
-
-              return acc;
-            }, {});
+            const recordObj: {
+              checkRecord: Record<string, unknown>;
+              passRecord: Record<string, unknown>;
+            } = headings.reduce(
+              (acc, heading, index) => {
+                if (heading === '_') return acc;
+
+                acc.checkRecord[heading] = multiSelectColumnHeadings.includes(heading)
+                  ? !record[index]
+                    ? []
+                    : record[index].split(',')
+                  : typeof record[index] === 'string'
+                  ? record[index].trim()
+                  : record[index];
+
+                acc.passRecord[heading] = typeof record[index] === 'string' ? record[index].trim() : record[index];
+
+                return acc;
+              },
+              {
+                checkRecord: {},
+                passRecord: {},
+              }
+            );
             const validationResultItem = this.validateRecord({
               index: totalRecords,
-              record: recordObj,
+              checkRecord: recordObj.checkRecord,
+              passRecord: recordObj.passRecord,
               validator,
               dateFormats,
             });
@@ -304,31 +337,33 @@ export class BaseReview {
 
   validateRecord({
     index,
-    record,
+    passRecord,
+    checkRecord,
     validator,
     dateFormats,
   }: {
     index: number;
-    record: Record<string, any>;
     validator: ValidateFunction;
+    passRecord: Record<string, any>;
+    checkRecord: Record<string, any>;
     dateFormats: Record<string, string[]>;
   }) {
-    const isValid = validator({ ...record });
+    const isValid = validator(checkRecord);
     if (!isValid) {
       const errors = this.getErrorsObject(validator.errors, dateFormats);
 
       return {
         index,
         errors: errors,
         isValid: false,
-        record,
+        record: passRecord,
         updated: {},
       };
     } else {
       return {
         index,
         isValid: true,
-        record,
+        record: passRecord,
         errors: {},
         updated: {},
       };
@@ -389,6 +424,7 @@ export class BaseReview {
     extra,
     csvFileStream,
     dateFormats,
+    multiSelectColumnHeadings,
   }: IRunData): Promise<IBatchItem[]> {
     return new Promise(async (resolve, reject) => {
       try {
@@ -403,16 +439,34 @@ export class BaseReview {
           step: (results: Papa.ParseStepResult<any>) => {
             recordsCount++;
             const record = results.data;
-            const recordObj = headings.reduce((acc, heading, index) => {
-              acc[heading] = typeof record[index] === 'string' ? record[index].trim() : record[index];
-
-              return acc;
-            }, {});
+            const recordObj: {
+              checkRecord: Record<string, unknown>;
+              passRecord: Record<string, unknown>;
+            } = headings.reduce(
+              (acc, heading, index) => {
+                if (heading === '_') return acc;
+
+                acc.checkRecord[heading] = multiSelectColumnHeadings.includes(heading)
+                  ? record[index]?.split(',')
+                  : typeof record[index] === 'string'
+                  ? record[index].trim()
+                  : record[index];
+
+                acc.passRecord[heading] = typeof record[index] === 'string' ? record[index].trim() : record[index];
+
+                return acc;
+              },
+              {
+                checkRecord: {},
+                passRecord: {},
+              }
+            );
 
             if (recordsCount >= 1) {
               const validationResultItem = this.validateRecord({
                 index: recordsCount,
-                record: recordObj,
+                checkRecord: recordObj.checkRecord,
+                passRecord: recordObj.passRecord,
                 validator,
                 dateFormats,
               });

apps/api/src/app/review/usecases/do-review/do-review.usecase.ts

@@ -3,7 +3,7 @@ import { Writable } from 'stream';
 import { Injectable, BadRequestException } from '@nestjs/common';
 
 import { APIMessages } from '@shared/constants';
-import { UploadStatusEnum } from '@impler/shared';
+import { ITemplateSchemaItem, UploadStatusEnum } from '@impler/shared';
 import { BaseReview } from './base-review.usecase';
 import { BATCH_LIMIT } from '@shared/services/sandbox';
 import { StorageService } from '@impler/shared/dist/services/storage';
@@ -33,22 +33,33 @@ export class DoReview extends BaseReview {
   async execute(_uploadId: string) {
     this._modal = await this.dalService.createRecordCollection(_uploadId);
 
-    const uploadInfo = await this.uploadRepository.findById(_uploadId);
+    const uploadInfo = await this.uploadRepository.findById(
+      _uploadId,
+      'customSchema _uploadedFileId _templateId extra headings'
+    );
     if (!uploadInfo) {
       throw new BadRequestException(APIMessages.UPLOAD_NOT_FOUND);
     }
     const dateFormats: Record<string, string[]> = {};
     const uniqueItems: Record<string, Set<any>> = {};
+    const columns = JSON.parse(uploadInfo.customSchema);
+    const multiSelectColumnHeadings = [];
+    (columns as ITemplateSchemaItem[]).forEach((column) => {
+      if (column.allowMultiSelect) multiSelectColumnHeadings.push(column.key);
+    });
     const schema = this.buildAJVSchema({
-      columns: JSON.parse(uploadInfo.customSchema),
+      columns,
       dateFormats,
       uniqueItems,
     });
     const ajv = this.getAjvValidator(dateFormats, uniqueItems);
     const validator = ajv.compile(schema);
 
-    const uploadedFileInfo = await this.fileRepository.findById(uploadInfo._uploadedFileId);
-    const validations = await this.validatorRepository.findOne({ _templateId: uploadInfo._templateId });
+    const uploadedFileInfo = await this.fileRepository.findById(uploadInfo._uploadedFileId, 'path');
+    const validations = await this.validatorRepository.findOne(
+      { _templateId: uploadInfo._templateId },
+      'onBatchInitialize'
+    );
 
     let response: ISaveResults;
 
@@ -66,6 +77,7 @@ export class DoReview extends BaseReview {
         extra: uploadInfo.extra,
         dataStream, // not-used
         dateFormats,
+        multiSelectColumnHeadings,
       });
 
       response = {
@@ -98,6 +110,7 @@ export class DoReview extends BaseReview {
         uploadId: _uploadId,
         validator,
         dateFormats,
+        multiSelectColumnHeadings,
       });
     }