0.26.1 (#846)

apps/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@impler/api",
-  "version": "0.26.0",
+  "version": "0.26.1",
   "author": "implerhq",
   "license": "MIT",
   "private": true,
@@ -19,6 +19,7 @@
     "test": "cross-env TZ=UTC NODE_ENV=test E2E_RUNNER=true mocha --timeout 10000 --require ts-node/register --exit"
   },
   "dependencies": {
+    "@amplitude/analytics-node": "^1.3.6",
     "@impler/client": "workspace:^",
     "@impler/dal": "workspace:^",
     "@impler/services": "workspace:^",
@@ -51,6 +52,7 @@
     "dayjs": "^1.11.11",
     "dotenv": "^16.0.2",
     "envalid": "^7.3.1",
+    "exceljs": "^4.4.0",
     "hat": "^0.0.3",
     "jsonwebtoken": "^9.0.0",
     "jszip": "^3.10.1",

apps/api/src/app/project/usecases/create-project/create-project.usecase.ts

@@ -1,6 +1,6 @@
 import { Injectable } from '@nestjs/common';
 import { ProjectRepository } from '@impler/dal';
-import { ColumnTypesEnum } from '@impler/shared';
+import { ColumnTypesEnum, IntegrationEnum } from '@impler/shared';
 
 import { CreateProjectCommand } from './create-project.command';
 import { CreateEnvironment } from 'app/environment/usecases';
@@ -38,6 +38,7 @@ export class CreateProject {
       _projectId,
       chunkSize: 100,
       name: 'Sales Data Import',
+      integration: IntegrationEnum.JAVASCRIPT,
     });
     await this.updateTemplateColumns.execute(
       [

apps/api/src/app/review/review.module.ts

@@ -5,10 +5,11 @@ import { SharedModule } from '@shared/shared.module';
 import { AJVService } from './service/AJV.service';
 import { Sandbox, SManager } from '../shared/services/sandbox';
 import { QueueService } from '@shared/services/queue.service';
+import { AmplitudeService } from '@shared/services/amplitude.service';
 
 @Module({
   imports: [SharedModule],
-  providers: [...USE_CASES, AJVService, QueueService, SManager, Sandbox],
+  providers: [...USE_CASES, AJVService, QueueService, SManager, Sandbox, AmplitudeService],
   controllers: [ReviewController],
 })
 export class ReviewModule {}

apps/api/src/app/review/usecases/start-process/start-process.usecase.ts

@@ -10,6 +10,7 @@ import {
 } from '@impler/shared';
 import { PaymentAPIService } from '@impler/services';
 import { QueueService } from '@shared/services/queue.service';
+import { AmplitudeService } from '@shared/services/amplitude.service';
 import { DalService, TemplateEntity, UploadRepository } from '@impler/dal';
 
 @Injectable()
@@ -18,6 +19,7 @@ export class StartProcess {
     private dalService: DalService,
     private queueService: QueueService,
     private uploadRepository: UploadRepository,
+    private amplitudeService: AmplitudeService,
     private paymentAPIService: PaymentAPIService
   ) {}
 
@@ -58,9 +60,16 @@ export class StartProcess {
       );
     }
 
+    this.amplitudeService.recordsImported(userEmail, {
+      records: uploadInfo.totalRecords,
+      valid: uploadInfo.validRecords,
+      invalid: uploadInfo.invalidRecords,
+    });
+
     this.queueService.publishToQueue(QueuesEnum.END_IMPORT, {
       uploadId: _uploadId,
       destination: destination,
+      uploadedFileId: uploadInfo._uploadedFileId,
     });
 
     return { uploadInfo, importedData, email };

apps/api/src/app/shared/services/amplitude.service.ts

@@ -0,0 +1,18 @@
+import { Injectable } from '@nestjs/common';
+import { init, track } from '@amplitude/analytics-node';
+
+@Injectable()
+export class AmplitudeService {
+  constructor() {
+    if (process.env.AMPLITUDE_ID) {
+      init(process.env.AMPLITUDE_ID);
+    }
+  }
+  recordsImported(email: string, data: { records: number; valid: number; invalid: number }) {
+    if (process.env.AMPLITUDE_ID) {
+      track('RECORDS IMPORTED', data, {
+        user_id: email,
+      });
+    }
+  }
+}

apps/api/src/app/shared/services/file/file.service.ts

@@ -1,4 +1,5 @@
 import * as XLSX from 'xlsx';
+import * as exceljs from 'exceljs';
 import { cwd } from 'node:process';
 import * as xlsxPopulate from 'xlsx-populate';
 import { CONSTANTS } from '@shared/constants';
@@ -154,31 +155,13 @@ export class ExcelFileService {
   getExcelRowsColumnsCount(file: Express.Multer.File, sheetName?: string): Promise<{ rows: number; columns: number }> {
     return new Promise(async (resolve, reject) => {
       try {
-        const wb = XLSX.read(file.buffer);
-        const ws = sheetName && wb.SheetNames.includes(sheetName) ? wb.Sheets[sheetName] : wb.Sheets[wb.SheetNames[0]];
-        const range = ws['!ref'];
-        const regex = /([A-Z]+)(\d+):([A-Z]+)(\d+)/;
-        const match = range.match(regex);
-
-        if (!match) reject(new InvalidFileException());
-
-        const [, startCol, startRow, endCol, endRow] = match;
-
-        function columnToNumber(col: string) {
-          let num = 0;
-          for (let i = 0; i < col.length; i++) {
-            num = num * 26 + (col.charCodeAt(i) - 64);
-          }
-
-          return num;
-        }
-
-        const columns = columnToNumber(endCol) - columnToNumber(startCol) + 1;
-        const rows = parseInt(endRow) - parseInt(startRow) + 1;
+        const workbook = new exceljs.Workbook();
+        await workbook.xlsx.load(file.buffer);
+        const worksheet = workbook.getWorksheet(sheetName || workbook.worksheets[0].name);
 
         resolve({
-          columns,
-          rows,
+          columns: worksheet.actualColumnCount,
+          rows: worksheet.actualRowCount,
         });
       } catch (error) {
         reject(error);

apps/api/src/app/template/dtos/create-template-request.dto.ts

@@ -1,6 +1,6 @@
 import { ApiProperty } from '@nestjs/swagger';
-import { Defaults } from '@impler/shared';
-import { IsDefined, IsString, IsNumber, IsUrl, IsOptional, IsMongoId } from 'class-validator';
+import { Defaults, IntegrationEnum } from '@impler/shared';
+import { IsDefined, IsString, IsNumber, IsUrl, IsOptional, IsMongoId, IsEnum } from 'class-validator';
 
 export class CreateTemplateRequestDto {
   @ApiProperty({
@@ -10,6 +10,14 @@ export class CreateTemplateRequestDto {
   @IsDefined()
   name: string;
 
+  @ApiProperty({
+    description: 'Where do the user wanted to integrate the import',
+  })
+  @IsEnum(IntegrationEnum)
+  @IsOptional()
+  @IsDefined()
+  integration: IntegrationEnum;
+
   @ApiProperty({
     description: 'Callback URL of the template, gets called when sending data to the application',
   })

apps/api/src/app/template/dtos/template-response.dto.ts

@@ -1,5 +1,6 @@
+import { IntegrationEnum } from '@impler/shared';
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
-import { IsDefined, IsNumber, IsString } from 'class-validator';
+import { IsDefined, IsEnum, IsNumber, IsOptional, IsString } from 'class-validator';
 
 export class TemplateResponseDto {
   @ApiPropertyOptional({
@@ -16,6 +17,14 @@ export class TemplateResponseDto {
   @IsDefined()
   name: string;
 
+  @ApiProperty({
+    description: 'Where do the user wanted to integrate the import',
+  })
+  @IsOptional()
+  @IsEnum(IntegrationEnum)
+  @IsDefined()
+  integration: IntegrationEnum;
+
   @ApiProperty({
     description: 'URL to download samle csv file',
   })

apps/api/src/app/template/dtos/update-template-request.dto.ts

@@ -1,5 +1,6 @@
+import { IntegrationEnum } from '@impler/shared';
 import { ApiProperty } from '@nestjs/swagger';
-import { IsOptional } from 'class-validator';
+import { IsEnum, IsOptional } from 'class-validator';
 
 export class UpdateTemplateRequestDto {
   @ApiProperty({
@@ -15,4 +16,12 @@ export class UpdateTemplateRequestDto {
   })
   @IsOptional()
   mode?: string;
+
+  @ApiProperty({
+    description: 'Update Where do the user wanted to integrate the import',
+    nullable: true,
+  })
+  @IsOptional()
+  @IsEnum(IntegrationEnum)
+  integration?: IntegrationEnum;
 }

apps/api/src/app/template/template.controller.ts

@@ -17,7 +17,7 @@ import {
 } from '@nestjs/common';
 
 import { UploadEntity } from '@impler/dal';
-import { ACCESS_KEY_NAME, IJwtPayload } from '@impler/shared';
+import { ACCESS_KEY_NAME, IJwtPayload, IntegrationEnum } from '@impler/shared';
 import { JwtAuthGuard } from '@shared/framework/auth.gaurd';
 import { ValidateMongoId } from '@shared/validations/valid-mongo-id.validation';
 import { DocumentNotFoundException } from '@shared/exceptions/document-not-found.exception';
@@ -150,6 +150,7 @@ export class TemplateController {
         callbackUrl: body.callbackUrl,
         chunkSize: body.chunkSize,
         name: body.name,
+        integration: body.integration as IntegrationEnum,
       })
     );
   }

apps/api/src/app/template/usecases/create-template/create-template.command.ts

@@ -1,11 +1,17 @@
-import { IsDefined, IsString, IsNumber, IsOptional } from 'class-validator';
+import { IsDefined, IsString, IsNumber, IsOptional, IsEnum } from 'class-validator';
 import { BaseCommand } from '@shared/commands/base.command';
+import { IntegrationEnum } from '@impler/shared';
 
 export class CreateTemplateCommand extends BaseCommand {
   @IsDefined()
   @IsString()
   name: string;
 
+  @IsOptional()
+  @IsOptional()
+  @IsEnum(IntegrationEnum)
+  integration: IntegrationEnum;
+
   @IsString()
   @IsOptional()
   callbackUrl?: string;

apps/api/src/app/template/usecases/get-template-details/get-template-details.usecase.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
 import { TemplateRepository } from '@impler/dal';
 import { TemplateResponseDto } from 'app/template/dtos/template-response.dto';
 import { DocumentNotFoundException } from '@shared/exceptions/document-not-found.exception';
+import { IntegrationEnum } from '@impler/shared';
 
 @Injectable()
 export class GetTemplateDetails {
@@ -10,7 +11,7 @@ export class GetTemplateDetails {
   async execute(_id: string): Promise<TemplateResponseDto> {
     const template = await this.templateRepository.findOne(
       { _id },
-      '_projectId name sampleFileUrl _id totalUploads totalInvalidRecords totalRecords mode'
+      '_projectId name sampleFileUrl _id totalUploads totalInvalidRecords totalRecords mode integration'
     );
     if (!template) {
       throw new DocumentNotFoundException('Template', _id);
@@ -25,6 +26,7 @@ export class GetTemplateDetails {
       totalInvalidRecords: template.totalInvalidRecords,
       totalRecords: template.totalRecords,
       mode: template.mode,
+      integration: template.integration as IntegrationEnum,
     };
   }
 }

apps/api/src/app/template/usecases/update-template/update-template.command.ts

@@ -1,5 +1,6 @@
-import { IsString, IsOptional, IsMongoId } from 'class-validator';
+import { IsString, IsOptional, IsMongoId, IsEnum } from 'class-validator';
 import { BaseCommand } from '@shared/commands/base.command';
+import { IntegrationEnum } from '@impler/shared';
 
 export class UpdateTemplateCommand extends BaseCommand {
   @IsString()
@@ -15,4 +16,8 @@ export class UpdateTemplateCommand extends BaseCommand {
   @IsString()
   @IsOptional()
   mode?: string;
+
+  @IsEnum(IntegrationEnum)
+  @IsOptional()
+  integration?: IntegrationEnum;
 }

apps/api/src/types/env.d.ts

@@ -42,5 +42,7 @@ declare namespace NodeJS {
     PAYMENT_API_BASE_URL: string;
     PAYMENT_AUTH_KEY: string;
     PAYMENT_AUTH_VALUE: string;
+
+    AMPLITUDE_ID: string;
   }
 }

apps/queue-manager/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@impler/queue-manager",
-  "version": "0.26.0",
+  "version": "0.26.1",
   "author": "implerhq",
   "license": "MIT",
   "private": true,
@@ -23,6 +23,7 @@
     "axios": "1.6.2",
     "dotenv": "^16.0.2",
     "envalid": "^7.3.1",
+    "papaparse": "^5.4.1",
     "xml2js": "^0.6.2"
   },
   "devDependencies": {

apps/queue-manager/src/consumers/end-import.consumer.ts

@@ -1,21 +1,23 @@
-import { DalService, UploadRepository } from '@impler/dal';
+import { DalService, FileRepository, UploadRepository } from '@impler/dal';
 import { QueuesEnum, EndImportData, FileMimeTypesEnum, DestinationsEnum } from '@impler/shared';
 import { FileNameService, PaymentAPIService, StorageService } from '@impler/services';
 
 import { BaseConsumer } from './base.consumer';
 import { publishToQueue } from '../bootstrap';
+import * as Papa from 'papaparse';
 import { getStorageServiceClass } from '../helpers/serivces.helper';
 
 export class EndImportConsumer extends BaseConsumer {
   private dalService: DalService = new DalService();
+  private fileRepository: FileRepository = new FileRepository();
   private fileNameService: FileNameService = new FileNameService();
   private storageService: StorageService = getStorageServiceClass();
   private paymentAPIService: PaymentAPIService = new PaymentAPIService();
   private uploadRepository: UploadRepository = new UploadRepository();
 
   async message(message: { content: string }) {
     const data = JSON.parse(message.content) as EndImportData;
-    await this.convertRecordsToJsonFile(data.uploadId);
+    await this.convertRecordsToJsonFile(data.uploadId, data.uploadedFileId);
     const userEmail = await this.uploadRepository.getUserEmailFromUploadId(data.uploadId);
 
     const dataProcessingAllowed = await this.paymentAPIService.checkEvent({
@@ -35,10 +37,48 @@ export class EndImportConsumer extends BaseConsumer {
     }
   }
 
-  private async convertRecordsToJsonFile(uploadId: string) {
+  private async convertRecordsToJsonFile(uploadId: string, uploadedFileId?: string): Promise<void> {
     const importData = await this.dalService.getAllRecords(uploadId);
     const allJsonDataFilePath = this.fileNameService.getAllJsonDataFilePath(uploadId);
     await this.storageService.uploadFile(allJsonDataFilePath, JSON.stringify(importData), FileMimeTypesEnum.JSON);
     await this.dalService.dropRecordCollection(uploadId);
+
+    if (!uploadedFileId) {
+      const csvData = await this.convertImportDataToCsv(importData);
+      const importedFileName = this.fileNameService.getImportedFileName(uploadId);
+      const filePath = this.fileNameService.getImportedFilePath(uploadId);
+      const uploadedFileEntry = await this.fileRepository.create({
+        mimeType: FileMimeTypesEnum.CSV,
+        name: importedFileName,
+        originalName: importedFileName,
+        path: filePath,
+      });
+      await this.storageService.uploadFile(filePath, csvData, FileMimeTypesEnum.CSV);
+      await this.uploadRepository.findOneAndUpdate(
+        {
+          _id: uploadId,
+        },
+        {
+          $set: {
+            _uploadedFileId: uploadedFileEntry._id,
+            originalFileName: importedFileName,
+            originalFileType: FileMimeTypesEnum.CSV,
+          },
+        }
+      );
+    }
+  }
+
+  async convertImportDataToCsv(importData) {
+    const recordsData = importData.map((item) => item.record);
+
+    const csv = Papa.unparse(recordsData, {
+      header: true,
+      quotes: true,
+      quoteChar: '"',
+      escapeChar: '"',
+    });
+
+    return csv;
   }
 }