From b26620b79e17b5d59b7c5ede96af8b7749443304 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Tue, 25 Nov 2025 18:43:03 +0100 Subject: [PATCH 1/5] Feat: Upload Service --- src/services/network/upload/upload.service.ts | 231 +++++++++++ src/services/network/upload/upload.types.ts | 54 +++ .../network/upload.service.helpers.ts | 46 +++ test/services/network/upload.service.test.ts | 366 ++++++++++++++++++ 4 files changed, 697 insertions(+) create mode 100644 src/services/network/upload/upload.service.ts create mode 100644 src/services/network/upload/upload.types.ts create mode 100644 test/services/network/upload.service.helpers.ts create mode 100644 test/services/network/upload.service.test.ts diff --git a/src/services/network/upload/upload.service.ts b/src/services/network/upload/upload.service.ts new file mode 100644 index 00000000..129228cc --- /dev/null +++ b/src/services/network/upload/upload.service.ts @@ -0,0 +1,231 @@ +import { logger } from '../../../utils/logger.utils'; +import { + CreateFoldersParams, + CreateFolderWithRetryParams, + UploadFilesInBatchesParams, + UploadFileWithRetryParams, +} from './upload.types'; +import { DriveFolderService } from '../../drive/drive-folder.service'; +import { DriveFileService } from '../../drive/drive-file.service'; +import { ThumbnailService } from '../../thumbnail.service'; +import { dirname, extname } from 'node:path'; +import { isAlreadyExistsError, ErrorUtils } from '../../../utils/errors.utils'; +import { createReadStream } from 'node:fs'; +import { stat } from 'node:fs/promises'; +import { EncryptionVersion } from '@internxt/sdk/dist/drive/storage/types'; +import { BufferStream } from '../../../utils/stream.utils'; +import { isFileThumbnailable } from '../../../utils/thumbnail.utils'; +import { Readable } from 'node:stream'; + +export class UploadService { + static readonly instance = new UploadService(); + MAX_CONCURRENT_UPLOADS = 5; + DELAYS_MS = [500, 1000, 2000]; + MAX_RETRIES = 3; + + async createFolders({ + foldersToCreate, + destinationFolderUuid, + currentProgress, + emitProgress, + }: CreateFoldersParams): Promise> { + const folderMap = new Map(); + for (const folder of foldersToCreate) { + const parentPath = dirname(folder.relativePath); + const parentUuid = parentPath === '.' ? destinationFolderUuid : folderMap.get(parentPath); + + if (!parentUuid) { + logger.warn(`Parent folder not found for ${folder.relativePath}, skipping...`); + continue; + } + + const createdFolderUuid = await this.createFolderWithRetry({ + folderName: folder.name, + parentFolderUuid: parentUuid, + }); + + if (createdFolderUuid) { + folderMap.set(folder.relativePath, createdFolderUuid); + currentProgress.itemsUploaded++; + emitProgress(); + } + } + return folderMap; + } + + async createFolderWithRetry({ folderName, parentFolderUuid }: CreateFolderWithRetryParams): Promise { + for (let attempt = 0; attempt < this.MAX_RETRIES; attempt++) { + try { + const [createFolderPromise] = DriveFolderService.instance.createFolder({ + plainName: folderName, + parentFolderUuid, + }); + + const createdFolder = await createFolderPromise; + return createdFolder.uuid; + } catch (error: unknown) { + if (isAlreadyExistsError(error)) { + logger.info(`Folder ${folderName} already exists, skipping...`); + return null; + } + if (attempt < this.MAX_RETRIES - 1) { + const delay = this.DELAYS_MS[attempt]; + logger.warn( + `Failed to create folder ${folderName}, + retrying in ${delay}ms... (attempt ${attempt + 1}/${this.MAX_RETRIES})`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger.error(`Failed to create folder ${folderName} after ${this.MAX_RETRIES} attempts: ${error}`); + throw error; + } + } + } + return null; + } + + async uploadFilesInBatches({ + network, + filesToUpload, + folderMap, + bucket, + destinationFolderUuid, + currentProgress, + emitProgress, + }: UploadFilesInBatchesParams): Promise { + let bytesUploaded = 0; + + for (let i = 0; i < filesToUpload.length; i += this.MAX_CONCURRENT_UPLOADS) { + const batch = filesToUpload.slice(i, i + this.MAX_CONCURRENT_UPLOADS); + + await Promise.allSettled( + batch.map(async (file) => { + try { + const createdFileUuid = await this.uploadFileWithRetry({ + file, + folderMap, + network, + bucket, + destinationFolderUuid, + }); + if (createdFileUuid) { + bytesUploaded += file.size; + currentProgress.bytesUploaded += file.size; + } + } catch (error) { + logger.error(`Failed to upload file ${file.relativePath}: ${error}`); + } finally { + currentProgress.itemsUploaded++; + emitProgress(); + } + }), + ); + } + + return bytesUploaded; + } + + async uploadFileWithRetry({ + file, + folderMap, + network, + bucket, + destinationFolderUuid, + }: UploadFileWithRetryParams): Promise { + for (let attempt = 0; attempt < this.MAX_RETRIES; attempt++) { + try { + const parentRelativePath = dirname(file.relativePath); + const parentUuid = + parentRelativePath === '.' || parentRelativePath === '' + ? destinationFolderUuid + : folderMap.get(parentRelativePath); + + if (!parentUuid) { + logger.warn(`Parent folder not found for ${file.relativePath}, skipping...`); + return null; + } + + const stats = await stat(file.absolutePath); + if (!stats.size) { + logger.warn(`Skipping empty file: ${file.relativePath}`); + return null; + } + + const fileType = extname(file.absolutePath).replaceAll('.', ''); + + let bufferStream: BufferStream | undefined; + let fileStream: Readable = createReadStream(file.absolutePath); + const isThumbnailable = isFileThumbnailable(fileType); + + if (isThumbnailable) { + bufferStream = new BufferStream(); + fileStream = fileStream.pipe(bufferStream); + } + + const fileId = await new Promise((resolve, reject) => { + network.uploadFile( + fileStream, + stats.size, + bucket, + (err: Error | null, res: string | null) => { + if (err) { + return reject(err); + } + resolve(res as string); + }, + () => {}, + ); + }); + + const createdDriveFile = await DriveFileService.instance.createFile({ + plainName: file.name, + type: fileType, + size: stats.size, + folderUuid: parentUuid, + fileId, + bucket, + encryptVersion: EncryptionVersion.Aes03, + creationTime: stats.birthtime?.toISOString(), + modificationTime: stats.mtime?.toISOString(), + }); + + try { + if (isThumbnailable && bufferStream) { + const thumbnailBuffer = bufferStream.getBuffer(); + + if (thumbnailBuffer) { + await ThumbnailService.instance.uploadThumbnail( + thumbnailBuffer, + fileType, + bucket, + createdDriveFile.uuid, + network, + ); + } + } + } catch (error) { + ErrorUtils.report(error); + } + + return createdDriveFile.fileId; + } catch (error: unknown) { + if (isAlreadyExistsError(error)) { + const msg = `File ${file.name} already exists, skipping...`; + logger.info(msg); + return null; + } + + if (attempt < this.MAX_RETRIES - 1) { + const delay = this.DELAYS_MS[attempt]; + const retryMsg = `Failed to upload file ${file.name}, retrying in ${delay}ms...`; + logger.warn(`${retryMsg} (attempt ${attempt + 1}/${this.MAX_RETRIES})`); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger.error(`Failed to upload file ${file.name} after ${this.MAX_RETRIES} attempts`); + return null; + } + } + } + return null; + } +} diff --git a/src/services/network/upload/upload.types.ts b/src/services/network/upload/upload.types.ts new file mode 100644 index 00000000..229e15f0 --- /dev/null +++ b/src/services/network/upload/upload.types.ts @@ -0,0 +1,54 @@ +import { LoginUserDetails } from '../../../types/command.types'; +import { FileSystemNode } from '../../local-filesystem/local-filesystem.types'; +import { NetworkFacade } from '../network-facade.service'; + +export interface UploadResult { + totalBytes: number; + rootFolderId: string; + uploadTimeMs: number; +} + +export interface UploadFolderHandlerParams { + localPath: string; + destinationFolderUuid: string; + loginUserDetails: LoginUserDetails; + jsonFlag?: boolean; + onProgress: (progress: UploadProgress) => void; +} + +export type UploadFolderHandlerResult = { data: UploadResult; error?: undefined } | { error: Error; data?: undefined }; + +export interface UploadProgress { + percentage: number; + currentFile?: string; +} + +export interface CreateFoldersParams { + foldersToCreate: FileSystemNode[]; + destinationFolderUuid: string; + currentProgress: { itemsUploaded: number; bytesUploaded: number }; + emitProgress: () => void; +} + +export interface CreateFolderWithRetryParams { + folderName: string; + parentFolderUuid: string; +} + +export interface UploadFilesInBatchesParams { + network: NetworkFacade; + filesToUpload: FileSystemNode[]; + folderMap: Map; + bucket: string; + destinationFolderUuid: string; + currentProgress: { itemsUploaded: number; bytesUploaded: number }; + emitProgress: () => void; +} + +export interface UploadFileWithRetryParams { + file: FileSystemNode; + folderMap: Map; + network: NetworkFacade; + bucket: string; + destinationFolderUuid: string; +} diff --git a/test/services/network/upload.service.helpers.ts b/test/services/network/upload.service.helpers.ts new file mode 100644 index 00000000..f552f585 --- /dev/null +++ b/test/services/network/upload.service.helpers.ts @@ -0,0 +1,46 @@ +import { Readable } from 'stream'; +import { vi } from 'vitest'; +import { FileSystemNode } from '../../../src/services/local-filesystem/local-filesystem.types'; + +export function createFileSystemNodeFixture({ + type, + name, + relativePath, + size = 0, + absolutePath = `/absolute/${relativePath}`, +}: { + type: 'file' | 'folder'; + name: string; + relativePath: string; + size?: number; + absolutePath?: string; +}) { + return { type, name, relativePath, size, absolutePath } as FileSystemNode; +} + +export function createProgressFixtures() { + return { + currentProgress: { itemsUploaded: 0, bytesUploaded: 0 }, + emitProgress: vi.fn(), + }; +} + +export function createMockStats(size: number) { + return { + size, + birthtime: new Date('2024-01-01'), + mtime: new Date('2024-01-02'), + }; +} + +export function createMockReadStream() { + const stream = new Readable(); + stream.push(null); + return stream; +} + +export function createMockTimer() { + return { + stop: vi.fn().mockReturnValue(1000), + }; +} diff --git a/test/services/network/upload.service.test.ts b/test/services/network/upload.service.test.ts new file mode 100644 index 00000000..e0708421 --- /dev/null +++ b/test/services/network/upload.service.test.ts @@ -0,0 +1,366 @@ +import { beforeEach, describe, it, vi, expect } from 'vitest'; +import { UploadService } from '../../../src/services/network/upload/upload.service'; +import { NetworkFacade } from '../../../src/services/network/network-facade.service'; +import { DriveFolderService } from '../../../src/services/drive/drive-folder.service'; +import { DriveFileService } from '../../../src/services/drive/drive-file.service'; +import { logger } from '../../../src/utils/logger.utils'; +import { isAlreadyExistsError } from '../../../src/utils/errors.utils'; +import { stat } from 'fs/promises'; +import { createReadStream } from 'fs'; +import { isFileThumbnailable } from '../../../src/utils/thumbnail.utils'; +import { + createFileSystemNodeFixture, + createMockReadStream, + createMockStats, + createProgressFixtures, +} from './upload.service.helpers'; + +vi.mock('fs', () => ({ + createReadStream: vi.fn(), +})); + +vi.mock('fs/promises', () => ({ + stat: vi.fn(), +})); + +vi.mock('../../../src/services/drive/drive-file.service', () => ({ + DriveFileService: { + instance: { + createFile: vi.fn(), + }, + }, +})); + +vi.mock('../../../src/utils/thumbnail.utils', () => ({ + isFileThumbnailable: vi.fn(), +})); + +vi.mock('../../../src/services/drive/drive-folder.service', () => ({ + DriveFolderService: { + instance: { + createFolder: vi.fn(), + }, + }, +})); + +vi.mock('../../../src/utils/logger.utils', () => ({ + logger: { + warn: vi.fn(), + info: vi.fn(), + error: vi.fn(), + }, +})); + +vi.mock('../../../src/utils/errors.utils', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + isAlreadyExistsError: vi.fn(), + ErrorUtils: { + report: vi.fn(), + }, + }; +}); + +describe('UploadService', () => { + let sut: UploadService; + + const mockNetworkFacade = { + uploadFile: vi.fn((_stream, _size, _bucket, callback) => { + callback(null, 'mock-uploaded-file-id'); + return { stop: vi.fn() }; + }), + } as unknown as NetworkFacade; + + beforeEach(() => { + vi.clearAllMocks(); + sut = UploadService.instance; + vi.mocked(DriveFolderService.instance.createFolder).mockReturnValue([ + Promise.resolve({ uuid: 'mock-folder-uuid' }), + ] as unknown as ReturnType); + vi.mocked(isAlreadyExistsError).mockReturnValue(false); + vi.mocked(stat).mockResolvedValue(createMockStats(1024) as Awaited>); + vi.mocked(createReadStream).mockReturnValue(createMockReadStream() as ReturnType); + vi.mocked(isFileThumbnailable).mockReturnValue(false); + vi.mocked(DriveFileService.instance.createFile).mockResolvedValue({ + uuid: 'mock-file-uuid', + fileId: 'mock-file-id', + } as Awaited>); + }); + + describe('createFolders', () => { + const destinationFolderUuid = 'dest-uuid'; + + it('should properly return a map of created folders where key is relativePath and value is uuid', async () => { + const { currentProgress, emitProgress } = createProgressFixtures(); + vi.mocked(DriveFolderService.instance.createFolder) + .mockReturnValueOnce([Promise.resolve({ uuid: 'root-uuid' })] as unknown as ReturnType< + typeof DriveFolderService.instance.createFolder + >) + .mockReturnValueOnce([Promise.resolve({ uuid: 'subfolder-uuid' })] as unknown as ReturnType< + typeof DriveFolderService.instance.createFolder + >); + + const result = await sut.createFolders({ + foldersToCreate: [ + createFileSystemNodeFixture({ type: 'folder', name: 'root', relativePath: 'root' }), + createFileSystemNodeFixture({ type: 'folder', name: 'subfolder', relativePath: 'root/subfolder' }), + ], + destinationFolderUuid, + currentProgress, + emitProgress, + }); + + expect(result.size).toBe(2); + expect(result.get('root')).toBe('root-uuid'); + expect(result.get('root/subfolder')).toBe('subfolder-uuid'); + }); + + it('should properly skip over folders that dont have a parentUuid', async () => { + const { currentProgress, emitProgress } = createProgressFixtures(); + + const result = await sut.createFolders({ + foldersToCreate: [ + createFileSystemNodeFixture({ type: 'folder', name: 'orphan', relativePath: 'nonexistent/orphan' }), + ], + destinationFolderUuid, + currentProgress, + emitProgress, + }); + + expect(result.size).toBe(0); + expect(logger.warn).toHaveBeenCalledWith('Parent folder not found for nonexistent/orphan, skipping...'); + expect(DriveFolderService.instance.createFolder).not.toHaveBeenCalled(); + }); + + it('should properly set as parent folder the destinationFolderUuid for base folder', async () => { + const { currentProgress, emitProgress } = createProgressFixtures(); + + await sut.createFolders({ + foldersToCreate: [createFileSystemNodeFixture({ type: 'folder', name: 'root', relativePath: 'root' })], + destinationFolderUuid, + currentProgress, + emitProgress, + }); + + expect(DriveFolderService.instance.createFolder).toHaveBeenCalledWith({ + plainName: 'root', + parentFolderUuid: destinationFolderUuid, + }); + }); + + it('should properly update the progress on successful folder creation', async () => { + const { currentProgress, emitProgress } = createProgressFixtures(); + + await sut.createFolders({ + foldersToCreate: [createFileSystemNodeFixture({ type: 'folder', name: 'root', relativePath: 'root' })], + destinationFolderUuid, + currentProgress, + emitProgress, + }); + + expect(currentProgress.itemsUploaded).toBe(1); + expect(emitProgress).toHaveBeenCalledTimes(1); + }); + }); + describe('createFolderWithRetry', () => { + const folderName = 'test-folder'; + const parentFolderUuid = 'parent-uuid'; + + it('should properly create a folder and return the created folder uuid', async () => { + vi.mocked(DriveFolderService.instance.createFolder).mockReturnValueOnce([ + Promise.resolve({ uuid: 'created-folder-uuid' }), + ] as unknown as ReturnType); + + const result = await sut.createFolderWithRetry({ folderName, parentFolderUuid }); + + expect(result).toBe('created-folder-uuid'); + expect(DriveFolderService.instance.createFolder).toHaveBeenCalledWith({ + plainName: folderName, + parentFolderUuid, + }); + }); + + it('should properly return null if the folder already exists', async () => { + const alreadyExistsError = new Error('Folder already exists'); + vi.mocked(isAlreadyExistsError).mockReturnValue(true); + vi.mocked(DriveFolderService.instance.createFolder).mockReturnValueOnce([ + Promise.reject(alreadyExistsError), + ] as unknown as ReturnType); + + const result = await sut.createFolderWithRetry({ folderName, parentFolderUuid }); + + expect(result).toBeNull(); + expect(logger.info).toHaveBeenCalledWith(`Folder ${folderName} already exists, skipping...`); + }); + }); + describe('uploadFilesInBatches', () => { + const bucket = 'test-bucket'; + const destinationFolderUuid = 'dest-uuid'; + const folderMap = new Map(); + + it('should properly return the total amount of bytes uploaded once finished with the uploads', async () => { + const files = [ + createFileSystemNodeFixture({ type: 'file', name: 'file1.txt', relativePath: 'file1.txt', size: 100 }), + createFileSystemNodeFixture({ type: 'file', name: 'file2.txt', relativePath: 'file2.txt', size: 200 }), + createFileSystemNodeFixture({ type: 'file', name: 'file3.txt', relativePath: 'file3.txt', size: 300 }), + ]; + const { currentProgress, emitProgress } = createProgressFixtures(); + + const uploadFileWithRetrySpy = vi.spyOn(sut, 'uploadFileWithRetry').mockResolvedValue('mock-file-id'); + + const result = await sut.uploadFilesInBatches({ + network: mockNetworkFacade, + filesToUpload: files, + folderMap, + bucket, + destinationFolderUuid, + currentProgress, + emitProgress, + }); + + expect(result).toBe(600); + expect(uploadFileWithRetrySpy).toHaveBeenCalledTimes(3); + uploadFileWithRetrySpy.mockRestore(); + }); + + it('should properly upload files in batches of max 5', async () => { + const files = Array.from({ length: 12 }, (_, i) => + createFileSystemNodeFixture({ + type: 'file', + name: `file${i}.txt`, + relativePath: `file${i}.txt`, + size: 100, + }), + ); + const { currentProgress, emitProgress } = createProgressFixtures(); + + const uploadFileWithRetrySpy = vi.spyOn(sut, 'uploadFileWithRetry').mockResolvedValue('mock-file-id'); + + await sut.uploadFilesInBatches({ + network: mockNetworkFacade, + filesToUpload: files, + folderMap, + bucket, + destinationFolderUuid, + currentProgress, + emitProgress, + }); + + expect(uploadFileWithRetrySpy).toHaveBeenCalledTimes(12); + uploadFileWithRetrySpy.mockRestore(); + }); + + it('should properly emit progress and update the currentProgress object', async () => { + const files = [ + createFileSystemNodeFixture({ type: 'file', name: 'file1.txt', relativePath: 'file1.txt', size: 500 }), + createFileSystemNodeFixture({ type: 'file', name: 'file2.txt', relativePath: 'file2.txt', size: 1000 }), + ]; + const { currentProgress, emitProgress } = createProgressFixtures(); + + const uploadFileWithRetrySpy = vi.spyOn(sut, 'uploadFileWithRetry').mockResolvedValue('mock-file-id'); + + await sut.uploadFilesInBatches({ + network: mockNetworkFacade, + filesToUpload: files, + folderMap, + bucket, + destinationFolderUuid, + currentProgress, + emitProgress, + }); + + expect(currentProgress.itemsUploaded).toBe(2); + expect(currentProgress.bytesUploaded).toBe(1500); + expect(emitProgress).toHaveBeenCalledTimes(2); + uploadFileWithRetrySpy.mockRestore(); + }); + }); + describe('uploadFileWithRetry', () => { + const bucket = 'test-bucket'; + const destinationFolderUuid = 'dest-uuid'; + const folderMap = new Map(); + + it('should properly create a file and return the created file id', async () => { + const file = createFileSystemNodeFixture({ + type: 'file', + name: 'test.txt', + relativePath: 'test.txt', + size: 1024, + absolutePath: '/path/to/test.txt', + }); + + const result = await sut.uploadFileWithRetry({ + file, + folderMap, + network: mockNetworkFacade, + bucket, + destinationFolderUuid, + }); + + expect(result).toBe('mock-file-id'); + expect(stat).toHaveBeenCalledWith(file.absolutePath); + expect(createReadStream).toHaveBeenCalledWith(file.absolutePath); + expect(mockNetworkFacade.uploadFile).toHaveBeenCalledWith( + expect.anything(), + 1024, + bucket, + expect.any(Function), + expect.any(Function), + ); + expect(DriveFileService.instance.createFile).toHaveBeenCalledWith( + expect.objectContaining({ + plainName: 'test.txt', + type: 'txt', + size: 1024, + folderUuid: destinationFolderUuid, + fileId: 'mock-uploaded-file-id', + bucket, + }), + ); + }); + + it('should retry a maximum of 3 tries if an exception is thrown while uploading', async () => { + vi.useFakeTimers(); + const file = createFileSystemNodeFixture({ + type: 'file', + name: 'test.txt', + relativePath: 'test.txt', + size: 1024, + }); + const error = new Error('Network error'); + + vi.mocked(mockNetworkFacade.uploadFile) + .mockImplementationOnce((_stream, _size, _bucket, callback) => { + callback(error, null); + return { stop: vi.fn() } as unknown as ReturnType; + }) + .mockImplementationOnce((_stream, _size, _bucket, callback) => { + callback(error, null); + return { stop: vi.fn() } as unknown as ReturnType; + }) + .mockImplementationOnce((_stream, _size, _bucket, callback) => { + callback(null, 'success-file-id'); + return { stop: vi.fn() } as unknown as ReturnType; + }); + + const resultPromise = sut.uploadFileWithRetry({ + file, + folderMap, + network: mockNetworkFacade, + bucket, + destinationFolderUuid, + }); + + await vi.runAllTimersAsync(); + + const result = await resultPromise; + + expect(result).toBe('mock-file-id'); + expect(mockNetworkFacade.uploadFile).toHaveBeenCalledTimes(3); + expect(logger.warn).toHaveBeenCalledTimes(2); + + vi.useRealTimers(); + }); + }); +}); From 3384ef25bdcaf0b4d4780fc4928200950b3f7e64 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Wed, 26 Nov 2025 19:44:43 +0100 Subject: [PATCH 2/5] Feat: rename upload service to upload-file-service --- .../network/upload/upload-file.service.ts | 146 +++++++++++ src/services/network/upload/upload.service.ts | 231 ------------------ src/services/network/upload/upload.types.ts | 8 +- src/utils/thumbnail.utils.ts | 51 ++++ .../upload-file.service.test.ts} | 187 +++----------- .../{ => upload}/upload.service.helpers.ts | 2 +- 6 files changed, 242 insertions(+), 383 deletions(-) create mode 100644 src/services/network/upload/upload-file.service.ts delete mode 100644 src/services/network/upload/upload.service.ts rename test/services/network/{upload.service.test.ts => upload/upload-file.service.test.ts} (51%) rename test/services/network/{ => upload}/upload.service.helpers.ts (90%) diff --git a/src/services/network/upload/upload-file.service.ts b/src/services/network/upload/upload-file.service.ts new file mode 100644 index 00000000..b0e8acca --- /dev/null +++ b/src/services/network/upload/upload-file.service.ts @@ -0,0 +1,146 @@ +import { logger } from '../../../utils/logger.utils'; +import { + DELAYS_MS, + MAX_CONCURRENT_UPLOADS, + MAX_RETRIES, + UploadFilesInBatchesParams, + UploadFileWithRetryParams, +} from './upload.types'; +import { DriveFileService } from '../../drive/drive-file.service'; +import { dirname, extname } from 'node:path'; +import { isAlreadyExistsError } from '../../../utils/errors.utils'; +import { stat } from 'node:fs/promises'; +import { EncryptionVersion } from '@internxt/sdk/dist/drive/storage/types'; +import { createFileStreamWithBuffer, tryUploadThumbnail } from '../../../utils/thumbnail.utils'; + +export class UploadFileService { + static readonly instance = new UploadFileService(); + + async uploadFilesInChunks({ + network, + filesToUpload, + folderMap, + bucket, + destinationFolderUuid, + currentProgress, + emitProgress, + }: UploadFilesInBatchesParams): Promise { + let bytesUploaded = 0; + + const chunks = this.chunkArray(filesToUpload, MAX_CONCURRENT_UPLOADS); + + for (const chunk of chunks) { + await Promise.allSettled( + chunk.map(async (file) => { + const parentPath = dirname(file.relativePath); + const parentFolderUuid = + parentPath === '.' || parentPath === '' ? destinationFolderUuid : folderMap.get(parentPath); + + if (!parentFolderUuid) { + logger.warn(`Parent folder not found for ${file.relativePath}, skipping...`); + return null; + } + const createdFileUuid = await this.uploadFileWithRetry({ + file, + network, + bucket, + parentFolderUuid, + }); + if (createdFileUuid) { + bytesUploaded += file.size; + currentProgress.bytesUploaded += file.size; + currentProgress.itemsUploaded++; + } + emitProgress(); + }), + ); + } + return bytesUploaded; + } + + async uploadFileWithRetry({ + file, + network, + bucket, + parentFolderUuid, + }: UploadFileWithRetryParams): Promise { + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + try { + const stats = await stat(file.absolutePath); + if (!stats.size) { + logger.warn(`Skipping empty file: ${file.relativePath}`); + return null; + } + + const fileType = extname(file.absolutePath).replaceAll('.', ''); + const { fileStream, bufferStream } = createFileStreamWithBuffer({ + path: file.absolutePath, + fileType, + }); + + const fileId = await new Promise((resolve, reject) => { + network.uploadFile( + fileStream, + stats.size, + bucket, + (err: Error | null, res: string | null) => { + if (err) { + return reject(err); + } + resolve(res as string); + }, + () => {}, + ); + }); + + const createdDriveFile = await DriveFileService.instance.createFile({ + plainName: file.name, + type: fileType, + size: stats.size, + folderUuid: parentFolderUuid, + fileId, + bucket, + encryptVersion: EncryptionVersion.Aes03, + creationTime: stats.birthtime?.toISOString(), + modificationTime: stats.mtime?.toISOString(), + }); + + if (bufferStream) { + void tryUploadThumbnail({ + bufferStream, + fileType, + userBucket: bucket, + fileUuid: createdDriveFile.uuid, + networkFacade: network, + }); + } + + return createdDriveFile.fileId; + } catch (error: unknown) { + if (isAlreadyExistsError(error)) { + const msg = `File ${file.name} already exists, skipping...`; + logger.info(msg); + return null; + } + + if (attempt < MAX_RETRIES) { + const delay = DELAYS_MS[attempt]; + const retryMsg = `Failed to upload file ${file.name}, retrying in ${delay}ms...`; + logger.warn(`${retryMsg} (attempt ${attempt + 1}/${MAX_RETRIES + 1})`); + await new Promise((resolve) => setTimeout(resolve, delay)); + } else { + logger.error(`Failed to upload file ${file.name} after ${MAX_RETRIES + 1} attempts`); + return null; + } + } + } + return null; + } + private chunkArray(array: T[], chunkSize: number): T[][] { + const chunks: T[][] = []; + for (let i = 0; i < array.length; i += chunkSize) { + chunks.push(array.slice(i, i + chunkSize)); + } + return chunks; + } +} diff --git a/src/services/network/upload/upload.service.ts b/src/services/network/upload/upload.service.ts deleted file mode 100644 index 129228cc..00000000 --- a/src/services/network/upload/upload.service.ts +++ /dev/null @@ -1,231 +0,0 @@ -import { logger } from '../../../utils/logger.utils'; -import { - CreateFoldersParams, - CreateFolderWithRetryParams, - UploadFilesInBatchesParams, - UploadFileWithRetryParams, -} from './upload.types'; -import { DriveFolderService } from '../../drive/drive-folder.service'; -import { DriveFileService } from '../../drive/drive-file.service'; -import { ThumbnailService } from '../../thumbnail.service'; -import { dirname, extname } from 'node:path'; -import { isAlreadyExistsError, ErrorUtils } from '../../../utils/errors.utils'; -import { createReadStream } from 'node:fs'; -import { stat } from 'node:fs/promises'; -import { EncryptionVersion } from '@internxt/sdk/dist/drive/storage/types'; -import { BufferStream } from '../../../utils/stream.utils'; -import { isFileThumbnailable } from '../../../utils/thumbnail.utils'; -import { Readable } from 'node:stream'; - -export class UploadService { - static readonly instance = new UploadService(); - MAX_CONCURRENT_UPLOADS = 5; - DELAYS_MS = [500, 1000, 2000]; - MAX_RETRIES = 3; - - async createFolders({ - foldersToCreate, - destinationFolderUuid, - currentProgress, - emitProgress, - }: CreateFoldersParams): Promise> { - const folderMap = new Map(); - for (const folder of foldersToCreate) { - const parentPath = dirname(folder.relativePath); - const parentUuid = parentPath === '.' ? destinationFolderUuid : folderMap.get(parentPath); - - if (!parentUuid) { - logger.warn(`Parent folder not found for ${folder.relativePath}, skipping...`); - continue; - } - - const createdFolderUuid = await this.createFolderWithRetry({ - folderName: folder.name, - parentFolderUuid: parentUuid, - }); - - if (createdFolderUuid) { - folderMap.set(folder.relativePath, createdFolderUuid); - currentProgress.itemsUploaded++; - emitProgress(); - } - } - return folderMap; - } - - async createFolderWithRetry({ folderName, parentFolderUuid }: CreateFolderWithRetryParams): Promise { - for (let attempt = 0; attempt < this.MAX_RETRIES; attempt++) { - try { - const [createFolderPromise] = DriveFolderService.instance.createFolder({ - plainName: folderName, - parentFolderUuid, - }); - - const createdFolder = await createFolderPromise; - return createdFolder.uuid; - } catch (error: unknown) { - if (isAlreadyExistsError(error)) { - logger.info(`Folder ${folderName} already exists, skipping...`); - return null; - } - if (attempt < this.MAX_RETRIES - 1) { - const delay = this.DELAYS_MS[attempt]; - logger.warn( - `Failed to create folder ${folderName}, - retrying in ${delay}ms... (attempt ${attempt + 1}/${this.MAX_RETRIES})`, - ); - await new Promise((resolve) => setTimeout(resolve, delay)); - } else { - logger.error(`Failed to create folder ${folderName} after ${this.MAX_RETRIES} attempts: ${error}`); - throw error; - } - } - } - return null; - } - - async uploadFilesInBatches({ - network, - filesToUpload, - folderMap, - bucket, - destinationFolderUuid, - currentProgress, - emitProgress, - }: UploadFilesInBatchesParams): Promise { - let bytesUploaded = 0; - - for (let i = 0; i < filesToUpload.length; i += this.MAX_CONCURRENT_UPLOADS) { - const batch = filesToUpload.slice(i, i + this.MAX_CONCURRENT_UPLOADS); - - await Promise.allSettled( - batch.map(async (file) => { - try { - const createdFileUuid = await this.uploadFileWithRetry({ - file, - folderMap, - network, - bucket, - destinationFolderUuid, - }); - if (createdFileUuid) { - bytesUploaded += file.size; - currentProgress.bytesUploaded += file.size; - } - } catch (error) { - logger.error(`Failed to upload file ${file.relativePath}: ${error}`); - } finally { - currentProgress.itemsUploaded++; - emitProgress(); - } - }), - ); - } - - return bytesUploaded; - } - - async uploadFileWithRetry({ - file, - folderMap, - network, - bucket, - destinationFolderUuid, - }: UploadFileWithRetryParams): Promise { - for (let attempt = 0; attempt < this.MAX_RETRIES; attempt++) { - try { - const parentRelativePath = dirname(file.relativePath); - const parentUuid = - parentRelativePath === '.' || parentRelativePath === '' - ? destinationFolderUuid - : folderMap.get(parentRelativePath); - - if (!parentUuid) { - logger.warn(`Parent folder not found for ${file.relativePath}, skipping...`); - return null; - } - - const stats = await stat(file.absolutePath); - if (!stats.size) { - logger.warn(`Skipping empty file: ${file.relativePath}`); - return null; - } - - const fileType = extname(file.absolutePath).replaceAll('.', ''); - - let bufferStream: BufferStream | undefined; - let fileStream: Readable = createReadStream(file.absolutePath); - const isThumbnailable = isFileThumbnailable(fileType); - - if (isThumbnailable) { - bufferStream = new BufferStream(); - fileStream = fileStream.pipe(bufferStream); - } - - const fileId = await new Promise((resolve, reject) => { - network.uploadFile( - fileStream, - stats.size, - bucket, - (err: Error | null, res: string | null) => { - if (err) { - return reject(err); - } - resolve(res as string); - }, - () => {}, - ); - }); - - const createdDriveFile = await DriveFileService.instance.createFile({ - plainName: file.name, - type: fileType, - size: stats.size, - folderUuid: parentUuid, - fileId, - bucket, - encryptVersion: EncryptionVersion.Aes03, - creationTime: stats.birthtime?.toISOString(), - modificationTime: stats.mtime?.toISOString(), - }); - - try { - if (isThumbnailable && bufferStream) { - const thumbnailBuffer = bufferStream.getBuffer(); - - if (thumbnailBuffer) { - await ThumbnailService.instance.uploadThumbnail( - thumbnailBuffer, - fileType, - bucket, - createdDriveFile.uuid, - network, - ); - } - } - } catch (error) { - ErrorUtils.report(error); - } - - return createdDriveFile.fileId; - } catch (error: unknown) { - if (isAlreadyExistsError(error)) { - const msg = `File ${file.name} already exists, skipping...`; - logger.info(msg); - return null; - } - - if (attempt < this.MAX_RETRIES - 1) { - const delay = this.DELAYS_MS[attempt]; - const retryMsg = `Failed to upload file ${file.name}, retrying in ${delay}ms...`; - logger.warn(`${retryMsg} (attempt ${attempt + 1}/${this.MAX_RETRIES})`); - await new Promise((resolve) => setTimeout(resolve, delay)); - } else { - logger.error(`Failed to upload file ${file.name} after ${this.MAX_RETRIES} attempts`); - return null; - } - } - } - return null; - } -} diff --git a/src/services/network/upload/upload.types.ts b/src/services/network/upload/upload.types.ts index 229e15f0..8ac17c6b 100644 --- a/src/services/network/upload/upload.types.ts +++ b/src/services/network/upload/upload.types.ts @@ -16,8 +16,6 @@ export interface UploadFolderHandlerParams { onProgress: (progress: UploadProgress) => void; } -export type UploadFolderHandlerResult = { data: UploadResult; error?: undefined } | { error: Error; data?: undefined }; - export interface UploadProgress { percentage: number; currentFile?: string; @@ -47,8 +45,10 @@ export interface UploadFilesInBatchesParams { export interface UploadFileWithRetryParams { file: FileSystemNode; - folderMap: Map; network: NetworkFacade; bucket: string; - destinationFolderUuid: string; + parentFolderUuid: string; } +export const MAX_CONCURRENT_UPLOADS = 5; +export const DELAYS_MS = [500, 1000, 2000]; +export const MAX_RETRIES = 2; diff --git a/src/utils/thumbnail.utils.ts b/src/utils/thumbnail.utils.ts index 0684826b..1ae75295 100644 --- a/src/utils/thumbnail.utils.ts +++ b/src/utils/thumbnail.utils.ts @@ -1,3 +1,10 @@ +import { Readable } from 'node:stream'; +import { NetworkFacade } from '../services/network/network-facade.service'; +import { ThumbnailService } from '../services/thumbnail.service'; +import { ErrorUtils } from './errors.utils'; +import { BufferStream } from './stream.utils'; +import { createReadStream } from 'node:fs'; + export const ThumbnailConfig = { MaxWidth: 300, MaxHeight: 300, @@ -41,3 +48,47 @@ export const isPDFThumbnailable = (fileType: string) => { export const isImageThumbnailable = (fileType: string) => { return fileType.trim().length > 0 && thumbnailableImageExtension.includes(fileType.trim().toLowerCase()); }; + +export const tryUploadThumbnail = async ({ + bufferStream, + fileType, + userBucket, + fileUuid, + networkFacade, +}: { + bufferStream: BufferStream; + fileType: string; + userBucket: string; + fileUuid: string; + networkFacade: NetworkFacade; +}) => { + try { + const thumbnailBuffer = bufferStream.getBuffer(); + if (thumbnailBuffer) { + await ThumbnailService.instance.uploadThumbnail(thumbnailBuffer, fileType, userBucket, fileUuid, networkFacade); + } + } catch (error) { + ErrorUtils.report(error); + } +}; + +export const createFileStreamWithBuffer = ({ + path, + fileType, +}: { + path: string; + fileType: string; +}): { + bufferStream?: BufferStream; + fileStream: Readable; +} => { + const readable: Readable = createReadStream(path); + if (isFileThumbnailable(fileType)) { + const bufferStream = new BufferStream(); + return { + bufferStream, + fileStream: readable.pipe(bufferStream), + }; + } + return { fileStream: readable }; +}; diff --git a/test/services/network/upload.service.test.ts b/test/services/network/upload/upload-file.service.test.ts similarity index 51% rename from test/services/network/upload.service.test.ts rename to test/services/network/upload/upload-file.service.test.ts index e0708421..060d8cea 100644 --- a/test/services/network/upload.service.test.ts +++ b/test/services/network/upload/upload-file.service.test.ts @@ -1,13 +1,16 @@ import { beforeEach, describe, it, vi, expect } from 'vitest'; -import { UploadService } from '../../../src/services/network/upload/upload.service'; -import { NetworkFacade } from '../../../src/services/network/network-facade.service'; -import { DriveFolderService } from '../../../src/services/drive/drive-folder.service'; -import { DriveFileService } from '../../../src/services/drive/drive-file.service'; -import { logger } from '../../../src/utils/logger.utils'; -import { isAlreadyExistsError } from '../../../src/utils/errors.utils'; +import { UploadFileService } from '../../../../src/services/network/upload/upload-file.service'; +import { NetworkFacade } from '../../../../src/services/network/network-facade.service'; +import { DriveFileService } from '../../../../src/services/drive/drive-file.service'; +import { logger } from '../../../../src/utils/logger.utils'; +import { isAlreadyExistsError } from '../../../../src/utils/errors.utils'; import { stat } from 'fs/promises'; import { createReadStream } from 'fs'; -import { isFileThumbnailable } from '../../../src/utils/thumbnail.utils'; +import { + createFileStreamWithBuffer, + isFileThumbnailable, + tryUploadThumbnail, +} from '../../../../src/utils/thumbnail.utils'; import { createFileSystemNodeFixture, createMockReadStream, @@ -23,7 +26,7 @@ vi.mock('fs/promises', () => ({ stat: vi.fn(), })); -vi.mock('../../../src/services/drive/drive-file.service', () => ({ +vi.mock('../../../../src/services/drive/drive-file.service', () => ({ DriveFileService: { instance: { createFile: vi.fn(), @@ -31,19 +34,19 @@ vi.mock('../../../src/services/drive/drive-file.service', () => ({ }, })); -vi.mock('../../../src/utils/thumbnail.utils', () => ({ +vi.mock('../../../../src/utils/thumbnail.utils', () => ({ isFileThumbnailable: vi.fn(), + tryUploadThumbnail: vi.fn(), + createFileStreamWithBuffer: vi.fn(), })); -vi.mock('../../../src/services/drive/drive-folder.service', () => ({ - DriveFolderService: { - instance: { - createFolder: vi.fn(), - }, +vi.mock('../../../../src/utils/stream.utils', () => ({ + StreamUtils: { + createFileStreamWithBuffer: vi.fn(), }, })); -vi.mock('../../../src/utils/logger.utils', () => ({ +vi.mock('../../../../src/utils/logger.utils', () => ({ logger: { warn: vi.fn(), info: vi.fn(), @@ -51,19 +54,16 @@ vi.mock('../../../src/utils/logger.utils', () => ({ }, })); -vi.mock('../../../src/utils/errors.utils', async (importOriginal) => { - const actual = await importOriginal(); +vi.mock('../../../../src/utils/errors.utils', async (importOriginal) => { + const actual = await importOriginal(); return { ...actual, isAlreadyExistsError: vi.fn(), - ErrorUtils: { - report: vi.fn(), - }, }; }); -describe('UploadService', () => { - let sut: UploadService; +describe('UploadFileService', () => { + let sut: UploadFileService; const mockNetworkFacade = { uploadFile: vi.fn((_stream, _size, _bucket, callback) => { @@ -74,127 +74,23 @@ describe('UploadService', () => { beforeEach(() => { vi.clearAllMocks(); - sut = UploadService.instance; - vi.mocked(DriveFolderService.instance.createFolder).mockReturnValue([ - Promise.resolve({ uuid: 'mock-folder-uuid' }), - ] as unknown as ReturnType); + sut = UploadFileService.instance; vi.mocked(isAlreadyExistsError).mockReturnValue(false); vi.mocked(stat).mockResolvedValue(createMockStats(1024) as Awaited>); vi.mocked(createReadStream).mockReturnValue(createMockReadStream() as ReturnType); vi.mocked(isFileThumbnailable).mockReturnValue(false); + vi.mocked(createFileStreamWithBuffer).mockReturnValue({ + fileStream: createMockReadStream() as ReturnType, + bufferStream: undefined, + }); + vi.mocked(tryUploadThumbnail).mockResolvedValue(undefined); vi.mocked(DriveFileService.instance.createFile).mockResolvedValue({ uuid: 'mock-file-uuid', fileId: 'mock-file-id', } as Awaited>); }); - describe('createFolders', () => { - const destinationFolderUuid = 'dest-uuid'; - - it('should properly return a map of created folders where key is relativePath and value is uuid', async () => { - const { currentProgress, emitProgress } = createProgressFixtures(); - vi.mocked(DriveFolderService.instance.createFolder) - .mockReturnValueOnce([Promise.resolve({ uuid: 'root-uuid' })] as unknown as ReturnType< - typeof DriveFolderService.instance.createFolder - >) - .mockReturnValueOnce([Promise.resolve({ uuid: 'subfolder-uuid' })] as unknown as ReturnType< - typeof DriveFolderService.instance.createFolder - >); - - const result = await sut.createFolders({ - foldersToCreate: [ - createFileSystemNodeFixture({ type: 'folder', name: 'root', relativePath: 'root' }), - createFileSystemNodeFixture({ type: 'folder', name: 'subfolder', relativePath: 'root/subfolder' }), - ], - destinationFolderUuid, - currentProgress, - emitProgress, - }); - - expect(result.size).toBe(2); - expect(result.get('root')).toBe('root-uuid'); - expect(result.get('root/subfolder')).toBe('subfolder-uuid'); - }); - - it('should properly skip over folders that dont have a parentUuid', async () => { - const { currentProgress, emitProgress } = createProgressFixtures(); - - const result = await sut.createFolders({ - foldersToCreate: [ - createFileSystemNodeFixture({ type: 'folder', name: 'orphan', relativePath: 'nonexistent/orphan' }), - ], - destinationFolderUuid, - currentProgress, - emitProgress, - }); - - expect(result.size).toBe(0); - expect(logger.warn).toHaveBeenCalledWith('Parent folder not found for nonexistent/orphan, skipping...'); - expect(DriveFolderService.instance.createFolder).not.toHaveBeenCalled(); - }); - - it('should properly set as parent folder the destinationFolderUuid for base folder', async () => { - const { currentProgress, emitProgress } = createProgressFixtures(); - - await sut.createFolders({ - foldersToCreate: [createFileSystemNodeFixture({ type: 'folder', name: 'root', relativePath: 'root' })], - destinationFolderUuid, - currentProgress, - emitProgress, - }); - - expect(DriveFolderService.instance.createFolder).toHaveBeenCalledWith({ - plainName: 'root', - parentFolderUuid: destinationFolderUuid, - }); - }); - - it('should properly update the progress on successful folder creation', async () => { - const { currentProgress, emitProgress } = createProgressFixtures(); - - await sut.createFolders({ - foldersToCreate: [createFileSystemNodeFixture({ type: 'folder', name: 'root', relativePath: 'root' })], - destinationFolderUuid, - currentProgress, - emitProgress, - }); - - expect(currentProgress.itemsUploaded).toBe(1); - expect(emitProgress).toHaveBeenCalledTimes(1); - }); - }); - describe('createFolderWithRetry', () => { - const folderName = 'test-folder'; - const parentFolderUuid = 'parent-uuid'; - - it('should properly create a folder and return the created folder uuid', async () => { - vi.mocked(DriveFolderService.instance.createFolder).mockReturnValueOnce([ - Promise.resolve({ uuid: 'created-folder-uuid' }), - ] as unknown as ReturnType); - - const result = await sut.createFolderWithRetry({ folderName, parentFolderUuid }); - - expect(result).toBe('created-folder-uuid'); - expect(DriveFolderService.instance.createFolder).toHaveBeenCalledWith({ - plainName: folderName, - parentFolderUuid, - }); - }); - - it('should properly return null if the folder already exists', async () => { - const alreadyExistsError = new Error('Folder already exists'); - vi.mocked(isAlreadyExistsError).mockReturnValue(true); - vi.mocked(DriveFolderService.instance.createFolder).mockReturnValueOnce([ - Promise.reject(alreadyExistsError), - ] as unknown as ReturnType); - - const result = await sut.createFolderWithRetry({ folderName, parentFolderUuid }); - - expect(result).toBeNull(); - expect(logger.info).toHaveBeenCalledWith(`Folder ${folderName} already exists, skipping...`); - }); - }); - describe('uploadFilesInBatches', () => { + describe('uploadFilesInChunks', () => { const bucket = 'test-bucket'; const destinationFolderUuid = 'dest-uuid'; const folderMap = new Map(); @@ -209,7 +105,7 @@ describe('UploadService', () => { const uploadFileWithRetrySpy = vi.spyOn(sut, 'uploadFileWithRetry').mockResolvedValue('mock-file-id'); - const result = await sut.uploadFilesInBatches({ + const result = await sut.uploadFilesInChunks({ network: mockNetworkFacade, filesToUpload: files, folderMap, @@ -224,7 +120,7 @@ describe('UploadService', () => { uploadFileWithRetrySpy.mockRestore(); }); - it('should properly upload files in batches of max 5', async () => { + it('should properly upload files in chunks of max 5', async () => { const files = Array.from({ length: 12 }, (_, i) => createFileSystemNodeFixture({ type: 'file', @@ -237,7 +133,7 @@ describe('UploadService', () => { const uploadFileWithRetrySpy = vi.spyOn(sut, 'uploadFileWithRetry').mockResolvedValue('mock-file-id'); - await sut.uploadFilesInBatches({ + await sut.uploadFilesInChunks({ network: mockNetworkFacade, filesToUpload: files, folderMap, @@ -260,7 +156,7 @@ describe('UploadService', () => { const uploadFileWithRetrySpy = vi.spyOn(sut, 'uploadFileWithRetry').mockResolvedValue('mock-file-id'); - await sut.uploadFilesInBatches({ + await sut.uploadFilesInChunks({ network: mockNetworkFacade, filesToUpload: files, folderMap, @@ -279,12 +175,11 @@ describe('UploadService', () => { describe('uploadFileWithRetry', () => { const bucket = 'test-bucket'; const destinationFolderUuid = 'dest-uuid'; - const folderMap = new Map(); it('should properly create a file and return the created file id', async () => { const file = createFileSystemNodeFixture({ type: 'file', - name: 'test.txt', + name: 'test', relativePath: 'test.txt', size: 1024, absolutePath: '/path/to/test.txt', @@ -292,15 +187,13 @@ describe('UploadService', () => { const result = await sut.uploadFileWithRetry({ file, - folderMap, network: mockNetworkFacade, bucket, - destinationFolderUuid, + parentFolderUuid: destinationFolderUuid, }); expect(result).toBe('mock-file-id'); expect(stat).toHaveBeenCalledWith(file.absolutePath); - expect(createReadStream).toHaveBeenCalledWith(file.absolutePath); expect(mockNetworkFacade.uploadFile).toHaveBeenCalledWith( expect.anything(), 1024, @@ -310,21 +203,22 @@ describe('UploadService', () => { ); expect(DriveFileService.instance.createFile).toHaveBeenCalledWith( expect.objectContaining({ - plainName: 'test.txt', + plainName: 'test', type: 'txt', size: 1024, folderUuid: destinationFolderUuid, fileId: 'mock-uploaded-file-id', bucket, + encryptVersion: '03-aes', }), ); }); - it('should retry a maximum of 3 tries if an exception is thrown while uploading', async () => { + it('should retry a maximum of 2 retries (3 total attempts) if an exception is thrown while uploading', async () => { vi.useFakeTimers(); const file = createFileSystemNodeFixture({ type: 'file', - name: 'test.txt', + name: 'test', relativePath: 'test.txt', size: 1024, }); @@ -346,10 +240,9 @@ describe('UploadService', () => { const resultPromise = sut.uploadFileWithRetry({ file, - folderMap, network: mockNetworkFacade, bucket, - destinationFolderUuid, + parentFolderUuid: destinationFolderUuid, }); await vi.runAllTimersAsync(); diff --git a/test/services/network/upload.service.helpers.ts b/test/services/network/upload/upload.service.helpers.ts similarity index 90% rename from test/services/network/upload.service.helpers.ts rename to test/services/network/upload/upload.service.helpers.ts index f552f585..a8a0c918 100644 --- a/test/services/network/upload.service.helpers.ts +++ b/test/services/network/upload/upload.service.helpers.ts @@ -1,6 +1,6 @@ import { Readable } from 'stream'; import { vi } from 'vitest'; -import { FileSystemNode } from '../../../src/services/local-filesystem/local-filesystem.types'; +import { FileSystemNode } from '../../../../src/services/local-filesystem/local-filesystem.types'; export function createFileSystemNodeFixture({ type, From f2719a10f4a1666e64cb72f030f7f42219ac2357 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Wed, 26 Nov 2025 19:56:24 +0100 Subject: [PATCH 3/5] added tests to createFileStreamWithBuffer to improve coverage --- test/utils/thumbnail.utils.test.ts | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 test/utils/thumbnail.utils.test.ts diff --git a/test/utils/thumbnail.utils.test.ts b/test/utils/thumbnail.utils.test.ts new file mode 100644 index 00000000..60d1037a --- /dev/null +++ b/test/utils/thumbnail.utils.test.ts @@ -0,0 +1,30 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createFileStreamWithBuffer } from '../../src/utils/thumbnail.utils'; +import { BufferStream } from '../../src/utils/stream.utils'; +import path from 'node:path'; +import { Readable } from 'node:stream'; + +describe('createFileStreamWithBuffer', () => { + const testFilePath = path.join(process.cwd(), 'test/fixtures/test-content.fixture.txt'); + + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('should create BufferStream and pipe stream when file type is thumbnailable', () => { + const result = createFileStreamWithBuffer({ path: testFilePath, fileType: 'png' }); + + expect(result.bufferStream).toBeDefined(); + expect(result.bufferStream).toBeInstanceOf(BufferStream); + expect(result.fileStream).toBeDefined(); + expect(result.fileStream).toBeInstanceOf(Readable); + }); + + it('should not create BufferStream when file type is not thumbnailable', () => { + const result = createFileStreamWithBuffer({ path: testFilePath, fileType: 'txt' }); + + expect(result.bufferStream).toBeUndefined(); + expect(result.fileStream).toBeDefined(); + expect(result.fileStream).toBeInstanceOf(Readable); + }); +}); From 00404375c38268ff00fda4a8b678d42d4e28cd16 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Wed, 26 Nov 2025 20:05:21 +0100 Subject: [PATCH 4/5] More tests in upload file service to improve the coverage --- .../upload/upload-file.service.test.ts | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/test/services/network/upload/upload-file.service.test.ts b/test/services/network/upload/upload-file.service.test.ts index 060d8cea..6cf58e37 100644 --- a/test/services/network/upload/upload-file.service.test.ts +++ b/test/services/network/upload/upload-file.service.test.ts @@ -171,6 +171,38 @@ describe('UploadFileService', () => { expect(emitProgress).toHaveBeenCalledTimes(2); uploadFileWithRetrySpy.mockRestore(); }); + it('should skip files when parent folder is not found in folderMap', async () => { + const bucket = 'test-bucket'; + const destinationFolderUuid = 'dest-uuid'; + const folderMap = new Map(); + const files = [ + createFileSystemNodeFixture({ + type: 'file', + name: 'nested.txt', + relativePath: 'subfolder/nested.txt', + size: 100, + }), + ]; + const { currentProgress, emitProgress } = createProgressFixtures(); + + const uploadFileWithRetrySpy = vi.spyOn(sut, 'uploadFileWithRetry'); + + const result = await sut.uploadFilesInChunks({ + network: mockNetworkFacade, + filesToUpload: files, + folderMap, + bucket, + destinationFolderUuid, + currentProgress, + emitProgress, + }); + + expect(result).toBe(0); + expect(logger.warn).toHaveBeenCalledWith('Parent folder not found for subfolder/nested.txt, skipping...'); + expect(uploadFileWithRetrySpy).not.toHaveBeenCalled(); + + uploadFileWithRetrySpy.mockRestore(); + }); }); describe('uploadFileWithRetry', () => { const bucket = 'test-bucket'; @@ -255,5 +287,115 @@ describe('UploadFileService', () => { vi.useRealTimers(); }); + + it('should skip empty files and return null', async () => { + const file = createFileSystemNodeFixture({ + type: 'file', + name: 'empty.txt', + relativePath: 'empty.txt', + size: 0, + }); + + vi.mocked(stat).mockResolvedValue(createMockStats(0) as Awaited>); + + const result = await sut.uploadFileWithRetry({ + file, + network: mockNetworkFacade, + bucket, + parentFolderUuid: destinationFolderUuid, + }); + + expect(result).toBeNull(); + expect(logger.warn).toHaveBeenCalledWith('Skipping empty file: empty.txt'); + expect(mockNetworkFacade.uploadFile).not.toHaveBeenCalled(); + }); + + it('should call tryUploadThumbnail when bufferStream is present', async () => { + const mockBufferStream = { getBuffer: vi.fn() }; + vi.mocked(createFileStreamWithBuffer).mockReturnValue({ + fileStream: createMockReadStream() as ReturnType, + bufferStream: mockBufferStream as unknown as ReturnType['bufferStream'], + }); + + const file = createFileSystemNodeFixture({ + type: 'file', + name: 'image.png', + relativePath: 'image.png', + size: 1024, + }); + + await sut.uploadFileWithRetry({ + file, + network: mockNetworkFacade, + bucket, + parentFolderUuid: destinationFolderUuid, + }); + + expect(tryUploadThumbnail).toHaveBeenCalledWith({ + bufferStream: mockBufferStream, + fileType: 'png', + userBucket: bucket, + fileUuid: 'mock-file-uuid', + networkFacade: mockNetworkFacade, + }); + }); + + it('should return null when file already exists', async () => { + vi.mocked(isAlreadyExistsError).mockReturnValue(true); + vi.mocked(mockNetworkFacade.uploadFile).mockImplementation((_stream, _size, _bucket, callback) => { + callback(new Error('File already exists'), null); + return { stop: vi.fn() } as unknown as ReturnType; + }); + + const file = createFileSystemNodeFixture({ + type: 'file', + name: 'duplicate.txt', + relativePath: 'duplicate.txt', + size: 1024, + }); + + const result = await sut.uploadFileWithRetry({ + file, + network: mockNetworkFacade, + bucket, + parentFolderUuid: destinationFolderUuid, + }); + + expect(result).toBeNull(); + expect(logger.info).toHaveBeenCalledWith('File duplicate.txt already exists, skipping...'); + }); + + it('should return null after max retries exceeded', async () => { + vi.useFakeTimers(); + const file = createFileSystemNodeFixture({ + type: 'file', + name: 'fail.txt', + relativePath: 'fail.txt', + size: 1024, + }); + const error = new Error('Network error'); + + vi.mocked(mockNetworkFacade.uploadFile).mockImplementation((_stream, _size, _bucket, callback) => { + callback(error, null); + return { stop: vi.fn() } as unknown as ReturnType; + }); + + const resultPromise = sut.uploadFileWithRetry({ + file, + network: mockNetworkFacade, + bucket, + parentFolderUuid: destinationFolderUuid, + }); + + await vi.runAllTimersAsync(); + + const result = await resultPromise; + + expect(result).toBeNull(); + expect(logger.error).toHaveBeenCalledWith('Failed to upload file fail.txt after 3 attempts'); + expect(mockNetworkFacade.uploadFile).toHaveBeenCalledTimes(3); + + vi.useRealTimers(); + }); }); }); From 2a801909d30475e4a3284112e414538f917a3720 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Thu, 27 Nov 2025 12:01:00 +0100 Subject: [PATCH 5/5] delete unused fixture --- test/services/network/upload/upload.service.helpers.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/services/network/upload/upload.service.helpers.ts b/test/services/network/upload/upload.service.helpers.ts index b4fcdac4..9d43a911 100644 --- a/test/services/network/upload/upload.service.helpers.ts +++ b/test/services/network/upload/upload.service.helpers.ts @@ -37,9 +37,3 @@ export function createMockReadStream() { stream.push(null); return stream; } - -export function createMockTimer() { - return { - stop: vi.fn().mockReturnValue(1000), - }; -}