Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 171 additions & 20 deletions src/app/drive/services/downloadManager.service.test.ts

Large diffs are not rendered by default.

28 changes: 20 additions & 8 deletions src/app/drive/services/downloadManager.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@ export class DownloadManagerService {
readonly downloadFolder = async (
downloadTask: DownloadTask,
updateProgressCallback: (progress: number) => void,
updateDownloadedProgress: (progress: number) => void,
incrementItemCount: () => void,
) => {
const { connectionLost, cleanup } = this.handleConnectionLost(5000);
Expand All @@ -226,7 +227,6 @@ export class DownloadManagerService {
merge: {
status: TaskStatus.InProcess,
progress: Infinity,
nItems: 0,
},
});

Expand All @@ -237,6 +237,7 @@ export class DownloadManagerService {
foldersIterator: createFoldersIterator,
filesIterator: createFilesIterator,
updateProgress: updateProgressCallback,
downloadProgress: updateDownloadedProgress,
updateNumItems: incrementItemCount,
options: {
closeWhenFinished: true,
Expand Down Expand Up @@ -312,6 +313,7 @@ export class DownloadManagerService {
readonly downloadItems = async (
downloadTask: DownloadTask,
updateProgressCallback: (progress: number) => void,
updateDownloadedProgress: (progress: number) => void,
incrementItemCount: () => void,
) => {
const { connectionLost, cleanup } = this.handleConnectionLost(5000);
Expand All @@ -320,10 +322,13 @@ export class DownloadManagerService {

const folderZip = new FlatFolderZip(options.downloadName, { abortController });
const downloadProgress: number[] = [];
const lastReportedBytes: number[] = [];
const failedItems: DownloadItemType[] = [];
let downloadedProgress = 0;

items.forEach((_, index) => {
downloadProgress[index] = 0;
lastReportedBytes[index] = 0;
});

const calculateProgress = () => {
Expand Down Expand Up @@ -356,6 +361,18 @@ export class DownloadManagerService {
return;
}

const notifyProgressCallback = (totalBytes: number, downloadedBytes: number) => {
const progress = downloadedBytes / totalBytes;
downloadProgress[index] = progress;

const bytesDelta = downloadedBytes - lastReportedBytes[index];
downloadedProgress += bytesDelta;
lastReportedBytes[index] = downloadedBytes;

updateDownloadedProgress(bytesDelta);
updateProgressCallback(calculateProgress());
};

const downloadedFileStream = await downloadFile({
fileId: (driveItem as DriveFileData).fileId,
bucketId: (driveItem as DriveFileData).bucket,
Expand All @@ -366,13 +383,7 @@ export class DownloadManagerService {
mnemonic: (driveItem as AdvancedSharedItem).credentials?.mnemonic ?? credentials.mnemonic,
options: {
abortController,
notifyProgress: (totalBytes, downloadedBytes) => {
const progress = downloadedBytes / totalBytes;

downloadProgress[index] = progress;

updateProgressCallback(calculateProgress());
},
notifyProgress: notifyProgressCallback,
},
});

Expand All @@ -398,6 +409,7 @@ export class DownloadManagerService {
downloadProgress[index] = progress;
updateProgressCallback(calculateProgress());
},
downloadProgress: updateDownloadedProgress,
updateNumItems: incrementItemCount,
options: {
destination: folderZip,
Expand Down
98 changes: 69 additions & 29 deletions src/app/drive/services/folder.service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,37 +104,77 @@ describe('Folder Service', () => {
expect(downloadFile).not.toHaveBeenCalled();
});

test('When the file has content and is not cached, then it should download the file', async () => {
const { getFileStream } = await import('./folder.service');

const mockLruCache = {
get: vi.fn().mockResolvedValue(undefined),
};
vi.spyOn(LRUFilesCacheManager, 'getInstance').mockResolvedValue(mockLruCache as any);

const mockDownloadedStream = new ReadableStream();
vi.mocked(downloadFile).mockResolvedValue(mockDownloadedStream as any);

const mockBlob = new Blob(['downloaded content']);
vi.mocked(binaryStreamToBlob).mockResolvedValue(mockBlob);
vi.mocked(updateDatabaseFileSourceData).mockResolvedValue();

const stream = await getFileStream({
file: mockFile,
creds: { user: 'test-user', pass: 'test-pass' },
mnemonic: 'test-mnemonic',
describe('The file is not empty and is not cached', () => {
test('When the download starts, the download progress of the file should be reported correctly', async () => {
const { getFileStream } = await import('./folder.service');

const mockDownloadProgress = vi.fn();
const mockLruCache = {
get: vi.fn().mockResolvedValue(undefined),
};
vi.spyOn(LRUFilesCacheManager, 'getInstance').mockResolvedValue(mockLruCache as any);

const mockDownloadedStream = new ReadableStream();

// Simulate the download progress
vi.mocked(downloadFile).mockImplementation(async ({ options }) => {
if (options?.notifyProgress) {
options.notifyProgress(0, 25);
options.notifyProgress(0, 50);
options.notifyProgress(0, 75);
options.notifyProgress(0, 100);
}
return mockDownloadedStream as any;
});

const mockBlob = new Blob(['downloaded content']);
vi.mocked(binaryStreamToBlob).mockResolvedValue(mockBlob);
vi.mocked(updateDatabaseFileSourceData).mockResolvedValue();

await getFileStream({
file: mockFile,
creds: { user: 'test-user', pass: 'test-pass' },
mnemonic: 'test-mnemonic',
downloadProgress: mockDownloadProgress,
});

const totalBytesReported = mockDownloadProgress.mock.calls.reduce((sum, call) => sum + call[0], 0);

expect(totalBytesReported).toBe(mockFile.size);
});

expect(stream).toBeInstanceOf(ReadableStream);
expect(downloadFile).toHaveBeenCalledWith({
bucketId: mockFile.bucket,
fileId: mockFile.fileId,
creds: { user: 'test-user', pass: 'test-pass' },
mnemonic: 'test-mnemonic',
options: {
notifyProgress: expect.any(Function),
abortController: undefined,
},
test('When the download starts, then it should be completed successfully', async () => {
const { getFileStream } = await import('./folder.service');

const mockLruCache = {
get: vi.fn().mockResolvedValue(undefined),
};
vi.spyOn(LRUFilesCacheManager, 'getInstance').mockResolvedValue(mockLruCache as any);

const mockDownloadedStream = new ReadableStream();
vi.mocked(downloadFile).mockResolvedValue(mockDownloadedStream as any);

const mockBlob = new Blob(['downloaded content']);
vi.mocked(binaryStreamToBlob).mockResolvedValue(mockBlob);
vi.mocked(updateDatabaseFileSourceData).mockResolvedValue();

const stream = await getFileStream({
file: mockFile,
creds: { user: 'test-user', pass: 'test-pass' },
mnemonic: 'test-mnemonic',
});

expect(stream).toBeInstanceOf(ReadableStream);
expect(downloadFile).toHaveBeenCalledWith({
bucketId: mockFile.bucket,
fileId: mockFile.fileId,
creds: { user: 'test-user', pass: 'test-pass' },
mnemonic: 'test-mnemonic',
options: {
notifyProgress: expect.any(Function),
abortController: undefined,
},
});
});
});
});
Expand Down
15 changes: 13 additions & 2 deletions src/app/drive/services/folder.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ export interface GetFileStreamParams {
creds: { user: string; pass: string };
mnemonic: string;
abortController?: AbortController;
downloadProgress?: (progress: number) => void;
}

export interface IFolders {
Expand Down Expand Up @@ -227,26 +228,33 @@ export async function getFileStream({
creds,
mnemonic,
abortController,
downloadProgress,
}: GetFileStreamParams): Promise<ReadableStream<Uint8Array>> {
const lruFilesCacheManager = await LRUFilesCacheManager.getInstance();
const cachedFile = await lruFilesCacheManager.get(file.id?.toString());
const isCachedFileOlder = checkIfCachedSourceIsOlder({ cachedFile, file });

if (cachedFile?.source && !isCachedFileOlder) {
downloadProgress?.(file.size);
return cachedFile.source.stream();
}

if (isFileEmpty(file)) {
return new Blob([]).stream();
}

let lastReportedProgress = 0;
const downloadedFileStream = await downloadFile({
bucketId: file.bucket,
fileId: file.fileId,
creds,
mnemonic,
options: {
notifyProgress: () => {},
notifyProgress: (_, progress) => {
const progressDelta = progress - lastReportedProgress;
downloadProgress?.(progressDelta);
lastReportedProgress = progress;
},
abortController,
},
});
Expand Down Expand Up @@ -279,6 +287,7 @@ export async function downloadFolderAsZip({
updateNumItems,
options,
abortController,
downloadProgress,
}: {
folder: DriveFolderData;
isSharedFolder: boolean;
Expand All @@ -288,6 +297,7 @@ export async function downloadFolderAsZip({
updateNumItems: () => void;
options: DownloadFolderAsZipOptions;
abortController?: AbortController;
downloadProgress?: (progress: number) => void;
}): Promise<{
totalItems: DownloadFilesType;
failedItems: DownloadFilesType;
Expand Down Expand Up @@ -319,7 +329,7 @@ export async function downloadFolderAsZip({
const downloadQueue: QueueObject<FolderRef> = queue<FolderRef>((folderToDownload, next: (err?: Error) => void) => {
if (abortController?.signal.aborted) return next(new Error('Download aborted'));

const newConcurrency = QueueUtilsService.instance.getConcurrencyUsingPerfomance(
const newConcurrency = QueueUtilsService.instance.getConcurrencyUsingPerformance(
downloadQueue.concurrency,
maxConcurrency,
);
Expand Down Expand Up @@ -365,6 +375,7 @@ export async function downloadFolderAsZip({
creds: options.credentials,
mnemonic: options.mnemonic,
abortController,
downloadProgress,
});
} catch (error: unknown) {
if (isLostConnectionError(error)) {
Expand Down
Loading
Loading