From ea983ee59bb1be936421813da94e9c130109ee80 Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Wed, 7 Jan 2026 19:00:16 -0600 Subject: [PATCH 1/3] fix: add migration to clean up duplicate backup folders with batching --- ...030036-cleanup-duplicate-backup-folders.js | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 migrations/20260122030036-cleanup-duplicate-backup-folders.js diff --git a/migrations/20260122030036-cleanup-duplicate-backup-folders.js b/migrations/20260122030036-cleanup-duplicate-backup-folders.js new file mode 100644 index 000000000..a9cf2df11 --- /dev/null +++ b/migrations/20260122030036-cleanup-duplicate-backup-folders.js @@ -0,0 +1,108 @@ +'use strict'; + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const MAX_ATTEMPTS = 10; +const BATCH_SIZE = 100; +const SLEEP_TIME_MS = 5000; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface) { + let totalDeleted = 0; + let batchCount = 0; + let attempts = 0; + + console.info(`Batch size: ${BATCH_SIZE} duplicate groups per batch`); + + console.info('Starting cleanup of duplicate backup folders...'); + + const deleteQuery = ` + WITH duplicate_groups AS ( + SELECT + plain_name, + bucket, + user_id, + MIN(id) as id_to_keep + FROM folders + WHERE + created_at >= '2025-12-17 14:16:00' + AND created_at <= '2026-01-05 21:50:00' + AND parent_id IS NULL + AND parent_uuid IS NULL + AND deleted = false + AND removed = false + AND plain_name IS NOT NULL + GROUP BY plain_name, bucket, user_id + HAVING COUNT(*) > 1 + LIMIT ${BATCH_SIZE} + ), + folders_to_delete AS ( + SELECT f.id + FROM folders f + INNER JOIN duplicate_groups dg + ON f.plain_name = dg.plain_name + AND f.bucket = dg.bucket + AND f.user_id = dg.user_id + WHERE + f.id != dg.id_to_keep + AND NOT EXISTS ( + SELECT 1 + FROM files + WHERE folder_id = f.id + AND deleted = false + ) + ) + UPDATE folders + SET + deleted = true, + deleted_at = NOW(), + removed = true, + removed_at = NOW() + FROM folders_to_delete + WHERE folders.id = folders_to_delete.id + AND folders.deleted = false + RETURNING folders.id; + `; + + let hasMore = true; + + while (hasMore) { + try { + const [results] = await queryInterface.sequelize.query(deleteQuery); + const deletedInBatch = results.length; + batchCount++; + totalDeleted += deletedInBatch; + attempts = 0; + + console.info( + `Batch ${batchCount}: Deleted ${deletedInBatch} folders (Total: ${totalDeleted})`, + ); + + hasMore = deletedInBatch > 0; + + if (hasMore) { + await sleep(SLEEP_TIME_MS); + } + } catch (err) { + attempts++; + console.error( + `[ERROR]: Error in batch ${batchCount} (attempt ${attempts}/${MAX_ATTEMPTS}): ${err.message}`, + ); + + if (attempts >= MAX_ATTEMPTS) { + console.error( + '[ERROR]: Maximum retry attempts reached, exiting migration.', + ); + break; + } + + await sleep(SLEEP_TIME_MS); + } + } + + console.info('\n=== Cleanup Complete ==='); + console.info(`Total batches processed: ${batchCount}`); + console.info(`Total folders deleted: ${totalDeleted}`); + }, + async down() {}, +}; From 189fb5f1d68bc2b88b2bb2ec8bc6a9d0e84d90d9 Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Thu, 22 Jan 2026 10:24:48 -0600 Subject: [PATCH 2/3] fix: update migration to refine duplicate backup folder cleanup logic and enhance test coverage for folder creation --- ...030036-cleanup-duplicate-backup-folders.js | 8 ++++- src/modules/backups/backup.usecase.spec.ts | 31 +++++++++++++++++-- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/migrations/20260122030036-cleanup-duplicate-backup-folders.js b/migrations/20260122030036-cleanup-duplicate-backup-folders.js index a9cf2df11..ba48a1083 100644 --- a/migrations/20260122030036-cleanup-duplicate-backup-folders.js +++ b/migrations/20260122030036-cleanup-duplicate-backup-folders.js @@ -49,7 +49,13 @@ module.exports = { SELECT 1 FROM files WHERE folder_id = f.id - AND deleted = false + AND status != 'DELETED' + ) + AND NOT EXISTS ( + SELECT 1 + FROM folders child + WHERE child.parent_uuid = f.uuid + AND child.deleted = false ) ) UPDATE folders diff --git a/src/modules/backups/backup.usecase.spec.ts b/src/modules/backups/backup.usecase.spec.ts index bf84686b6..8f438b390 100644 --- a/src/modules/backups/backup.usecase.spec.ts +++ b/src/modules/backups/backup.usecase.spec.ts @@ -80,17 +80,42 @@ describe('BackupUseCase', () => { }); describe('createDeviceAsFolder', () => { - it('When a folder with the same name exists, then it should throw a ConflictException', async () => { + it('When a folder with the same plainName exists, then it should throw a ConflictException', async () => { + const existingFolder = newFolder({ + attributes: { + plainName: 'Device Folder', + bucket: userMocked.backupsBucket, + }, + }); jest .spyOn(folderUseCases, 'getFolders') - .mockResolvedValue([{ id: 1, name: 'Device Folder' }] as any); + .mockResolvedValue([existingFolder]); await expect( backupUseCase.createDeviceAsFolder(userMocked, 'Device Folder'), ).rejects.toThrow(ConflictException); }); - it('When no folder with the same name exists, then it should create the folder', async () => { + it('When checking for duplicates, then it should use plainName (not encrypted name) and filter by bucket', async () => { + const getFoldersSpy = jest + .spyOn(folderUseCases, 'getFolders') + .mockResolvedValue([]); + const mockFolder = newFolder(); + jest + .spyOn(folderUseCases, 'createFolderDevice') + .mockResolvedValue(mockFolder); + + await backupUseCase.createDeviceAsFolder(userMocked, 'My Device'); + + expect(getFoldersSpy).toHaveBeenCalledWith(userMocked.id, { + bucket: userMocked.backupsBucket, + plainName: 'My Device', + deleted: false, + removed: false, + }); + }); + + it('When no folder with the same plainName exists, then it should create the folder', async () => { const mockFolder = newFolder(); jest.spyOn(folderUseCases, 'getFolders').mockResolvedValue([]); jest From 84025d3fb57ae14d252e1fd3c62f71287436e94a Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Thu, 22 Jan 2026 10:28:25 -0600 Subject: [PATCH 3/3] fix: refine cleanup logic in migration to exclude removed folders from duplicate backup folder deletion --- migrations/20260122030036-cleanup-duplicate-backup-folders.js | 1 + 1 file changed, 1 insertion(+) diff --git a/migrations/20260122030036-cleanup-duplicate-backup-folders.js b/migrations/20260122030036-cleanup-duplicate-backup-folders.js index ba48a1083..355af5710 100644 --- a/migrations/20260122030036-cleanup-duplicate-backup-folders.js +++ b/migrations/20260122030036-cleanup-duplicate-backup-folders.js @@ -67,6 +67,7 @@ module.exports = { FROM folders_to_delete WHERE folders.id = folders_to_delete.id AND folders.deleted = false + AND folders.removed = false RETURNING folders.id; `;