From 0d26e88f3e837d9a6ea629712bbe82ca31768516 Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Wed, 4 Feb 2026 18:57:22 -0600 Subject: [PATCH 1/3] feat: add migration to clean up duplicate folders --- ...0260205010629-cleanup-duplicate-folders.js | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 migrations/20260205010629-cleanup-duplicate-folders.js diff --git a/migrations/20260205010629-cleanup-duplicate-folders.js b/migrations/20260205010629-cleanup-duplicate-folders.js new file mode 100644 index 000000000..6b06ce042 --- /dev/null +++ b/migrations/20260205010629-cleanup-duplicate-folders.js @@ -0,0 +1,84 @@ +'use strict'; + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const MAX_ATTEMPTS = 10; +const BATCH_SIZE = 500; +const SLEEP_TIME_MS = 1000; + +module.exports = { + up: async (queryInterface) => { + let totalRenamed = 0; + let batchCount = 0; + let attempts = 0; + + console.info(`Batch size: ${BATCH_SIZE} duplicate folders per batch`); + console.info('Starting cleanup of duplicate folders...'); + + const renameQuery = ` + WITH duplicate_groups AS ( + SELECT parent_uuid, plain_name, MIN(id) as id_to_keep + FROM folders + WHERE deleted = false + AND removed = false + AND parent_uuid IS NOT NULL + AND plain_name IS NOT NULL + GROUP BY parent_uuid, plain_name + HAVING COUNT(*) > 1 + LIMIT ${BATCH_SIZE} + ) + UPDATE folders f + SET + plain_name = f.plain_name || '_' || f.id::text, + updated_at = NOW() + FROM duplicate_groups dg + WHERE f.parent_uuid = dg.parent_uuid + AND f.plain_name = dg.plain_name + AND f.id != dg.id_to_keep + AND f.deleted = false + AND f.removed = false + RETURNING f.id; + `; + + let hasMore = true; + + while (hasMore) { + try { + const [results] = await queryInterface.sequelize.query(renameQuery); + const renamedInBatch = results.length; + batchCount++; + totalRenamed += renamedInBatch; + attempts = 0; + + console.info( + `Batch ${batchCount}: Renamed ${renamedInBatch} folders (Total: ${totalRenamed})`, + ); + + hasMore = renamedInBatch > 0; + + if (hasMore) { + await sleep(SLEEP_TIME_MS); + } + } catch (err) { + attempts++; + console.error( + `[ERROR]: Error in batch ${batchCount} (attempt ${attempts}/${MAX_ATTEMPTS}): ${err.message}`, + ); + + if (attempts >= MAX_ATTEMPTS) { + console.error( + '[ERROR]: Maximum retry attempts reached, exiting migration.', + ); + throw err; + } + + await sleep(SLEEP_TIME_MS); + } + } + + console.info('\n=== Cleanup Complete ==='); + console.info(`Total batches processed: ${batchCount}`); + console.info(`Total folders renamed: ${totalRenamed}`); + }, + + down: async () => {}, +}; From 1ba09218ac65ecdde05c81f96b77f3a4b203e51d Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Thu, 5 Feb 2026 09:39:28 -0600 Subject: [PATCH 2/3] feat: enhance duplicate folder cleanup migration with supoort index creation and removal --- ...20260205010629-cleanup-duplicate-folders.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/migrations/20260205010629-cleanup-duplicate-folders.js b/migrations/20260205010629-cleanup-duplicate-folders.js index 6b06ce042..72410c447 100644 --- a/migrations/20260205010629-cleanup-duplicate-folders.js +++ b/migrations/20260205010629-cleanup-duplicate-folders.js @@ -14,6 +14,13 @@ module.exports = { console.info(`Batch size: ${BATCH_SIZE} duplicate folders per batch`); console.info('Starting cleanup of duplicate folders...'); + console.info('Creating supporting index...'); + await queryInterface.sequelize.query(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS folders_parentuuid_plainname_not_deleted_support_index + ON folders (parent_uuid, plain_name) + WHERE deleted = false AND removed = false; + `); + const renameQuery = ` WITH duplicate_groups AS ( SELECT parent_uuid, plain_name, MIN(id) as id_to_keep @@ -78,7 +85,16 @@ module.exports = { console.info('\n=== Cleanup Complete ==='); console.info(`Total batches processed: ${batchCount}`); console.info(`Total folders renamed: ${totalRenamed}`); + + console.info('Dropping supporting index...'); + await queryInterface.sequelize.query(` + DROP INDEX CONCURRENTLY IF EXISTS folders_parentuuid_plainname_not_deleted_support_index; + `); }, - down: async () => {}, + down: async (queryInterface) => { + await queryInterface.sequelize.query(` + DROP INDEX CONCURRENTLY IF EXISTS folders_parentuuid_plainname_not_deleted_support_index; + `); + }, }; From 21b7c8cc781ca6fd68f45e88ffdb060d7d377d88 Mon Sep 17 00:00:00 2001 From: jzunigax2 <125698953+jzunigax2@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:08:36 -0600 Subject: [PATCH 3/3] chore: reduce batch size in duplicate folder cleanup migration for improved performance --- migrations/20260205010629-cleanup-duplicate-folders.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/20260205010629-cleanup-duplicate-folders.js b/migrations/20260205010629-cleanup-duplicate-folders.js index 72410c447..1e13d3e4e 100644 --- a/migrations/20260205010629-cleanup-duplicate-folders.js +++ b/migrations/20260205010629-cleanup-duplicate-folders.js @@ -2,7 +2,7 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); const MAX_ATTEMPTS = 10; -const BATCH_SIZE = 500; +const BATCH_SIZE = 10; const SLEEP_TIME_MS = 1000; module.exports = {