From 700db2301b0b7cc4571555f056e0354c4cdebd0b Mon Sep 17 00:00:00 2001 From: cballevre Date: Mon, 1 Jul 2024 16:41:21 +0200 Subject: [PATCH] feat: Improve index naming --- .../src/DocumentCollection.js | 69 ++++- packages/cozy-stack-client/src/mangoIndex.js | 97 +++++++ .../cozy-stack-client/src/mangoIndex.spec.js | 238 +++++++++++++++++- 3 files changed, 392 insertions(+), 12 deletions(-) diff --git a/packages/cozy-stack-client/src/DocumentCollection.js b/packages/cozy-stack-client/src/DocumentCollection.js index 27e7814d40..68420f4b1d 100644 --- a/packages/cozy-stack-client/src/DocumentCollection.js +++ b/packages/cozy-stack-client/src/DocumentCollection.js @@ -26,7 +26,8 @@ import { transformSort, getIndexFields, isMatchingIndex, - normalizeDesignDoc + normalizeDesignDoc, + getNewIndexName } from './mangoIndex' import * as querystring from './querystring' import { FetchError } from './errors' @@ -219,6 +220,19 @@ class DocumentCollection { } } + async migrateOldNamedIndex( + indexedFields, + partialFilter, + existingIndex, + indexName + ) { + await this.destroyIndex(existingIndex) + await this.createIndex(indexedFields, { + partialFilter, + indexName + }) + } + /** * Handle index creation if it is missing. * @@ -240,15 +254,25 @@ class DocumentCollection { ? getIndexFields({ partialFilter }) : null - const existingIndex = await this.findExistingIndex(selector, options) - const indexName = getIndexNameFromFields(indexedFields, { + const oldName = `_design/${getIndexNameFromFields(indexedFields, { partialFilterFields - }) + })}` + + const existingIndex = await this.findExistingIndex(selector, options) + + const indexName = getNewIndexName(selector, { partialFilter }) if (!existingIndex) { await this.createIndex(indexedFields, { partialFilter, indexName }) + } else if (existingIndex._id === oldName) { + await this.migrateOldNamedIndex( + indexedFields, + partialFilter, + existingIndex, + indexName + ) } else if (existingIndex._id !== `_design/${indexName}`) { await this.migrateUnamedIndex(existingIndex, indexName) } else { @@ -493,7 +517,7 @@ The returned documents are paginated by the stack. * @returns {MangoQueryOptions} Mango options */ toMangoOptions(selector, options = {}) { - let { sort, indexedFields, partialFilter } = options + let { sort, indexedFields, partialFilter, partialIndex } = options const { fields, skip = 0, limit, bookmark } = options sort = transformSort(sort) @@ -508,14 +532,14 @@ The returned documents are paginated by the stack. ? indexedFields : getIndexFields({ sort, selector }) - const partialFilterFields = partialFilter - ? getIndexFields({ partialFilter }) - : null const indexName = options.indexId || - `_design/${getIndexNameFromFields(indexedFields, { - partialFilterFields + `_design/${getNewIndexName(selector, { + partialFilter })}` + + console.log('indexName', indexName) + if (sort) { const sortOrders = uniq( sort.map(sortOption => head(Object.values(sortOption))) @@ -714,6 +738,7 @@ The returned documents are paginated by the stack. async findExistingIndex(selector, options) { let { sort, indexedFields, partialFilter } = options const indexes = await this.fetchAllMangoIndexes() + console.log('indexes', indexes) if (indexes.length < 1) { return null } @@ -725,7 +750,29 @@ The returned documents are paginated by the stack. const existingIndex = indexes.find(index => { return isMatchingIndex(index, fieldsToIndex, partialFilter) }) - return existingIndex + + if (existingIndex) { + return existingIndex + } + + const partialFilterFields = partialFilter + ? getIndexFields({ partialFilter }) + : null + + const oldName = `_design/${getIndexNameFromFields(indexedFields, { + partialFilterFields + })}` + + const existingIndexWithOldName = indexes.find( + index => index._id === oldName + ) + + if (existingIndexWithOldName) { + console.log('existingIndexWithOldName', existingIndexWithOldName) + return existingIndexWithOldName + } + + return null } /** diff --git a/packages/cozy-stack-client/src/mangoIndex.js b/packages/cozy-stack-client/src/mangoIndex.js index 1b16f6ca7c..70d165def5 100644 --- a/packages/cozy-stack-client/src/mangoIndex.js +++ b/packages/cozy-stack-client/src/mangoIndex.js @@ -46,6 +46,102 @@ export const normalizeDesignDoc = designDoc => { return { id, _id: id, ...designDoc.doc } } +/** + * Process a condition to generate a string key + * + * @param {object} condition - An object representing tcondition + * @param {number} [depth] - Level of recursion + * @returns {string} - The string key of the processed condition + */ +export const processCondition = (condition, depth = 0) => { + if (condition.$or && depth < 3) { + return `(${condition.$or + .map(subCondition => processCondition(subCondition, depth + 1)) + .join('_or_')})` + } else if (condition.$and && depth < 3) { + return `(${condition.$and + .map(subCondition => processCondition(subCondition, depth + 1)) + .join('_and_')})` + } else { + return Object.keys(condition) + .map(key => (key.startsWith('$') ? key.slice(1) : key)) + .join('_and_') + } +} + +/** + * Process a selector to generate a string key + * + * @example + * // returns `field1_and_(field2_or_field3)_and_field4` + * processCondition({ + * field1: 'value1', + * $or: [{ field2: 'value2' }, { field3: 'value3' }], + * field4: 'value4' + * }); + * + * @param {object} selector - An object representing the selector + * @returns {string} - The string key of the processed selector + */ +export const processSelector = selector => { + const conditions = Object.entries(selector).map(([key, value]) => + processCondition({ [key]: value }) + ) + return conditions.join('_and_') +} + +/** + * Flatten an object + * + * @param {*} obj - The object to flatten + * @param {*} parent - The parent key + * @param {*} res - The result object + * @returns {object} - And object with only one level of depth + */ +export const flattenObject = (obj, parent = '', res = {}) => { + Object.entries(obj).forEach(([key, value]) => { + const cleanedKey = key.startsWith('$') ? key.slice(1) : key + const propName = parent ? `${parent}_${cleanedKey}` : cleanedKey + if (typeof value === 'object' && value !== null && !Array.isArray(value)) { + flattenObject(value, propName, res) + } else { + res[propName] = value + } + }) + return res +} + +/** + * Process a partial filter to generate a string key + * + * @param {object} partialFilter - An object representing the partial filter + * @returns {string} - The string key of the processed partial filter + */ +export const processPartialFilter = partialFilter => { + const flatPartialFilter = flattenObject(partialFilter) + return `(${Object.keys(flatPartialFilter) + .map(key => { + if (Array.isArray(flatPartialFilter[key])) { + return `${key}_${flatPartialFilter[key].join('_')}` + } else { + return `${key}_${flatPartialFilter[key]}` + } + }) + .join(')_and_(')})` +} + +export const getNewIndexName = (selector, options = {}) => { + let baseName = processSelector(selector) + + if (options.partialFilter) { + return `by_${baseName}_filter_${processPartialFilter( + options.partialFilter + )}` + } + + return `by_${baseName}` +} + /** * Name an index, based on its indexed fields and partial filter. * @@ -66,6 +162,7 @@ export const getIndexNameFromFields = ( ? `${indexName}_filter_${partialFilterFields.join('_and_')}` : indexName } + /** * Transform sort into Array * diff --git a/packages/cozy-stack-client/src/mangoIndex.spec.js b/packages/cozy-stack-client/src/mangoIndex.spec.js index 087d03008b..c7837c1620 100644 --- a/packages/cozy-stack-client/src/mangoIndex.spec.js +++ b/packages/cozy-stack-client/src/mangoIndex.spec.js @@ -1,7 +1,10 @@ import { isMatchingIndex, getIndexFields, - getIndexNameFromFields + getIndexNameFromFields, + getNewIndexName, + processCondition, + flattenObject } from './mangoIndex' const buildDesignDoc = (fields, { partialFilter, id } = {}) => { @@ -175,3 +178,236 @@ describe('getIndexNameFromFields', () => { ) }) }) + +describe('getNewIndexName', () => { + it('should return index for simple selector', () => { + const selector = { + dir_id: 'id123', + type: { $gt: null }, + name: { $gt: null } + } + + expect(getNewIndexName(selector)).toEqual('by_dir_id_and_type_and_name') + }) + it('should return index fields', () => { + const selector = { + $and: [ + { + $or: [ + { + state: { + $eq: 'running' + } + }, + { + worker: { + $eq: 'konnector' + } + }, + { + foo: { $ne: 'bar' } + } + ] + }, + { + $or: [ + { + foo: { + $eq: 'running' + } + }, + { + bar: { + $eq: 'konnector' + } + } + ] + } + ] + } + + expect(getNewIndexName(selector)).toEqual( + 'by_((state_or_worker_or_foo)_and_(foo_or_bar))' + ) + }) + + it('should return index fields with partial filter', () => { + const selector = { + dir_id: 'id123', + type: { $gt: null }, + name: { $gt: null } + } + + const partialFilter = { + _id: { + $nin: ['id456', 'id789'] + }, + trashed: { + $ne: true + } + } + + expect(getNewIndexName(selector, { partialFilter })).toEqual( + 'by_dir_id_and_type_and_name_filter_(_id_nin_id456_id789)_and_(trashed_ne_true)' + ) + }) + + it('should return index fields with partial filter', () => { + const selector = { + dir_id: 'id123', + type: { $gt: null }, + name: { $gt: null }, + $or: [ + { + foo: { + $eq: 'bar' + } + }, + { + bar: { + $eq: 'foo' + } + } + ] + } + + const partialFilter = { + _id: { + $nin: ['id456'] + }, + trashed: { + $ne: true + } + } + + expect(getNewIndexName(selector, { partialFilter })).toEqual( + 'by_dir_id_and_type_and_name_and_(foo_or_bar)_filter_(_id_nin_id456)_and_(trashed_ne_true)' + ) + }) + + it('should return index name for 3 level deep selector', () => { + const selector = { + $and: [ + { + $or: [ + { + $and: [ + { + state: { + $eq: 'running' + } + }, + { + worker: { + $eq: 'konnector' + } + } + ] + }, + { + foo: { $ne: 'bar' } + } + ] + } + ] + } + + expect(getNewIndexName(selector)).toEqual( + 'by_(((state_and_worker)_or_foo))' + ) + }) +}) + +describe('processCondition', () => { + it('should process a simple condition', () => { + const condition = { field1: 'value1' } + const processedCondition = processCondition(condition) + expect(processedCondition).toEqual('field1') + }) + + it('should process a condition with $or operator', () => { + const condition = { + $or: [{ field1: 'value1' }, { field2: 'value2' }, { field3: 'value3' }] + } + const processedCondition = processCondition(condition) + expect(processedCondition).toEqual('(field1_or_field2_or_field3)') + }) + + it('should process a condition with $and operator', () => { + const condition = { + $and: [{ field1: 'value1' }, { field2: 'value2' }, { field3: 'value3' }] + } + const processedCondition = processCondition(condition) + expect(processedCondition).toEqual('(field1_and_field2_and_field3)') + }) + + it('should process a deeply nested condition', () => { + const condition = { + $and: [ + { + $or: [ + { + $and: [ + { field1: 'value1' }, + { $or: [{ field2: 'value2' }, { field3: 'value3' }] } + ] + }, + { field4: 'value4' } + ] + } + ] + } + const processedCondition = processCondition(condition) + expect(processedCondition).toEqual('(((field1_and_or)_or_field4))') + }) +}) + +describe('flattenObject', () => { + it('should flatten an object with nested properties', () => { + const obj = { + foo: 'bar', + nested1: { + prop1: 'value1', + prop2: 'value2' + } + } + const flattenedObj = flattenObject(obj) + expect(flattenedObj).toEqual({ + foo: 'bar', + nested1_prop1: 'value1', + nested1_prop2: 'value2' + }) + }) + + it('should flatten an object with arrays', () => { + const obj = { + arr: [1, 2, 3], + nested: { + arr: ['a', 'b', 'c'] + } + } + const flattenedObj = flattenObject(obj) + expect(flattenedObj).toEqual({ + arr: [1, 2, 3], + nested_arr: ['a', 'b', 'c'] + }) + }) + + it('should flatten an object with null and undefined values', () => { + const obj = { + foo: null, + bar: undefined, + nested: { + prop1: null, + prop2: undefined + } + } + const flattenedObj = flattenObject(obj) + expect(flattenedObj).toEqual({ + foo: null, + bar: undefined, + nested_prop1: null, + nested_prop2: undefined + }) + }) +})