From 5da0d55ab8cce0a77c881da4a7745f006c6c1465 Mon Sep 17 00:00:00 2001 From: cballevre Date: Thu, 4 Jul 2024 09:19:00 +0200 Subject: [PATCH] feat: Remove the implicit inside partialFilter to find existing index When CouchDB creates an index with a partialFilter, it adds explicit operators when they are implicit, typically the $and and $eq operators. This causes a mismatch when comparing the partialFilter from the request definition with the partialFilter from the index. To address this, we added a step to make the operators in the partialFilter from the request definition explicit. This makes it possible to migrate indexes after they have been renamed rather than re-creating them, which has a non-negligible cost. --- docs/api/cozy-stack-client.md | 33 ++++ packages/cozy-stack-client/src/mangoIndex.js | 67 ++++++- .../cozy-stack-client/src/mangoIndex.spec.js | 163 +++++++++++++++++- 3 files changed, 261 insertions(+), 2 deletions(-) diff --git a/docs/api/cozy-stack-client.md b/docs/api/cozy-stack-client.md index 6d1f8a6665..0faf81fadf 100644 --- a/docs/api/cozy-stack-client.md +++ b/docs/api/cozy-stack-client.md @@ -101,6 +101,9 @@ query to work

isMatchingIndex ⇒ boolean

Check if an index is matching the given fields

+
makeOperatorsExplicit ⇒ object
+

Transform a query to make all operators explicit

+
getPermissionsFor ⇒ object

Build a permission set

@@ -132,6 +135,10 @@ See handleNorOperator(conditions) ⇒ Array +

Handle the $nor operator in a query +CouchDB transforms $nor into $and with $ne operators

+
garbageCollect()

Delete outdated results from cache

@@ -2224,6 +2231,19 @@ Check if an index is matching the given fields | fields | Array | The fields that the index must have | | partialFilter | object | An optional partial filter | + + +## makeOperatorsExplicit ⇒ object +Transform a query to make all operators explicit + +**Kind**: global constant +**Returns**: object - - The transformed query with all operators explicit + +| Param | Type | Description | +| --- | --- | --- | +| query | object | The query to transform | +| reverseEq | boolean | If true, $eq will be transformed to $ne (useful for manage $nor) | + ## getPermissionsFor ⇒ object @@ -2312,6 +2332,19 @@ Get Icon URL using blob mechanism if OAuth connected or using preloaded url when blob not needed **Kind**: global function + + +## handleNorOperator(conditions) ⇒ Array +Handle the $nor operator in a query +CouchDB transforms $nor into $and with $ne operators + +**Kind**: global function +**Returns**: Array - - The reversed conditions + +| Param | Type | Description | +| --- | --- | --- | +| conditions | Array | The conditions inside the $nor operator | + ## garbageCollect() diff --git a/packages/cozy-stack-client/src/mangoIndex.js b/packages/cozy-stack-client/src/mangoIndex.js index 38661395ef..38d662c6cb 100644 --- a/packages/cozy-stack-client/src/mangoIndex.js +++ b/packages/cozy-stack-client/src/mangoIndex.js @@ -1,6 +1,7 @@ import head from 'lodash/head' import get from 'lodash/get' import isEqual from 'lodash/isEqual' +import isObject from 'lodash/isObject' /** * @typedef {Object} MangoPartialFilter @@ -148,9 +149,73 @@ export const isMatchingIndex = (index, fields, partialFilter) => { if (!partialFilter && !partialFilterInIndex) { return true } - if (isEqual(partialFilter, partialFilterInIndex)) { + + const explicitPartialFilter = makeOperatorsExplicit(partialFilter ?? {}) + if (isEqual(explicitPartialFilter, partialFilterInIndex)) { return true } } + return false } + +/** + * Handle the $nor operator in a query + * CouchDB transforms $nor into $and with $ne operators + * + * @param {Array} conditions - The conditions inside the $nor operator + * @returns {Array} - The reversed conditions + */ +const handleNorOperator = conditions => { + return conditions.map(condition => + Object.entries(condition).reduce((acc, [key, value]) => { + if (typeof value === 'string') { + acc[key] = { $ne: value } + } else { + acc[key] = makeOperatorsExplicit(value, true) + } + return acc + }, {}) + ) +} + +/** + * Transform a query to make all operators explicit + * + * @param {object} query - The query to transform + * @param {boolean} reverseEq - If true, $eq will be transformed to $ne (useful for manage $nor) + * @returns {object} - The transformed query with all operators explicit + */ +export const makeOperatorsExplicit = (query, reverseEq = false) => { + const explicitQuery = Object.entries(query).reduce((acc, [key, value]) => { + if (key === '$nor') { + acc['$and'] = handleNorOperator(value) + } else if (value['$or']?.every(v => typeof v === 'string')) { + acc['$or'] = value['$or'].map(v => + makeOperatorsExplicit({ [key]: v }, reverseEq) + ) // To manage $or with list of strings + } else if (Array.isArray(value) && value.every(isObject)) { + acc[key] = value.map(v => makeOperatorsExplicit(v, reverseEq)) // To manage $and and $or with multiple conditions inside + } else if (isObject(value) && !Array.isArray(value)) { + acc[key] = makeOperatorsExplicit(value, reverseEq) // To manage nested objects + } else if (reverseEq && key === '$eq') { + acc['$ne'] = value + } else if (!key.startsWith('$')) { + acc[key] = { $eq: value } // To manage implicit $eq + } else { + acc[key] = value // To manage explicit operators + } + return acc + }, {}) + + const explicitQueryKeys = Object.keys(explicitQuery) + if (explicitQueryKeys.length === 1) { + return explicitQuery + } + + return { + $and: explicitQueryKeys.map(key => ({ + [key]: explicitQuery[key] + })) + } +} diff --git a/packages/cozy-stack-client/src/mangoIndex.spec.js b/packages/cozy-stack-client/src/mangoIndex.spec.js index 4a7458f3e4..e60ba422fa 100644 --- a/packages/cozy-stack-client/src/mangoIndex.spec.js +++ b/packages/cozy-stack-client/src/mangoIndex.spec.js @@ -1,7 +1,8 @@ import { isMatchingIndex, getIndexFields, - getIndexNameFromFields + getIndexNameFromFields, + makeOperatorsExplicit } from './mangoIndex' const buildDesignDoc = (fields, { partialFilter, id } = {}) => { @@ -262,3 +263,163 @@ describe('getIndexNameFromFields', () => { ) }) }) + +describe('makeOperatorsExplicit', () => { + it('Transforms implicit $eq operator to explicit', () => { + const query = { name: 'test' } + const expected = { name: { $eq: 'test' } } + expect(makeOperatorsExplicit(query)).toEqual(expected) + }) + + it('Transforms implicit $and operator to explicit', () => { + const query = { name: 'test', age: 42 } + const expected = { $and: [{ name: { $eq: 'test' } }, { age: { $eq: 42 } }] } + expect(makeOperatorsExplicit(query)).toEqual(expected) + }) + + it('Maintains explicit $eq operator', () => { + const query = { name: { $eq: 'test' } } + const expected = { name: { $eq: 'test' } } + expect(makeOperatorsExplicit(query)).toEqual(expected) + }) + + it('Maintains explicit $and operator', () => { + const query = { $and: [{ name: 'test' }, { age: 42 }] } + const expected = { $and: [{ name: { $eq: 'test' } }, { age: { $eq: 42 } }] } + expect(makeOperatorsExplicit(query)).toEqual(expected) + }) + + it('Handles nested implicit $eq operators', () => { + const query = { user: { name: 'test', age: 42 } } + const expected = { + user: { $and: [{ name: { $eq: 'test' } }, { age: { $eq: 42 } }] } + } + expect(makeOperatorsExplicit(query)).toEqual(expected) + }) + + it('Handles nested explicit operators', () => { + const query = { user: { $or: [{ name: 'test' }, { age: { $ne: 42 } }] } } + const expected = { + user: { $or: [{ name: { $eq: 'test' } }, { age: { $ne: 42 } }] } + } + expect(makeOperatorsExplicit(query)).toEqual(expected) + }) + + it('Handles mixed implicit and explicit operators', () => { + const query = { name: 'test', age: { $ne: 42 } } + const expected = { $and: [{ name: { $eq: 'test' } }, { age: { $ne: 42 } }] } + expect(makeOperatorsExplicit(query)).toEqual(expected) + }) + + it('Handles operator with string array', () => { + const query = { + _id: { + $nin: ['id123', 'id456'] + }, + type: 'file' + } + const expected = { + $and: [ + { + _id: { + $nin: ['id123', 'id456'] + } + }, + { type: { $eq: 'file' } } + ] + } + expect(makeOperatorsExplicit(query)).toEqual(expected) + }) + + it('Handles $or operator with string array', () => { + const query = { + type: { + $or: ['konnector', 'worker'] + } + } + const expected = { + $or: [{ type: { $eq: 'konnector' } }, { type: { $eq: 'worker' } }] + } + expect(makeOperatorsExplicit(query)).toEqual(expected) + }) + + it('Handles $or operator with object array', () => { + const query = { + type: 'file', + $or: [ + { + trashed: { + $exists: false + } + }, + { + trashed: false + } + ] + } + const expected = { + $and: [ + { type: { $eq: 'file' } }, + { $or: [{ trashed: { $exists: false } }, { trashed: { $eq: false } }] } + ] + } + expect(makeOperatorsExplicit(query)).toEqual(expected) + }) + + it('Handles explicit $and operator with nested object to make explicit', () => { + const query = { + type: 'file', + trashed: true, + 'metadata.notifiedAt': { + $exists: false + } + } + const expected = { + $and: [ + { type: { $eq: 'file' } }, + { trashed: { $eq: true } }, + { 'metadata.notifiedAt': { $exists: false } } + ] + } + expect(makeOperatorsExplicit(query)).toEqual(expected) + }) + + it('Handles explicit $nor operator', () => { + const query = { + $nor: [ + { + type: { + $eq: 'directory' + } + }, + { dir_id: 'id1234' }, + { + 'metadata.notifiedAt': { + $exists: false + } + } + ] + } + const expected = { + $and: [ + { + type: { + $ne: 'directory' + } + }, + { + dir_id: { + $ne: 'id1234' + } + }, + { + 'metadata.notifiedAt': { + $exists: false + } + } + ] + } + + expect(makeOperatorsExplicit(query)).toEqual(expected) + }) +})