From 6fe85a7f23d06a79ad30989d07c03195931ffa0c Mon Sep 17 00:00:00 2001 From: Ldoppea Date: Thu, 23 May 2024 18:26:43 +0200 Subject: [PATCH] feat: Inject Pouch engine and local storage --- packages/cozy-pouch-link/src/CozyPouchLink.js | 32 ++++--- packages/cozy-pouch-link/src/PouchManager.js | 73 +++++++++------- packages/cozy-pouch-link/src/localStorage.js | 87 ++++++++++++------- packages/cozy-pouch-link/src/platformWeb.js | 7 ++ .../cozy-pouch-link/src/startReplication.js | 22 ++--- packages/cozy-pouch-link/src/types.js | 32 +++++++ 6 files changed, 165 insertions(+), 88 deletions(-) create mode 100644 packages/cozy-pouch-link/src/platformWeb.js create mode 100644 packages/cozy-pouch-link/src/types.js diff --git a/packages/cozy-pouch-link/src/CozyPouchLink.js b/packages/cozy-pouch-link/src/CozyPouchLink.js index 373ff776f9..07b11bf2ca 100644 --- a/packages/cozy-pouch-link/src/CozyPouchLink.js +++ b/packages/cozy-pouch-link/src/CozyPouchLink.js @@ -18,13 +18,8 @@ import * as jsonapi from './jsonapi' import PouchManager from './PouchManager' import logger from './logger' import { migratePouch } from './migrations/adapter' +import { platformWeb } from './platformWeb' import { getDatabaseName, getPrefix } from './utils' -import { - getPersistedSyncedDoctypes, - persistAdapterName, - getAdapterName, - destroyWarmedUpQueries -} from './localStorage' PouchDB.plugin(PouchDBFind) @@ -95,6 +90,7 @@ class PouchLink extends CozyLink { this.doctypes = doctypes this.doctypesReplicationOptions = doctypesReplicationOptions this.indexes = {} + this.storage = options.platform?.storage || platformWeb.storage /** @type {Record} - Stores replication states per doctype */ this.replicationStatus = this.replicationStatus || {} @@ -149,15 +145,15 @@ class PouchLink extends CozyLink { for (const plugin of plugins) { PouchDB.plugin(plugin) } - const doctypes = getPersistedSyncedDoctypes() + const doctypes = await this.storage.getPersistedSyncedDoctypes() for (const doctype of Object.keys(doctypes)) { const prefix = getPrefix(url) const dbName = getDatabaseName(prefix, doctype) await migratePouch({ dbName, fromAdapter, toAdapter }) - destroyWarmedUpQueries() // force recomputing indexes + await this.storage.destroyWarmedUpQueries() // force recomputing indexes } - persistAdapterName('indexeddb') + await this.storage.persistAdapterName('indexeddb') } catch (err) { console.error('PouchLink: PouchDB migration failed. ', err) } @@ -195,9 +191,9 @@ class PouchLink extends CozyLink { logger.log('Create pouches with ' + prefix + ' prefix') } - if (!getAdapterName()) { + if (!(await this.storage.getAdapterName())) { const adapter = get(this.options, 'pouch.options.adapter') - persistAdapterName(adapter) + await this.storage.persistAdapterName(adapter) } this.pouches = new PouchManager(this.doctypes, { @@ -209,8 +205,10 @@ class PouchLink extends CozyLink { onDoctypeSyncStart: this.handleDoctypeSyncStart.bind(this), onDoctypeSyncEnd: this.handleDoctypeSyncEnd.bind(this), prefix, - executeQuery: this.executeQuery.bind(this) + executeQuery: this.executeQuery.bind(this), + platform: this.options.platform }) + await this.pouches.init() if (this.client && this.options.initialSync) { this.startReplication() @@ -334,7 +332,7 @@ class PouchLink extends CozyLink { return !!this.getPouch(impactedDoctype) } - request(operation, result = null, forward = doNothing) { + async request(operation, result = null, forward = doNothing) { const doctype = getDoctypeFromOperation(operation) if (!this.pouches) { @@ -387,18 +385,18 @@ class PouchLink extends CozyLink { * and return if those queries are already warmed up or not * * @param {string} doctype - Doctype to check - * @returns {boolean} the need to wait for the warmup + * @returns {Promise} the need to wait for the warmup */ - needsToWaitWarmup(doctype) { + async needsToWaitWarmup(doctype) { if ( this.doctypesReplicationOptions && this.doctypesReplicationOptions[doctype] && this.doctypesReplicationOptions[doctype].warmupQueries ) { - return !this.pouches.areQueriesWarmedUp( + return !(await this.pouches.areQueriesWarmedUp( doctype, this.doctypesReplicationOptions[doctype].warmupQueries - ) + )) } return false } diff --git a/packages/cozy-pouch-link/src/PouchManager.js b/packages/cozy-pouch-link/src/PouchManager.js index 4c04ae4e29..ac52ab8c1e 100644 --- a/packages/cozy-pouch-link/src/PouchManager.js +++ b/packages/cozy-pouch-link/src/PouchManager.js @@ -1,4 +1,3 @@ -import PouchDB from 'pouchdb-browser' import fromPairs from 'lodash/fromPairs' import forEach from 'lodash/forEach' import get from 'lodash/get' @@ -10,9 +9,9 @@ import { QueryDefinition } from 'cozy-client' import Loop from './loop' import logger from './logger' +import { platformWeb } from './platformWeb' import { fetchRemoteLastSequence } from './remote' import { startReplication } from './startReplication' -import * as localStorage from './localStorage' import { getDatabaseName } from './utils' const DEFAULT_DELAY = 30 * 1000 @@ -35,20 +34,33 @@ const getQueryAlias = query => { class PouchManager { constructor(doctypes, options) { this.options = options - const pouchPlugins = get(options, 'pouch.plugins', []) - const pouchOptions = get(options, 'pouch.options', {}) + this.doctypes = doctypes - forEach(pouchPlugins, plugin => PouchDB.plugin(plugin)) + /** + * @type {import("./types").PouchLocalStorage} + */ + this.storage = options.platform?.storage || platformWeb.storage + this.PouchDB = options.platform?.pouchEngine || platformWeb.pouchEngine + } + + async init() { + const pouchPlugins = get(this.options, 'pouch.plugins', []) + const pouchOptions = get(this.options, 'pouch.options', {}) + + forEach(pouchPlugins, plugin => this.PouchDB.plugin(plugin)) this.pouches = fromPairs( - doctypes.map(doctype => [ + this.doctypes.map(doctype => [ doctype, - new PouchDB(getDatabaseName(options.prefix, doctype), pouchOptions) + new this.PouchDB( + getDatabaseName(this.options.prefix, doctype), + pouchOptions + ) ]) ) - this.syncedDoctypes = localStorage.getPersistedSyncedDoctypes() - this.warmedUpQueries = localStorage.getPersistedWarmedUpQueries() - this.getReplicationURL = options.getReplicationURL - this.doctypesReplicationOptions = options.doctypesReplicationOptions || {} + this.syncedDoctypes = await this.storage.getPersistedSyncedDoctypes() + this.warmedUpQueries = await this.storage.getPersistedWarmedUpQueries() + this.getReplicationURL = this.options.getReplicationURL + this.doctypesReplicationOptions = this.options.doctypesReplicationOptions || {} this.listenerLaunched = false // We must ensure databases exist on the remote before @@ -85,13 +97,13 @@ class PouchManager { } } - destroy() { + async destroy() { this.stopReplicationLoop() this.removeListeners() - this.clearSyncedDoctypes() - this.clearWarmedUpQueries() - localStorage.destroyAllDoctypeLastSequence() - localStorage.destroyAllLastReplicatedDocID() + await this.clearSyncedDoctypes() + await this.clearWarmedUpQueries() + await this.storage.destroyAllDoctypeLastSequence() + await this.storage.destroyAllLastReplicatedDocID() return Promise.all( Object.values(this.pouches).map(pouch => pouch.destroy()) @@ -182,9 +194,9 @@ class PouchManager { // Before the first replication, get the last remote sequence, // which will be used as a checkpoint for the next replication const lastSeq = await fetchRemoteLastSequence(getReplicationURL()) - localStorage.persistDoctypeLastSequence(doctype, lastSeq) + await this.storage.persistDoctypeLastSequence(doctype, lastSeq) } else { - seq = localStorage.getDoctypeLastSequence(doctype) + seq = await this.storage.getDoctypeLastSequence(doctype) } const replicationOptions = get( @@ -203,15 +215,16 @@ class PouchManager { const res = await startReplication( pouch, replicationOptions, - getReplicationURL + getReplicationURL, + this.storage ) if (seq) { // We only need the sequence for the second replication, as PouchDB // will use a local checkpoint for the next runs. - localStorage.destroyDoctypeLastSequence(doctype) + await this.storage.destroyDoctypeLastSequence(doctype) } - this.updateSyncInfo(doctype) + await this.updateSyncInfo(doctype) this.checkToWarmupDoctype(doctype, replicationOptions) if (this.options.onDoctypeSyncEnd) { this.options.onDoctypeSyncEnd(doctype) @@ -273,9 +286,9 @@ class PouchManager { return this.pouches[doctype] } - updateSyncInfo(doctype) { + async updateSyncInfo(doctype) { this.syncedDoctypes[doctype] = { date: new Date().toISOString() } - localStorage.persistSyncedDoctypes(this.syncedDoctypes) + await this.storage.persistSyncedDoctypes(this.syncedDoctypes) } getSyncInfo(doctype) { @@ -287,9 +300,9 @@ class PouchManager { return info ? !!info.date : false } - clearSyncedDoctypes() { + async clearSyncedDoctypes() { this.syncedDoctypes = {} - localStorage.destroySyncedDoctypes() + await this.storage.destroySyncedDoctypes() } async warmupQueries(doctype, queries) { @@ -304,7 +317,7 @@ class PouchManager { } }) ) - localStorage.persistWarmedUpQueries(this.warmedUpQueries) + await this.storage.persistWarmedUpQueries(this.warmedUpQueries) logger.log('PouchManager: warmupQueries for ' + doctype + ' are done') } catch (err) { logger.error( @@ -324,8 +337,8 @@ class PouchManager { } } - areQueriesWarmedUp(doctype, queries) { - const persistWarmedUpQueries = localStorage.getPersistedWarmedUpQueries() + async areQueriesWarmedUp(doctype, queries) { + const persistWarmedUpQueries = await this.storage.getPersistedWarmedUpQueries() return queries.every( query => persistWarmedUpQueries[doctype] && @@ -333,9 +346,9 @@ class PouchManager { ) } - clearWarmedUpQueries() { + async clearWarmedUpQueries() { this.warmedUpQueries = {} - localStorage.destroyWarmedUpQueries() + await this.storage.destroyWarmedUpQueries() } } diff --git a/packages/cozy-pouch-link/src/localStorage.js b/packages/cozy-pouch-link/src/localStorage.js index ad05e48d6a..9aa266a511 100644 --- a/packages/cozy-pouch-link/src/localStorage.js +++ b/packages/cozy-pouch-link/src/localStorage.js @@ -12,9 +12,11 @@ export const LOCALSTORAGE_ADAPTERNAME = 'cozy-client-pouch-link-adaptername' * * @param {string} doctype - The replicated doctype * @param {string} id - The docid + * + * @returns {Promise} */ -export const persistLastReplicatedDocID = (doctype, id) => { - const docids = getAllLastReplicatedDocID() +export const persistLastReplicatedDocID = async (doctype, id) => { + const docids = await getAllLastReplicatedDocID() docids[doctype] = id window.localStorage.setItem( @@ -23,7 +25,10 @@ export const persistLastReplicatedDocID = (doctype, id) => { ) } -export const getAllLastReplicatedDocID = () => { +/** + * @returns {Promise>} + */ +export const getAllLastReplicatedDocID = async () => { const item = window.localStorage.getItem(LOCALSTORAGE_LASTREPLICATEDDOCID_KEY) return item ? JSON.parse(item) : {} } @@ -32,29 +37,30 @@ export const getAllLastReplicatedDocID = () => { * Get the last replicated doc id for a doctype * * @param {string} doctype - The doctype - * @returns {string} The last replicated docid + * @returns {Promise} The last replicated docid */ -export const getLastReplicatedDocID = doctype => { - const docids = getAllLastSequences() +export const getLastReplicatedDocID = async doctype => { + const docids = await getAllLastSequences() return docids[doctype] } /** * Destroy all the replicated doc id + * + * @returns {Promise} */ -export const destroyAllLastReplicatedDocID = () => { +export const destroyAllLastReplicatedDocID = async () => { window.localStorage.removeItem(LOCALSTORAGE_LASTREPLICATEDDOCID_KEY) } /** * Persist the synchronized doctypes * - * @typedef {object} SyncInfo - * @property {string} Date + * @param {Record} syncedDoctypes - The sync doctypes * - * @param {Record} syncedDoctypes - The sync doctypes + * @returns {Promise} */ -export const persistSyncedDoctypes = syncedDoctypes => { +export const persistSyncedDoctypes = async syncedDoctypes => { window.localStorage.setItem( LOCALSTORAGE_SYNCED_KEY, JSON.stringify(syncedDoctypes) @@ -64,22 +70,28 @@ export const persistSyncedDoctypes = syncedDoctypes => { /** * Get the persisted doctypes * - * @returns {object} The synced doctypes + * @returns {Promise} The synced doctypes */ -export const getPersistedSyncedDoctypes = () => { +export const getPersistedSyncedDoctypes = async () => { + console.log('🟣 getPersistedSyncedDoctypes 1b') const item = window.localStorage.getItem(LOCALSTORAGE_SYNCED_KEY) + console.log('🟣 getPersistedSyncedDoctypes 2', item) const parsed = item ? JSON.parse(item) : {} + console.log('🟣 getPersistedSyncedDoctypes 3') if (typeof parsed !== 'object') { + console.log('🟣 getPersistedSyncedDoctypes 4') return {} } + console.log('🟣 getPersistedSyncedDoctypes 5') return parsed } /** * Destroy the synced doctypes * + * @returns {Promise} */ -export const destroySyncedDoctypes = () => { +export const destroySyncedDoctypes = async () => { window.localStorage.removeItem(LOCALSTORAGE_SYNCED_KEY) } @@ -88,9 +100,11 @@ export const destroySyncedDoctypes = () => { * * @param {string} doctype - The synced doctype * @param {string} sequence - The sequence hash + * + * @returns {Promise} */ -export const persistDoctypeLastSequence = (doctype, sequence) => { - const seqs = getAllLastSequences() +export const persistDoctypeLastSequence = async (doctype, sequence) => { + const seqs = await getAllLastSequences() seqs[doctype] = sequence window.localStorage.setItem( @@ -99,7 +113,10 @@ export const persistDoctypeLastSequence = (doctype, sequence) => { ) } -export const getAllLastSequences = () => { +/** + * @returns {Promise} + */ +export const getAllLastSequences = async () => { const item = window.localStorage.getItem(LOCALSTORAGE_LASTSEQUENCES_KEY) return item ? JSON.parse(item) : {} } @@ -108,17 +125,20 @@ export const getAllLastSequences = () => { * Get the last CouchDB sequence for a doctype * * @param {string} doctype - The doctype - * @returns {string} the last sequence + * + * @returns {Promise} the last sequence */ -export const getDoctypeLastSequence = doctype => { - const seqs = getAllLastSequences() +export const getDoctypeLastSequence = async doctype => { + const seqs = await getAllLastSequences() return seqs[doctype] } /** * Destroy all the last sequence + * + * @returns {Promise} */ -export const destroyAllDoctypeLastSequence = () => { +export const destroyAllDoctypeLastSequence = async () => { window.localStorage.removeItem(LOCALSTORAGE_LASTSEQUENCES_KEY) } @@ -126,9 +146,11 @@ export const destroyAllDoctypeLastSequence = () => { * Destroy the last sequence for a doctype * * @param {string} doctype - The doctype + * + * @returns {Promise} */ -export const destroyDoctypeLastSequence = doctype => { - const seqs = getAllLastSequences() +export const destroyDoctypeLastSequence = async doctype => { + const seqs = await getAllLastSequences() delete seqs[doctype] window.localStorage.setItem( LOCALSTORAGE_LASTSEQUENCES_KEY, @@ -140,8 +162,10 @@ export const destroyDoctypeLastSequence = doctype => { * Persist the warmed up queries * * @param {object} warmedUpQueries - The warmedup queries + * + * @returns {Promise} */ -export const persistWarmedUpQueries = warmedUpQueries => { +export const persistWarmedUpQueries = async warmedUpQueries => { window.localStorage.setItem( LOCALSTORAGE_WARMUPEDQUERIES_KEY, JSON.stringify(warmedUpQueries) @@ -151,9 +175,9 @@ export const persistWarmedUpQueries = warmedUpQueries => { /** * Get the warmed up queries * - * @returns {object} the warmed up queries + * @returns {Promise} the warmed up queries */ -export const getPersistedWarmedUpQueries = () => { +export const getPersistedWarmedUpQueries = async () => { const item = window.localStorage.getItem(LOCALSTORAGE_WARMUPEDQUERIES_KEY) if (!item) { return {} @@ -164,17 +188,18 @@ export const getPersistedWarmedUpQueries = () => { /** * Destroy the warmed queries * + * @returns {Promise} */ -export const destroyWarmedUpQueries = () => { +export const destroyWarmedUpQueries = async () => { window.localStorage.removeItem(LOCALSTORAGE_WARMUPEDQUERIES_KEY) } /** * Get the adapter name * - * @returns {string} The adapter name + * @returns {Promise} The adapter name */ -export const getAdapterName = () => { +export const getAdapterName = async () => { return window.localStorage.getItem(LOCALSTORAGE_ADAPTERNAME) } @@ -182,7 +207,9 @@ export const getAdapterName = () => { * Persist the adapter name * * @param {string} adapter - The adapter name + * + * @returns {Promise} */ -export const persistAdapterName = adapter => { +export const persistAdapterName = async adapter => { window.localStorage.setItem(LOCALSTORAGE_ADAPTERNAME, adapter) } diff --git a/packages/cozy-pouch-link/src/platformWeb.js b/packages/cozy-pouch-link/src/platformWeb.js new file mode 100644 index 0000000000..f27cf832dd --- /dev/null +++ b/packages/cozy-pouch-link/src/platformWeb.js @@ -0,0 +1,7 @@ +import PouchDB from 'pouchdb-browser' +import * as storage from './localStorage' + +export const platformWeb = { + storage, + pouchEngine: PouchDB +} diff --git a/packages/cozy-pouch-link/src/startReplication.js b/packages/cozy-pouch-link/src/startReplication.js index 7867cf83cb..a0c62bec04 100644 --- a/packages/cozy-pouch-link/src/startReplication.js +++ b/packages/cozy-pouch-link/src/startReplication.js @@ -2,10 +2,7 @@ import { default as helpers } from './helpers' import startsWith from 'lodash/startsWith' import logger from './logger' import { fetchRemoteInstance } from './remote' -import { - getLastReplicatedDocID, - persistLastReplicatedDocID -} from './localStorage' + const { isDesignDocument, isDeletedDocument } = helpers const BATCH_SIZE = 1000 // we have mostly small documents @@ -38,13 +35,15 @@ const TIME_UNITS = [['ms', 1000], ['s', 60], ['m', 60], ['h', 24]] * @param {boolean} replicationOptions.initialReplication Whether or not this is an initial replication * @param {string} replicationOptions.doctype The doctype to replicate * @param {Function} getReplicationURL A function that should return the remote replication URL + * @param {import('./types').PouchLocalStorage} storage Methods to access local storage * * @returns {Promise} A cancelable promise that resolves at the end of the replication */ export const startReplication = ( pouch, replicationOptions, - getReplicationURL + getReplicationURL, + storage ) => { let replication let docs = {} @@ -68,7 +67,7 @@ export const startReplication = ( // For the first remote->local replication, we manually replicate all docs // as it avoids to replicate all revs history, which can lead to // performances issues - docs = await replicateAllDocs(pouch, url, doctype) + docs = await replicateAllDocs(pouch, url, doctype, storage) const end = new Date() if (process.env.NODE_ENV !== 'production') { logger.info( @@ -141,13 +140,14 @@ const filterDocs = docs => { * @param {object} db - Pouch instance * @param {string} baseUrl - The remote instance * @param {string} doctype - The doctype to replicate - * @returns {Array} The retrieved documents + * @param {import('./types').PouchLocalStorage} storage - Methods to access local storage + * @returns {Promise} The retrieved documents */ -export const replicateAllDocs = async (db, baseUrl, doctype) => { +export const replicateAllDocs = async (db, baseUrl, doctype, storage) => { const remoteUrlAllDocs = new URL(`${baseUrl}/_all_docs`) const batchSize = BATCH_SIZE let hasMore = true - let startDocId = getLastReplicatedDocID(doctype) // Get last replicated _id in localStorage + let startDocId = await storage.getLastReplicatedDocID(doctype) // Get last replicated _id in localStorage let docs = [] while (hasMore) { @@ -166,7 +166,7 @@ export const replicateAllDocs = async (db, baseUrl, doctype) => { hasMore = false } await helpers.insertBulkDocs(db, docs) - persistLastReplicatedDocID(doctype, startDocId) + await storage.persistLastReplicatedDocID(doctype, startDocId) } } else { const res = await fetchRemoteInstance(remoteUrlAllDocs, { @@ -181,7 +181,7 @@ export const replicateAllDocs = async (db, baseUrl, doctype) => { filteredDocs.shift() // Remove first element, already included in previous request startDocId = filteredDocs[filteredDocs.length - 1]._id await helpers.insertBulkDocs(db, filteredDocs) - persistLastReplicatedDocID(doctype, startDocId) + await storage.persistLastReplicatedDocID(doctype, startDocId) docs = docs.concat(filteredDocs) if (res.rows.length < batchSize) { diff --git a/packages/cozy-pouch-link/src/types.js b/packages/cozy-pouch-link/src/types.js new file mode 100644 index 0000000000..9b174a081c --- /dev/null +++ b/packages/cozy-pouch-link/src/types.js @@ -0,0 +1,32 @@ +/** @typedef {object} SyncInfo + * @property {string} Date + */ + +/** + * @typedef {object} PouchLocalStorage + * @property {function(string, string): Promise} persistLastReplicatedDocID Persist the last replicated doc id for a doctype + * @property {function(): Promise>} getAllLastReplicatedDocID TODO + * @property {function(string): Promise} getLastReplicatedDocID Get the last replicated doc id for a doctype + * @property {function(): Promise} destroyAllLastReplicatedDocID Destroy all the replicated doc id + * @property {function(Record): Promise} persistSyncedDoctypes Persist the synchronized doctypes + * @property {function(): Promise} getPersistedSyncedDoctypes Get the persisted doctypes + * @property {function(): Promise} destroySyncedDoctypes Destroy the synced doctypes + * @property {function(string, string): Promise} persistDoctypeLastSequence Persist the last CouchDB sequence for a synced doctype + * @property {function(): Promise} getAllLastSequences TODO + * @property {function(string): Promise} getDoctypeLastSequence Get the last CouchDB sequence for a doctype + * @property {function(): Promise} destroyAllDoctypeLastSequence Destroy all the last sequence + * @property {function(string): Promise} destroyDoctypeLastSequence Destroy the last sequence for a doctype + * @property {function(object): Promise} persistWarmedUpQueries Persist the warmed up queries + * @property {function(): Promise} getPersistedWarmedUpQueries Get the warmed up queries + * @property {function(): Promise} destroyWarmedUpQueries Destroy the warmed queries + * @property {function(): Promise} getAdapterName Get the adapter name + * @property {function(string): Promise} persistAdapterName Persist the adapter name + */ + +/** + * @typedef {object} PouchLinkPlatform + * @property {PouchLocalStorage} storage Methods to access local storage + * @property {any} pouchEngine PouchDB class (can be pouchdb-core or pouchdb-browser) + */ + +export default {}