diff --git a/connectors/src/connectors/notion/lib/parents.ts b/connectors/src/connectors/notion/lib/parents.ts index 61635f6e6939..e8f88dd015a9 100644 --- a/connectors/src/connectors/notion/lib/parents.ts +++ b/connectors/src/connectors/notion/lib/parents.ts @@ -14,6 +14,7 @@ import { NotionDatabase, NotionPage, } from "@connectors/lib/models"; +import logger from "@connectors/logger/logger"; /** Compute the parents field for a notion pageOrDb See the [Design * Doc](https://www.notion.so/dust-tt/Engineering-e0f834b5be5a43569baaf76e9c41adf2?p=3d26536a4e0a464eae0c3f8f27a7af97&pm=s) @@ -97,15 +98,30 @@ export async function updateAllParentsFields( connectorId ); + logger.info( + { + connectorId, + pageIdsToUpdateCount: pageIdsToUpdate.size, + }, + "Updating parents field for pages" + ); + // Update everybody's parents field. Use of a memoization key to control // sharing memoization across updateAllParentsFields calls, which // can be desired or not depending on the use case - const q = new PQueue({ concurrency: 16 }); + const q = new PQueue({ concurrency: 4 }); const promises: Promise[] = []; for (const pageId of pageIdsToUpdate) { promises.push( q.add(async () => { const parents = await getParents(connectorId, pageId, memoizationKey); + logger.info( + { + connectorId, + pageId, + }, + "Updating parents field for page" + ); await updateDocumentParentsField( { dataSourceName: connector.dataSourceName, diff --git a/connectors/src/connectors/notion/temporal/activities.ts b/connectors/src/connectors/notion/temporal/activities.ts index a0badc1c4db5..18e58cefc968 100644 --- a/connectors/src/connectors/notion/temporal/activities.ts +++ b/connectors/src/connectors/notion/temporal/activities.ts @@ -913,6 +913,14 @@ export async function updateParentsFields( }) ).map((db) => db.notionDatabaseId); + localLogger.info( + { + notionPageIdsCount: notionPageIds.length, + notionDatabaseIdsCount: notionDatabaseIds.length, + }, + "Starting parents fields update." + ); + const nbUpdated = await updateAllParentsFields( connectorId, notionPageIds,