Skip to content

Commit

Permalink
connectors: parents update, reduce concurrency and log more (#2608)
Browse files Browse the repository at this point in the history
* connectors: parents update, reduce concurrency and log more

* more logging
  • Loading branch information
spolu authored Nov 21, 2023
1 parent 1e9d8be commit 63fc77b
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 1 deletion.
18 changes: 17 additions & 1 deletion connectors/src/connectors/notion/lib/parents.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
NotionDatabase,
NotionPage,
} from "@connectors/lib/models";
import logger from "@connectors/logger/logger";

/** Compute the parents field for a notion pageOrDb See the [Design
* Doc](https://www.notion.so/dust-tt/Engineering-e0f834b5be5a43569baaf76e9c41adf2?p=3d26536a4e0a464eae0c3f8f27a7af97&pm=s)
Expand Down Expand Up @@ -97,15 +98,30 @@ export async function updateAllParentsFields(
connectorId
);

logger.info(
{
connectorId,
pageIdsToUpdateCount: pageIdsToUpdate.size,
},
"Updating parents field for pages"
);

// Update everybody's parents field. Use of a memoization key to control
// sharing memoization across updateAllParentsFields calls, which
// can be desired or not depending on the use case
const q = new PQueue({ concurrency: 16 });
const q = new PQueue({ concurrency: 4 });
const promises: Promise<void>[] = [];
for (const pageId of pageIdsToUpdate) {
promises.push(
q.add(async () => {
const parents = await getParents(connectorId, pageId, memoizationKey);
logger.info(
{
connectorId,
pageId,
},
"Updating parents field for page"
);
await updateDocumentParentsField(
{
dataSourceName: connector.dataSourceName,
Expand Down
8 changes: 8 additions & 0 deletions connectors/src/connectors/notion/temporal/activities.ts
Original file line number Diff line number Diff line change
Expand Up @@ -913,6 +913,14 @@ export async function updateParentsFields(
})
).map((db) => db.notionDatabaseId);

localLogger.info(
{
notionPageIdsCount: notionPageIds.length,
notionDatabaseIdsCount: notionDatabaseIds.length,
},
"Starting parents fields update."
);

const nbUpdated = await updateAllParentsFields(
connectorId,
notionPageIds,
Expand Down

0 comments on commit 63fc77b

Please sign in to comment.