fix(notion): add retries to update all parents (#2832)

* fix(notion): add retries to update all parents * fix migration type errors --------- Co-authored-by: Henry Fontanier <henry@dust.tt>
dust-tt · Dec 11, 2023 · 8ec42a1 · 8ec42a1
1 parent 86cc8fa
commit 8ec42a1
Show file tree

Hide file tree

Showing 6 changed files with 45 additions and 27 deletions.
diff --git a/connectors/migrations/20230906_2_slack_fill_parents_field.ts b/connectors/migrations/20230906_2_slack_fill_parents_field.ts
@@ -73,11 +73,11 @@ async function updateParentsFieldForConnector(connector: Connector) {
     await Promise.all(
       chunk.map(
         async (documentIdAndChannel) =>
-          await updateDocumentParentsField(
-            connector,
-            documentIdAndChannel.documentId,
-            [documentIdAndChannel.channelId]
-          )
+          await updateDocumentParentsField({
+            dataSourceConfig: connector,
+            documentId: documentIdAndChannel.documentId,
+            parents: [documentIdAndChannel.channelId],
+          })
       )
     );
   }

diff --git a/connectors/migrations/20230906_3_github_fill_parents_field.ts b/connectors/migrations/20230906_3_github_fill_parents_field.ts
@@ -79,10 +79,14 @@ async function updateDiscussionsParentsFieldForConnector(connector: Connector) {
           document.repoId,
           document.discussionNumber
         );
-        await updateDocumentParentsField(connector, docId, [
-          getDiscussionDocumentId(document.repoId, document.discussionNumber),
-          document.repoId,
-        ]);
+        await updateDocumentParentsField({
+          dataSourceConfig: connector,
+          documentId: docId,
+          parents: [
+            getDiscussionDocumentId(document.repoId, document.discussionNumber),
+            document.repoId,
+          ],
+        });
       })
     );
     process.stdout.write(".");
@@ -105,10 +109,14 @@ async function updateIssuesParentsFieldForConnector(connector: Connector) {
     await Promise.all(
       chunk.map(async (document) => {
         const docId = getIssueDocumentId(document.repoId, document.issueNumber);
-        await updateDocumentParentsField(connector, docId, [
-          getIssueDocumentId(document.repoId, document.issueNumber),
-          document.repoId,
-        ]);
+        await updateDocumentParentsField({
+          dataSourceConfig: connector,
+          documentId: docId,
+          parents: [
+            getIssueDocumentId(document.repoId, document.issueNumber),
+            document.repoId,
+          ],
+        });
       })
     );
     process.stdout.write(".");

diff --git a/connectors/src/connectors/notion/lib/parents.ts b/connectors/src/connectors/notion/lib/parents.ts
@@ -119,15 +119,15 @@ export async function updateAllParentsFields(
           },
           "Updating parents field for page"
         );
-        await updateDocumentParentsField(
-          {
+        await updateDocumentParentsField({
+          dataSourceConfig: {
             dataSourceName: connector.dataSourceName,
             workspaceId: connector.workspaceId,
             workspaceAPIKey: connector.workspaceAPIKey,
           },
-          `notion-${pageId}`,
-          parents
-        );
+          documentId: `notion-${pageId}`,
+          parents,
+        });
       })
     );
   }

diff --git a/connectors/src/connectors/notion/temporal/workflows.ts b/connectors/src/connectors/notion/temporal/workflows.ts
@@ -19,14 +19,15 @@ const { garbageCollect } = proxyActivities<typeof activities>({
   startToCloseTimeout: "120 minute",
 });
 
-const { upsertDatabase } = proxyActivities<typeof activities>({
+const { upsertDatabase, updateParentsFields } = proxyActivities<
+  typeof activities
+>({
   startToCloseTimeout: "60 minute",
 });
 
 const {
   getPagesAndDatabasesToSync,
   garbageCollectorMarkAsSeen,
-  updateParentsFields,
   fetchDatabaseChildPages,
   cachePage,
   cacheBlockChildren,

diff --git a/connectors/src/lib/data_sources.ts b/connectors/src/lib/data_sources.ts
@@ -174,12 +174,21 @@ export async function deleteFromDataSource(
   }
 }
 
-export async function updateDocumentParentsField(
-  dataSourceConfig: DataSourceConfig,
-  documentId: string,
-  parents: string[],
-  loggerArgs: Record<string, string | number> = {}
-) {
+export const updateDocumentParentsField = withRetries(
+  _updateDocumentParentsField
+);
+
+async function _updateDocumentParentsField({
+  dataSourceConfig,
+  documentId,
+  parents,
+  loggerArgs = {},
+}: {
+  dataSourceConfig: DataSourceConfig;
+  documentId: string;
+  parents: string[];
+  loggerArgs?: Record<string, string | number>;
+}) {
   const localLogger = logger.child({ ...loggerArgs, documentId });
   const urlSafeName = encodeURIComponent(dataSourceConfig.dataSourceName);
   const endpoint = `${DUST_FRONT_API}/api/v1/w/${dataSourceConfig.workspaceId}/data_sources/${urlSafeName}/documents/${documentId}/parents`;

diff --git a/connectors/src/lib/dust_front_api_helpers.ts b/connectors/src/lib/dust_front_api_helpers.ts
@@ -26,7 +26,7 @@ export function withRetries<T, U>(
             retries: retries,
             sleepTime: sleepTime,
           },
-          "Error upserting to data source. Retrying..."
+          "Error while executing retriable function. Retrying..."
         );
         await new Promise((resolve) => setTimeout(resolve, sleepTime));
         errors.push(e);