Fix snowflake missing and extra nodes (#10431)

* Modified log * logging undefined for consistency * Modified diff to ignore missing and extra nodes when relevant * Added check on viewType * Passing viewType to computeNodesDIff
dust-tt · Jan 31, 2025 · 53c22fc · 53c22fc
1 parent 8d732bf
commit 53c22fc
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 2 deletions.
diff --git a/front/lib/api/content_nodes.ts b/front/lib/api/content_nodes.ts
@@ -1,5 +1,6 @@
 import type {
   ConnectorProvider,
+  ContentNodesViewType,
   ContentNodeType,
   CoreAPIContentNode,
   DataSourceViewContentNode,
@@ -69,11 +70,13 @@ export function computeNodesDiff({
   connectorsContentNodes,
   coreContentNodes,
   provider,
+  viewType,
   localLogger,
 }: {
   connectorsContentNodes: DataSourceViewContentNode[];
   coreContentNodes: DataSourceViewContentNode[];
   provider: ConnectorProvider | null;
+  viewType: ContentNodesViewType;
   localLogger: typeof logger;
 }) {
   const missingNodes: DataSourceViewContentNode[] = [];
@@ -91,7 +94,22 @@ export function computeNodesDiff({
         connectorsNode.internalId !== "notion-unknown" &&
         !connectorsNode.internalId.startsWith("slack-channel-")
       ) {
-        missingNodes.push(connectorsNode);
+        if (
+          // For snowflake we ignore the missing nodes if we returned a node that is a parent.
+          // This is because connectors returns all children tables when fed with internalIds while core returns only these ids
+          // See https://github.com/orgs/dust-tt/projects/3/views/1?pane=issue&itemId=95859834&issue=dust-tt%7Cdust%7C10400
+          !(
+            provider === "snowflake" &&
+            coreNodes.some((n) =>
+              connectorsNode.internalId.startsWith(n.internalId)
+            )
+          )
+        ) {
+          // Connectors return tables even when viewType is documents, core doesn't
+          if (!(provider === "snowflake" && viewType === "documents")) {
+            missingNodes.push(connectorsNode);
+          }
+        }
       }
     } else if (coreNodes.length > 1) {
       // this one should never ever happen, it's a real red flag
@@ -350,6 +368,19 @@ export function computeNodesDiff({
       (coreNode) =>
         provider !== "slack" ||
         !coreNode.internalId.startsWith("slack-channel-")
+    )
+    // Snowflake schemas are returned from core, while connector only return tables
+    // Detect schema for which we have a table in connectors and ignore them.
+    // See https://github.com/orgs/dust-tt/projects/3/views/1?pane=issue&itemId=95859834&issue=dust-tt%7Cdust%7C10400
+    .filter(
+      (coreNode) =>
+        !(
+          provider === "snowflake" &&
+          coreNode.internalId.split(".").length === 2 &&
+          connectorsContentNodes.some((n) =>
+            n.internalId.startsWith(coreNode.internalId)
+          )
+        )
     );
   if (extraCoreNodes.length > 0) {
     localLogger.info(

diff --git a/front/lib/api/data_source_view.ts b/front/lib/api/data_source_view.ts
@@ -462,9 +462,11 @@ export async function getContentNodesForDataSourceView(
     dataSourceViewId: dataSourceView.sId,
     provider: dataSourceView.dataSource.connectorProvider,
     viewType: params.viewType,
-    isRootCall: !params.parentId && !params.internalIds,
+    isRootCall:
+      !params.parentId && !params.internalIds && !dataSourceView.parentsIn,
     parentId: params.parentId,
     internalIds: params.internalIds,
+    parentsIn: dataSourceView.parentsIn || undefined,
   });
 
   const coreStart = new Date();
@@ -495,6 +497,7 @@ export async function getContentNodesForDataSourceView(
       connectorsContentNodes: contentNodesResult.nodes,
       coreContentNodes: coreContentNodesRes.value.nodes,
       provider: dataSourceView.dataSource.connectorProvider,
+      viewType: params.viewType,
       localLogger,
     });