[Nodes Core] Ability to see diff core <-> connextors nodes in poke (#…

…10434) Description --- Seeing the diff directly will help debugging
dust-tt · Jan 31, 2025 · fb8926d · fb8926d
1 parent 8334e7d
commit fb8926d
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 16 deletions.
diff --git a/front/lib/api/content_nodes.ts b/front/lib/api/content_nodes.ts
@@ -76,7 +76,9 @@ export function computeNodesDiff({
   provider: ConnectorProvider | null;
   localLogger: typeof logger;
 }) {
-  const missingInternalIds: string[] = [];
+  const missingNodes: DataSourceViewContentNode[] = [];
+  const mismatchNodes: DataSourceViewContentNode[] = [];
+
   connectorsContentNodes.forEach((connectorsNode) => {
     const coreNodes = coreContentNodes.filter(
       (coreNode) => coreNode.internalId === connectorsNode.internalId
@@ -89,7 +91,7 @@ export function computeNodesDiff({
         connectorsNode.internalId !== "notion-unknown" &&
         !connectorsNode.internalId.startsWith("slack-channel-")
       ) {
-        missingInternalIds.push(connectorsNode.internalId);
+        missingNodes.push(connectorsNode);
       }
     } else if (coreNodes.length > 1) {
       // this one should never ever happen, it's a real red flag
@@ -276,29 +278,35 @@ export function computeNodesDiff({
           },
           "[CoreNodes] Node mismatch"
         );
+        connectorsNode.title = `[MISMATCH - CONNECTOR] ${connectorsNode.title}`;
+        coreNode.title = `[MISMATCH - CORE] ${coreNode.title}`;
+        mismatchNodes.push(connectorsNode);
+        mismatchNodes.push(coreNode);
       }
     }
   });
   if (
-    missingInternalIds.length > 0 &&
+    missingNodes.length > 0 &&
     // Snowflake's root call returns all the selected databases, schemas and tables even if non-root.
     !(
       provider === "snowflake" &&
-      missingInternalIds.every((id) =>
-        coreContentNodes.some((n) => n.parentInternalIds?.includes(id))
+      missingNodes.every((node) =>
+        coreContentNodes.some((n) =>
+          n.parentInternalIds?.includes(node.internalId)
+        )
       )
     )
   ) {
     localLogger.info(
       {
-        missingInternalIds,
+        missingInternalIds: missingNodes.map((n) => n.internalId),
         coreNodesCount: coreContentNodes.length,
         maxPageSizeReached: coreContentNodes.length === 1000, // max value determined by the limit set in getContentNodesForDataSourceViewFromCore
       },
       "[CoreNodes] Missing nodes from core"
     );
   }
-  const extraCoreInternalIds = coreContentNodes
+  const extraCoreNodes = coreContentNodes
     .filter(
       (coreNode) =>
         !connectorsContentNodes.some(
@@ -329,14 +337,24 @@ export function computeNodesDiff({
       (coreNode) =>
         provider !== "slack" ||
         !coreNode.internalId.startsWith("slack-channel-")
-    )
-    .map((coreNode) => coreNode.internalId);
-  if (extraCoreInternalIds.length > 0) {
+    );
+  if (extraCoreNodes.length > 0) {
     localLogger.info(
-      { extraCoreInternalIds },
+      {
+        extraCoreInternalIds: extraCoreNodes.map(
+          (coreNode) => coreNode.internalId
+        ),
+      },
       "[CoreNodes] Received extraneous core nodes"
     );
   }
+  missingNodes.forEach((node) => {
+    node.title = `[MISSING] ${node.title}`;
+  });
+  extraCoreNodes.forEach((coreNode) => {
+    coreNode.title = `[EXTRA] ${coreNode.title}`;
+  });
+  return [...mismatchNodes, ...extraCoreNodes, ...missingNodes];
 }
 
 export function getContentNodeType(node: CoreAPIContentNode): ContentNodeType {

diff --git a/front/lib/api/data_source_view.ts b/front/lib/api/data_source_view.ts
@@ -491,13 +491,34 @@ export async function getContentNodesForDataSourceView(
       "[CoreNodes] Could not fetch content nodes from core"
     );
   } else if (coreContentNodesRes.isOk()) {
-    computeNodesDiff({
+    const diff = computeNodesDiff({
       connectorsContentNodes: contentNodesResult.nodes,
       coreContentNodes: coreContentNodesRes.value.nodes,
       provider: dataSourceView.dataSource.connectorProvider,
       localLogger,
     });
 
+    if (showConnectorsNodes) {
+      if (diff.length == 0) {
+        diff.push({
+          internalId: "missing-connectors-nodes",
+          parentInternalId: null,
+          title: "ALL IS FINE",
+          type: "folder",
+          permission: "read",
+          lastUpdatedAt: new Date().getTime(),
+          parentInternalIds: [],
+          sourceUrl: null,
+          expandable: false,
+          preventSelection: false,
+        });
+      }
+      return new Ok({
+        nodes: filterAndCropContentNodesByView(dataSourceView, diff),
+        total: diff.length,
+      });
+    }
+
     // if development or dust workspace
     const workspaceModel = await getWorkspaceByModelId(
       dataSourceView.space.workspaceId
@@ -509,10 +530,7 @@ export async function getContentNodesForDataSourceView(
       );
     } else {
       const workspace = renderLightWorkspaceType({ workspace: workspaceModel });
-      if (
-        (isDevelopment() || isDustWorkspace(workspace)) &&
-        !showConnectorsNodes
-      ) {
+      if (isDevelopment() || isDustWorkspace(workspace)) {
         const contentNodesInView = filterAndCropContentNodesByView(
           dataSourceView,
           coreContentNodesRes.value.nodes