diff --git a/front/lib/api/content_nodes.ts b/front/lib/api/content_nodes.ts index 543d8787463c..de4f8000dd09 100644 --- a/front/lib/api/content_nodes.ts +++ b/front/lib/api/content_nodes.ts @@ -76,7 +76,9 @@ export function computeNodesDiff({ provider: ConnectorProvider | null; localLogger: typeof logger; }) { - const missingInternalIds: string[] = []; + const missingNodes: DataSourceViewContentNode[] = []; + const mismatchNodes: DataSourceViewContentNode[] = []; + connectorsContentNodes.forEach((connectorsNode) => { const coreNodes = coreContentNodes.filter( (coreNode) => coreNode.internalId === connectorsNode.internalId @@ -89,7 +91,7 @@ export function computeNodesDiff({ connectorsNode.internalId !== "notion-unknown" && !connectorsNode.internalId.startsWith("slack-channel-") ) { - missingInternalIds.push(connectorsNode.internalId); + missingNodes.push(connectorsNode); } } else if (coreNodes.length > 1) { // this one should never ever happen, it's a real red flag @@ -276,29 +278,35 @@ export function computeNodesDiff({ }, "[CoreNodes] Node mismatch" ); + connectorsNode.title = `[MISMATCH - CONNECTOR] ${connectorsNode.title}`; + coreNode.title = `[MISMATCH - CORE] ${coreNode.title}`; + mismatchNodes.push(connectorsNode); + mismatchNodes.push(coreNode); } } }); if ( - missingInternalIds.length > 0 && + missingNodes.length > 0 && // Snowflake's root call returns all the selected databases, schemas and tables even if non-root. !( provider === "snowflake" && - missingInternalIds.every((id) => - coreContentNodes.some((n) => n.parentInternalIds?.includes(id)) + missingNodes.every((node) => + coreContentNodes.some((n) => + n.parentInternalIds?.includes(node.internalId) + ) ) ) ) { localLogger.info( { - missingInternalIds, + missingInternalIds: missingNodes.map((n) => n.internalId), coreNodesCount: coreContentNodes.length, maxPageSizeReached: coreContentNodes.length === 1000, // max value determined by the limit set in getContentNodesForDataSourceViewFromCore }, "[CoreNodes] Missing nodes from core" ); } - const extraCoreInternalIds = coreContentNodes + const extraCoreNodes = coreContentNodes .filter( (coreNode) => !connectorsContentNodes.some( @@ -329,14 +337,24 @@ export function computeNodesDiff({ (coreNode) => provider !== "slack" || !coreNode.internalId.startsWith("slack-channel-") - ) - .map((coreNode) => coreNode.internalId); - if (extraCoreInternalIds.length > 0) { + ); + if (extraCoreNodes.length > 0) { localLogger.info( - { extraCoreInternalIds }, + { + extraCoreInternalIds: extraCoreNodes.map( + (coreNode) => coreNode.internalId + ), + }, "[CoreNodes] Received extraneous core nodes" ); } + missingNodes.forEach((node) => { + node.title = `[MISSING] ${node.title}`; + }); + extraCoreNodes.forEach((coreNode) => { + coreNode.title = `[EXTRA] ${coreNode.title}`; + }); + return [...mismatchNodes, ...extraCoreNodes, ...missingNodes]; } export function getContentNodeType(node: CoreAPIContentNode): ContentNodeType { diff --git a/front/lib/api/data_source_view.ts b/front/lib/api/data_source_view.ts index e2d03577abaa..d10191887cb6 100644 --- a/front/lib/api/data_source_view.ts +++ b/front/lib/api/data_source_view.ts @@ -491,13 +491,34 @@ export async function getContentNodesForDataSourceView( "[CoreNodes] Could not fetch content nodes from core" ); } else if (coreContentNodesRes.isOk()) { - computeNodesDiff({ + const diff = computeNodesDiff({ connectorsContentNodes: contentNodesResult.nodes, coreContentNodes: coreContentNodesRes.value.nodes, provider: dataSourceView.dataSource.connectorProvider, localLogger, }); + if (showConnectorsNodes) { + if (diff.length == 0) { + diff.push({ + internalId: "missing-connectors-nodes", + parentInternalId: null, + title: "ALL IS FINE", + type: "folder", + permission: "read", + lastUpdatedAt: new Date().getTime(), + parentInternalIds: [], + sourceUrl: null, + expandable: false, + preventSelection: false, + }); + } + return new Ok({ + nodes: filterAndCropContentNodesByView(dataSourceView, diff), + total: diff.length, + }); + } + // if development or dust workspace const workspaceModel = await getWorkspaceByModelId( dataSourceView.space.workspaceId @@ -509,10 +530,7 @@ export async function getContentNodesForDataSourceView( ); } else { const workspace = renderLightWorkspaceType({ workspace: workspaceModel }); - if ( - (isDevelopment() || isDustWorkspace(workspace)) && - !showConnectorsNodes - ) { + if (isDevelopment() || isDustWorkspace(workspace)) { const contentNodesInView = filterAndCropContentNodesByView( dataSourceView, coreContentNodesRes.value.nodes