blocknote ai

src/frontend/apps/impress/package.json

@@ -15,9 +15,14 @@
     "test:watch": "jest --watch"
   },
   "dependencies": {
-    "@blocknote/core": "0.21.0",
-    "@blocknote/mantine": "0.21.0",
-    "@blocknote/react": "0.21.0",
+    "ai": "^4.1.18",
+    "zod": "^3.24.1",
+    "@ai-sdk/openai": "^1.1.9",
+    "@blocknote/core": "https://gitpkg.vercel.app/typecellOS/blocknote/packages/core?ai-block-built",
+    "@blocknote/mantine": "https://gitpkg.vercel.app/typecellOS/blocknote/packages/mantine?ai-block-built",
+    "@blocknote/react": "https://gitpkg.vercel.app/typecellOS/blocknote/packages/react?ai-block-built",
+    "@blocknote/xl-ai": "https://gitpkg.vercel.app/typecellOS/blocknote/packages/xl-ai?ai-block-built",
+    "vitest": "^2.0.3",
     "@blocknote/xl-docx-exporter": "0.21.0",
     "@blocknote/xl-pdf-exporter": "0.21.0",
     "@gouvfr-lasuite/integration": "1.0.2",

src/frontend/apps/impress/src/features/docs/doc-editor/components/AIButton.tsx

@@ -95,7 +95,7 @@ export function AIGroupButton() {
   return (
     <Components.Generic.Menu.Root>
       <Components.Generic.Menu.Trigger>
-        <Components.FormattingToolbar.Button
+        <Components.Toolbar.Button
           className="bn-button bn-menu-item"
           data-test="ai-actions"
           label="AI"

src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx

@@ -1,18 +1,35 @@
-import { Dictionary, locales } from '@blocknote/core';
+import { Box, TextErrors } from '@/components';
+import { useAuthStore } from '@/core/auth';
+import { Doc } from '@/features/docs/doc-management';
+import { createOpenAI } from '@ai-sdk/openai';
+import {
+  BlockNoteEditor as BNEditor,
+  Dictionary,
+  filterSuggestionItems,
+  locales,
+} from '@blocknote/core';
 import '@blocknote/core/fonts/inter.css';
 import { BlockNoteView } from '@blocknote/mantine';
 import '@blocknote/mantine/style.css';
-import { useCreateBlockNote } from '@blocknote/react';
+import {
+  getDefaultReactSlashMenuItems,
+  SuggestionMenuController,
+  useCreateBlockNote,
+} from '@blocknote/react';
+import {
+  locales as aiLocales,
+  AIShowSelectionPlugin,
+  BlockNoteAIContextProvider,
+  BlockNoteAIUI,
+  createBlockNoteAIClient,
+  getAISlashMenuItems,
+  useBlockNoteAIContext,
+} from '@blocknote/xl-ai';
 import { HocuspocusProvider } from '@hocuspocus/provider';
-import { useEffect } from 'react';
+import { useEffect, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { css } from 'styled-components';
 import * as Y from 'yjs';
-
-import { Box, TextErrors } from '@/components';
-import { useAuthStore } from '@/core/auth';
-import { Doc } from '@/features/docs/doc-management';
-
 import { useUploadFile } from '../hook';
 import { useHeadings } from '../hook/useHeadings';
 import useSaveDoc from '../hook/useSaveDoc';
@@ -128,8 +145,36 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => {
     ? 'Reader'
     : userData?.full_name || userData?.email || t('Anonymous');
 
+  const model = useMemo(() => {
+    const blocknoteAIClient = createBlockNoteAIClient({
+      apiKey: 'BLOCKNOTE-API-KEY-CURRENTLY-NOT-NEEDED',
+      baseURL: 'https://blocknote-esy4.onrender.com/ai',
+    });
+
+    return createOpenAI({
+      baseURL: 'https://albert.api.staging.etalab.gouv.fr/v1',
+      ...blocknoteAIClient.getProviderSettings('albert-etalab'),
+      compatibility: 'compatible',
+    })('albert-etalab/neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8');
+
+    // We call the model via a proxy server (see above) that has the API key,
+    // but we could also call the model directly from the frontend.
+    // i.e., this should work as well (but it would leak your albert key to the frontend):
+    /*
+      return createOpenAI({
+      baseURL: 'https://albert.api.staging.etalab.gouv.fr/v1',
+      apiKey: 'ALBERT-API-KEY',
+      compatibility: 'compatible',
+    })('albert-etalab/neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8');
+     */
+  }, []);
+
   const editor = useCreateBlockNote(
     {
+      _extensions: {
+        aiSelection: new AIShowSelectionPlugin(),
+      },
+
       collaboration: {
         provider,
         fragment: provider.document.getXmlFragment('document-store'),
@@ -163,7 +208,10 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => {
           return cursor;
         },
       },
-      dictionary: locales[lang as keyof typeof locales] as Dictionary,
+      dictionary: {
+        ...(locales[lang as keyof typeof locales] as Dictionary),
+        ai: (aiLocales as any)[lang] as Dictionary,
+      },
       uploadFile,
     },
     [collabName, lang, provider, uploadFile],
@@ -199,13 +247,40 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => {
         formattingToolbar={false}
         editable={!readOnly}
         theme="light"
+        slashMenu={false}
       >
-        <BlockNoteToolbar />
+        <BlockNoteAIContextProvider
+          model={model}
+          dataFormat={'markdown'}
+          stream={false}
+        >
+          <BlockNoteAIUI />
+          <BlockNoteToolbar />
+          <SuggestionMenu editor={editor} />
+        </BlockNoteAIContextProvider>
       </BlockNoteView>
     </Box>
   );
 };
 
+function SuggestionMenu(props: { editor: BNEditor<any, any, any> }) {
+  const ctx = useBlockNoteAIContext();
+  return (
+    <SuggestionMenuController
+      triggerCharacter="/"
+      getItems={async (query) =>
+        filterSuggestionItems(
+          [
+            ...getDefaultReactSlashMenuItems(props.editor),
+            ...getAISlashMenuItems(props.editor, ctx),
+          ],
+          query,
+        )
+      }
+    />
+  );
+}
+
 interface BlockNoteEditorVersionProps {
   initialContent: Y.XmlFragment;
 }

src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteToolbar.tsx

@@ -5,9 +5,9 @@ import {
   FormattingToolbarProps,
   getFormattingToolbarItems,
 } from '@blocknote/react';
-import React, { useCallback } from 'react';
+import { AIToolbarButton } from '@blocknote/xl-ai';
+import { useCallback } from 'react';
 
-import { AIGroupButton } from './AIButton';
 import { MarkdownButton } from './MarkdownButton';
 
 export const BlockNoteToolbar = () => {
@@ -17,7 +17,8 @@ export const BlockNoteToolbar = () => {
         {getFormattingToolbarItems(blockTypeSelectItems)}
 
         {/* Extra button to do some AI powered actions */}
-        <AIGroupButton key="AIButton" />
+        {/* <AIGroupButton key="AIButton" /> */}
+        <AIToolbarButton key="AIButton" />
 
         {/* Extra button to convert from markdown to json */}
         <MarkdownButton key="customButton" />

src/frontend/apps/impress/src/features/docs/doc-editor/components/MarkdownButton.tsx

@@ -5,7 +5,7 @@ import {
   useSelectedBlocks,
 } from '@blocknote/react';
 import { forEach, isArray } from 'lodash';
-import React, { useMemo } from 'react';
+import { useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 
 type Block = {
@@ -80,11 +80,11 @@ export function MarkdownButton() {
   }
 
   return (
-    <Components.FormattingToolbar.Button
+    <Components.Toolbar.Button
       mainTooltip={t('Convert Markdown')}
       onClick={handleConvertMarkdown}
     >
       M
-    </Components.FormattingToolbar.Button>
+    </Components.Toolbar.Button>
   );
 }

src/frontend/package.json

@@ -35,6 +35,10 @@
     "eslint": "8.57.0",
     "react": "18.3.1",
     "react-dom": "18.3.1",
-    "typescript": "5.7.3"
+    "typescript": "5.7.3",
+    "@blocknote/core": "https://gitpkg.vercel.app/typecellOS/blocknote/packages/core?ai-block-built",
+    "@blocknote/mantine": "https://gitpkg.vercel.app/typecellOS/blocknote/packages/mantine?ai-block-built",
+    "@blocknote/react": "https://gitpkg.vercel.app/typecellOS/blocknote/packages/react?ai-block-built",
+    "@blocknote/xl-ai": "https://gitpkg.vercel.app/typecellOS/blocknote/packages/xl-ai?ai-block-built"
   }
 }