build: add matrix

.github/workflows/release.yml

@@ -21,6 +21,10 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest, windows-latest, ubuntu-latest]
+        arch: [x64, arm64]
+        exclude:
+          - os: windows-latest
+            arch: arm64
 
     steps:
       - name: Check out Git repository
@@ -30,6 +34,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 20
+          arch: ${{ matrix.arch }}
 
       - name: Install corepack
         run: corepack enable && corepack prepare yarn@4.3.1 --activate
@@ -42,6 +47,7 @@ jobs:
         run: yarn build:linux
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
+          ARCH: ${{ matrix.arch }}
 
       - name: Build Mac
         if: matrix.os == 'macos-latest'
@@ -53,12 +59,14 @@ jobs:
           APPLE_APP_SPECIFIC_PASSWORD: ${{ vars.APPLE_APP_SPECIFIC_PASSWORD }}
           APPLE_TEAM_ID: ${{ vars.APPLE_TEAM_ID }}
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
+          ARCH: ${{ matrix.arch }}
 
       - name: Build Windows
         if: matrix.os == 'windows-latest'
         run: yarn build:win
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
+          ARCH: ${{ matrix.arch }}
 
       - name: Replace spaces in filenames
         run: node scripts/replaceSpaces.js

electron.vite.config.ts

@@ -16,6 +16,7 @@ export default defineConfig({
           '@llm-tools/embedjs-loader-msoffice',
           '@llm-tools/embedjs-loader-xml',
           '@llm-tools/embedjs-loader-pdf',
+          '@llm-tools/embedjs-loader-sitemap',
           '@lancedb/lancedb'
         ]
       })
@@ -29,7 +30,7 @@ export default defineConfig({
     },
     build: {
       rollupOptions: {
-        external: ['@lancedb/lancedb', '@llm-tools/embedjs-loader-sitemap']
+        external: ['@lancedb/lancedb']
       }
     }
   },

package.json

@@ -28,9 +28,9 @@
     "build": "npm run typecheck && electron-vite build",
     "postinstall": "electron-builder install-app-deps",
     "build:unpack": "dotenv npm run build && electron-builder --dir",
-    "build:win": "dotenv npm run build && electron-builder --win --publish never",
-    "build:mac": "dotenv electron-vite build && electron-builder --mac --publish never",
-    "build:linux": "dotenv electron-vite build && electron-builder --linux --publish never",
+    "build:win": "dotenv npm run build && electron-builder --win --$ARCH",
+    "build:mac": "dotenv electron-vite build && electron-builder --mac --$ARCH",
+    "build:linux": "dotenv electron-vite build && electron-builder --linux --$ARCH",
     "release": "node scripts/version.js",
     "publish": "yarn release patch push",
     "pulish:artifacts": "cd packages/artifacts && npm publish && cd -",
@@ -46,6 +46,7 @@
     "@llm-tools/embedjs-loader-markdown": "^0.1.24",
     "@llm-tools/embedjs-loader-msoffice": "^0.1.24",
     "@llm-tools/embedjs-loader-pdf": "^0.1.24",
+    "@llm-tools/embedjs-loader-sitemap": "^0.1.24",
     "@llm-tools/embedjs-loader-web": "^0.1.24",
     "@llm-tools/embedjs-loader-xml": "^0.1.24",
     "@llm-tools/embedjs-ollama": "^0.1.24",

src/main/services/KnowledgeService.ts

@@ -7,9 +7,10 @@ import { LanceDb } from '@llm-tools/embedjs-lancedb'
 import { MarkdownLoader } from '@llm-tools/embedjs-loader-markdown'
 import { DocxLoader } from '@llm-tools/embedjs-loader-msoffice'
 import { PdfLoader } from '@llm-tools/embedjs-loader-pdf'
+import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap'
 import { WebLoader } from '@llm-tools/embedjs-loader-web'
 import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'
-import { FileType, RagAppRequestParams } from '@types'
+import { FileType, KnowledgeBaseParams, KnowledgeItem } from '@types'
 import { app } from 'electron'
 
 class KnowledgeService {
@@ -25,7 +26,7 @@ class KnowledgeService {
     }
   }
 
-  private getRagApplication = async ({ id, model, apiKey, baseURL }: RagAppRequestParams): Promise<RAGApplication> => {
+  private getRagApplication = async ({ id, model, apiKey, baseURL }: KnowledgeBaseParams): Promise<RAGApplication> => {
     return new RAGApplicationBuilder()
       .setModel('NO_MODEL')
       .setEmbeddingModel(
@@ -42,13 +43,13 @@ class KnowledgeService {
 
   public create = async (
     _: Electron.IpcMainInvokeEvent,
-    { id, model, apiKey, baseURL }: RagAppRequestParams
+    { id, model, apiKey, baseURL }: KnowledgeBaseParams
   ): Promise<void> => {
     this.getRagApplication({ id, model, apiKey, baseURL })
   }
 
-  public reset = async (_: Electron.IpcMainInvokeEvent, { config }: { config: RagAppRequestParams }): Promise<void> => {
-    const ragApplication = await this.getRagApplication(config)
+  public reset = async (_: Electron.IpcMainInvokeEvent, { base }: { base: KnowledgeBaseParams }): Promise<void> => {
+    const ragApplication = await this.getRagApplication(base)
     await ragApplication.reset()
   }
 
@@ -61,45 +62,59 @@ class KnowledgeService {
 
   public add = async (
     _: Electron.IpcMainInvokeEvent,
-    { data, config }: { data: string | FileType; config: RagAppRequestParams }
+    { base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem }
   ): Promise<AddLoaderReturn> => {
-    const ragApplication = await this.getRagApplication(config)
+    const ragApplication = await this.getRagApplication(base)
 
-    if (typeof data === 'string') {
-      if (data.startsWith('http')) {
-        return await ragApplication.addLoader(new WebLoader({ urlOrContent: data }))
+    if (item.type === 'url') {
+      const content = item.content as string
+      if (content.startsWith('http')) {
+        return await ragApplication.addLoader(new WebLoader({ urlOrContent: content }))
       }
-      return await ragApplication.addLoader(new TextLoader({ text: data }))
     }
 
-    if (data.ext === '.pdf') {
-      return await ragApplication.addLoader(new PdfLoader({ filePathOrUrl: data.path }) as any)
+    if (item.type === 'sitemap') {
+      const content = item.content as string
+      return await ragApplication.addLoader(new SitemapLoader({ url: content }))
     }
 
-    if (data.ext === '.docx') {
-      return await ragApplication.addLoader(new DocxLoader({ filePathOrUrl: data.path }) as any)
+    if (item.type === 'note') {
+      const content = item.content as string
+      return await ragApplication.addLoader(new TextLoader({ text: content }))
     }
 
-    if (data.ext === '.md' || data.ext === '.mdx') {
-      return await ragApplication.addLoader(new MarkdownLoader({ filePathOrUrl: data.path }) as any)
+    if (item.type === 'file') {
+      const file = item.content as FileType
+
+      if (file.ext === '.pdf') {
+        return await ragApplication.addLoader(new PdfLoader({ filePathOrUrl: file.path }) as any)
+      }
+
+      if (file.ext === '.docx') {
+        return await ragApplication.addLoader(new DocxLoader({ filePathOrUrl: file.path }) as any)
+      }
+
+      if (file.ext.startsWith('.md')) {
+        return await ragApplication.addLoader(new MarkdownLoader({ filePathOrUrl: file.path }) as any)
+      }
     }
 
     return { entriesAdded: 0, uniqueId: '', loaderType: '' }
   }
 
   public remove = async (
     _: Electron.IpcMainInvokeEvent,
-    { uniqueId, config }: { uniqueId: string; config: RagAppRequestParams }
+    { uniqueId, base }: { uniqueId: string; base: KnowledgeBaseParams }
   ): Promise<void> => {
-    const ragApplication = await this.getRagApplication(config)
+    const ragApplication = await this.getRagApplication(base)
     await ragApplication.deleteLoader(uniqueId)
   }
 
   public search = async (
     _: Electron.IpcMainInvokeEvent,
-    { search, config }: { search: string; config: RagAppRequestParams }
+    { search, base }: { search: string; base: KnowledgeBaseParams }
   ): Promise<ExtractChunkData[]> => {
-    const ragApplication = await this.getRagApplication(config)
+    const ragApplication = await this.getRagApplication(base)
     return await ragApplication.search(search)
   }
 }

src/preload/index.d.ts

@@ -2,7 +2,7 @@ import { ElectronAPI } from '@electron-toolkit/preload'
 import { AddLoaderReturn, ExtractChunkData } from '@llm-tools/embedjs-interfaces'
 import { FileType } from '@renderer/types'
 import { WebDavConfig } from '@renderer/types'
-import { AppInfo, LanguageVarious, RagAppRequestParams } from '@renderer/types'
+import { AppInfo, KnowledgeBaseParams, KnowledgeItem, LanguageVarious } from '@renderer/types'
 import type { OpenDialogOptions } from 'electron'
 import type { UpdateInfo } from 'electron-updater'
 import { Readable } from 'stream'
@@ -60,12 +60,12 @@ declare global {
         update: (shortcuts: Shortcut[]) => Promise<void>
       }
       knowledgeBase: {
-        create: ({ id, model, apiKey, baseURL }: RagAppRequestParams) => Promise<void>
-        reset: ({ config }: { config: RagAppRequestParams }) => Promise<void>
+        create: ({ id, model, apiKey, baseURL }: KnowledgeBaseParams) => Promise<void>
+        reset: ({ base }: { base: KnowledgeBaseParams }) => Promise<void>
         delete: (id: string) => Promise<void>
-        add: ({ data, config }: { data: string | FileType; config: RagAppRequestParams }) => Promise<AddLoaderReturn>
-        remove: ({ uniqueId, config }: { uniqueId: string; config: RagAppRequestParams }) => Promise<void>
-        search: ({ search, config }: { search: string; config: RagAppRequestParams }) => Promise<ExtractChunkData[]>
+        add: ({ base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem }) => Promise<AddLoaderReturn>
+        remove: ({ uniqueId, base }: { uniqueId: string; base: KnowledgeBaseParams }) => Promise<void>
+        search: ({ search, base }: { search: string; base: KnowledgeBaseParams }) => Promise<ExtractChunkData[]>
       }
     }
   }

src/preload/index.ts

@@ -1,5 +1,5 @@
 import { electronAPI } from '@electron-toolkit/preload'
-import { FileType, RagAppRequestParams, Shortcut, WebDavConfig } from '@types'
+import { KnowledgeBaseParams, KnowledgeItem, Shortcut, WebDavConfig } from '@types'
 import { contextBridge, ipcRenderer, OpenDialogOptions } from 'electron'
 
 // Custom APIs for renderer
@@ -52,16 +52,16 @@ const api = {
     update: (shortcuts: Shortcut[]) => ipcRenderer.invoke('shortcuts:update', shortcuts)
   },
   knowledgeBase: {
-    create: ({ id, model, apiKey, baseURL }: RagAppRequestParams) =>
+    create: ({ id, model, apiKey, baseURL }: KnowledgeBaseParams) =>
       ipcRenderer.invoke('knowledge-base:create', { id, model, apiKey, baseURL }),
-    reset: ({ config }: { config: RagAppRequestParams }) => ipcRenderer.invoke('knowledge-base:reset', { config }),
+    reset: ({ base }: { base: KnowledgeBaseParams }) => ipcRenderer.invoke('knowledge-base:reset', { base }),
     delete: (id: string) => ipcRenderer.invoke('knowledge-base:delete', id),
-    add: ({ data, config }: { data: string | FileType; config: RagAppRequestParams }) =>
-      ipcRenderer.invoke('knowledge-base:add', { data, config }),
-    remove: ({ uniqueId, config }: { uniqueId: string; config: RagAppRequestParams }) =>
-      ipcRenderer.invoke('knowledge-base:remove', { uniqueId, config }),
-    search: ({ search, config }: { search: string; config: RagAppRequestParams }) =>
-      ipcRenderer.invoke('knowledge-base:search', { search, config })
+    add: ({ base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem }) =>
+      ipcRenderer.invoke('knowledge-base:add', { base, item }),
+    remove: ({ uniqueId, base }: { uniqueId: string; base: KnowledgeBaseParams }) =>
+      ipcRenderer.invoke('knowledge-base:remove', { uniqueId, base }),
+    search: ({ search, base }: { search: string; base: KnowledgeBaseParams }) =>
+      ipcRenderer.invoke('knowledge-base:search', { search, base })
   }
 }
 

src/renderer/src/hooks/useknowledge.ts

@@ -2,7 +2,7 @@
 import { db } from '@renderer/databases/index'
 import KnowledgeQueue from '@renderer/queue/KnowledgeQueue'
 import FileManager from '@renderer/services/FileManager'
-import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
+import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService'
 import { RootState } from '@renderer/store'
 import {
   addBase,
@@ -143,9 +143,8 @@ export const useKnowledge = (baseId: string) => {
   const removeItem = async (item: KnowledgeItem) => {
     dispatch(removeItemAction({ baseId, item }))
     if (base) {
-      const config = getRagAppRequestParams(base)
       if (item?.uniqueId) {
-        await window.api.knowledgeBase.remove({ uniqueId: item.uniqueId, config })
+        await window.api.knowledgeBase.remove({ uniqueId: item.uniqueId, base: getKnowledgeBaseParams(base) })
       }
       if (item.type === 'file' && typeof item.content === 'object') {
         await FileManager.deleteFile(item.content.id)

src/renderer/src/i18n/locales/en-us.json

@@ -560,8 +560,9 @@
       "no_bases": "No knowledge bases available",
       "clear_selection": "Clear selection",
       "delete_confirm": "Are you sure you want to delete this knowledge base?",
-      "sitemaps": "Site Maps",
-      "add_sitemap": "Add Site Map"
+      "sitemaps": "Websites",
+      "add_sitemap": "Website Map",
+      "sitemap_placeholder": "Enter Website Map URL"
     }
   }
 }

src/renderer/src/i18n/locales/ru-ru.json

@@ -560,8 +560,9 @@
       "no_bases": "База знаний не найдена",
       "clear_selection": "Очистить выбор",
       "delete_confirm": "Вы уверены, что хотите удалить эту базу знаний?",
-      "sitemaps": "Карта сайта",
-      "add_sitemap": "Добавить карту сайта"
+      "sitemaps": "Сайты",
+      "add_sitemap": "Карта сайта",
+      "sitemap_placeholder": "Введите URL карты сайта"
     }
   }
 }

src/renderer/src/i18n/locales/zh-cn.json

@@ -549,8 +549,9 @@
       "no_bases": "暂无知识库",
       "clear_selection": "清除选择",
       "delete_confirm": "确定要删除此知识库吗?",
-      "sitemaps": "站点地图",
-      "add_sitemap": "添加站点地图"
+      "sitemaps": "网站",
+      "add_sitemap": "站点地图",
+      "sitemap_placeholder": "请输入站点地图 URL"
     }
   }
 }

src/renderer/src/i18n/locales/zh-tw.json

@@ -548,8 +548,9 @@
       "no_bases": "暫無知識庫",
       "clear_selection": "清除選擇",
       "delete_confirm": "確定要刪除此知識庫嗎?",
-      "sitemaps": "站點地圖",
-      "add_sitemap": "添加站點地圖"
+      "sitemaps": "網站",
+      "add_sitemap": "網站地圖",
+      "sitemap_placeholder": "請輸入網站地圖 URL"
     }
   }
 }

src/renderer/src/pages/knowledge/KnowledgePage.tsx

@@ -27,7 +27,7 @@ const KnowledgePage: FC = () => {
       if (!selectedBase) {
         return setSelectedBase(bases[0])
       }
-      if (selectedBase && !bases.includes(selectedBase)) {
+      if (selectedBase && !bases.find((base) => base.id === selectedBase.id)) {
         return setSelectedBase(bases[0])
       }
     }

src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx

@@ -2,7 +2,7 @@ import { TopView } from '@renderer/components/TopView'
 import { isEmbeddingModel } from '@renderer/config/models'
 import { useKnowledgeBases } from '@renderer/hooks/useknowledge'
 import { useProviders } from '@renderer/hooks/useProvider'
-import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
+import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService'
 import { getModelUniqId } from '@renderer/services/ModelService'
 import { Model } from '@renderer/types'
 import { Form, Input, Modal, Select } from 'antd'
@@ -64,7 +64,7 @@ const PopupContainer: React.FC<Props> = ({ title, resolve }) => {
           updated_at: Date.now()
         }
 
-        await window.api.knowledgeBase.create(getRagAppRequestParams(newBase))
+        await window.api.knowledgeBase.create(getKnowledgeBaseParams(newBase))
 
         addKnowledgeBase(newBase as any)
         setOpen(false)

src/renderer/src/pages/knowledge/components/KnowledgeSearchPopup.tsx

@@ -1,6 +1,6 @@
 import { ExtractChunkData } from '@llm-tools/embedjs-interfaces'
 import { TopView } from '@renderer/components/TopView'
-import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
+import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService'
 import { KnowledgeBase } from '@renderer/types'
 import { Input, List, Modal, Spin, Typography } from 'antd'
 import { useState } from 'react'
@@ -37,7 +37,7 @@ const PopupContainer: React.FC<Props> = ({ base, resolve }) => {
     try {
       const searchResults = await window.api.knowledgeBase.search({
         search: value,
-        config: getRagAppRequestParams(base)
+        base: getKnowledgeBaseParams(base)
       })
       setResults(searchResults)
     } catch (error) {

src/renderer/src/providers/BaseProvider.ts

@@ -1,5 +1,5 @@
 import { getOllamaKeepAliveTime } from '@renderer/hooks/useOllama'
-import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
+import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService'
 import store from '@renderer/store'
 import { Assistant, FileType, Message, Provider, Suggestion } from '@renderer/types'
 import { delay } from '@renderer/utils'
@@ -96,7 +96,7 @@ export default abstract class BaseProvider {
 
     const searchResults = await window.api.knowledgeBase.search({
       search: message.content,
-      config: getRagAppRequestParams(base)
+      base: getKnowledgeBaseParams(base)
     })
 
     const references = take(searchResults, 5)