build: add matrix

.github/workflows/build.yml

@@ -0,0 +1,94 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version (e.g. v1.2.3)'
+        required: true
+        type: string
+  push:
+    tags:
+      - v*.*.*
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [macos-13, macos-latest]
+        arch: [x64, arm64]
+        exclude:
+          - os: windows-latest
+            arch: arm64
+          - os: macos-latest
+            arch: x64
+          - os: macos-13
+            arch: arm64
+
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v3
+
+      - name: Install Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          arch: ${{ matrix.arch }}
+
+      - name: Install corepack
+        run: corepack enable && corepack prepare yarn@4.3.1 --activate
+
+      - name: Install Dependencies
+        run: yarn install
+
+      - name: Build Linux
+        if: matrix.os == 'ubuntu-latest'
+        run: yarn build:linux
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+          ARCH: ${{ matrix.arch }}
+
+      - name: Build Mac
+        if: matrix.os == 'macos-13'
+        run: yarn build:mac
+        env:
+          CSC_LINK: ${{ secrets.CSC_LINK }}
+          CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }}
+          APPLE_ID: ${{ vars.APPLE_ID }}
+          APPLE_APP_SPECIFIC_PASSWORD: ${{ vars.APPLE_APP_SPECIFIC_PASSWORD }}
+          APPLE_TEAM_ID: ${{ vars.APPLE_TEAM_ID }}
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+          ARCH: ${{ matrix.arch }}
+
+      - name: Build Windows
+        if: matrix.os == 'windows-latest'
+        run: yarn build:win
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+          ARCH: ${{ matrix.arch }}
+
+      - name: Replace spaces in filenames
+        run: node scripts/replaceSpaces.js
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          draft: true
+          files: |
+            dist/*.exe
+            dist/*.zip
+            dist/*.dmg
+            dist/*.AppImage
+            dist/*.snap
+            dist/*.deb
+            dist/*.rpm
+            dist/*.tar.gz
+            dist/latest*.yml
+            dist/*.blockmap
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}

.github/workflows/release.yml

@@ -20,7 +20,15 @@ jobs:
 
     strategy:
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-latest]
+        os: [macos-13, macos-latest, windows-latest, ubuntu-latest]
+        arch: [x64, arm64]
+        exclude:
+          - os: windows-latest
+            arch: arm64
+          - os: macos-latest
+            arch: x64
+          - os: macos-13
+            arch: arm64
 
     steps:
       - name: Check out Git repository
@@ -30,6 +38,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 20
+          arch: ${{ matrix.arch }}
 
       - name: Install corepack
         run: corepack enable && corepack prepare yarn@4.3.1 --activate
@@ -42,9 +51,10 @@ jobs:
         run: yarn build:linux
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
+          ARCH: ${{ matrix.arch }}
 
       - name: Build Mac
-        if: matrix.os == 'macos-latest'
+        if: matrix.os == 'macos-13'
         run: yarn build:mac
         env:
           CSC_LINK: ${{ secrets.CSC_LINK }}
@@ -53,12 +63,14 @@ jobs:
           APPLE_APP_SPECIFIC_PASSWORD: ${{ vars.APPLE_APP_SPECIFIC_PASSWORD }}
           APPLE_TEAM_ID: ${{ vars.APPLE_TEAM_ID }}
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
+          ARCH: ${{ matrix.arch }}
 
       - name: Build Windows
         if: matrix.os == 'windows-latest'
         run: yarn build:win
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
+          ARCH: ${{ matrix.arch }}
 
       - name: Replace spaces in filenames
         run: node scripts/replaceSpaces.js

electron-builder.yml

@@ -13,6 +13,7 @@ files:
   - '!local'
 asarUnpack:
   - resources/**
+  - '**/*.{node,dll,metal,exp,lib}'
 win:
   executableName: Cherry Studio
 nsis:

electron.vite.config.ts

@@ -16,6 +16,7 @@ export default defineConfig({
           '@llm-tools/embedjs-loader-msoffice',
           '@llm-tools/embedjs-loader-xml',
           '@llm-tools/embedjs-loader-pdf',
+          '@llm-tools/embedjs-loader-sitemap',
           '@lancedb/lancedb'
         ]
       })
@@ -29,7 +30,7 @@ export default defineConfig({
     },
     build: {
       rollupOptions: {
-        external: ['@lancedb/lancedb', '@llm-tools/embedjs-loader-sitemap']
+        external: ['@lancedb/lancedb']
       }
     }
   },

package.json

@@ -28,9 +28,14 @@
     "build": "npm run typecheck && electron-vite build",
     "postinstall": "electron-builder install-app-deps",
     "build:unpack": "dotenv npm run build && electron-builder --dir",
-    "build:win": "dotenv npm run build && electron-builder --win --publish never",
-    "build:mac": "dotenv electron-vite build && electron-builder --mac --publish never",
-    "build:linux": "dotenv electron-vite build && electron-builder --linux --publish never",
+    "build:win": "dotenv npm run build && electron-builder --win --$ARCH",
+    "build:win:x64": "dotenv npm run build && electron-builder --win --x64",
+    "build:mac": "dotenv electron-vite build && electron-builder --mac --$ARCH",
+    "build:mac:arm64": "dotenv electron-vite build && electron-builder --mac --arm64",
+    "build:mac:x64": "dotenv electron-vite build && electron-builder --mac --x64",
+    "build:linux": "dotenv electron-vite build && electron-builder --linux --$ARCH",
+    "build:linux:arm64": "dotenv electron-vite build && electron-builder --linux --arm64",
+    "build:linux:x64": "dotenv electron-vite build && electron-builder --linux --x64",
     "release": "node scripts/version.js",
     "publish": "yarn release patch push",
     "pulish:artifacts": "cd packages/artifacts && npm publish && cd -",
@@ -40,12 +45,14 @@
   "dependencies": {
     "@electron-toolkit/preload": "^3.0.0",
     "@electron-toolkit/utils": "^3.0.0",
+    "@electron/notarize": "^2.5.0",
     "@llm-tools/embedjs": "^0.1.24",
     "@llm-tools/embedjs-lancedb": "^0.1.24",
     "@llm-tools/embedjs-loader-csv": "^0.1.24",
     "@llm-tools/embedjs-loader-markdown": "^0.1.24",
     "@llm-tools/embedjs-loader-msoffice": "^0.1.24",
     "@llm-tools/embedjs-loader-pdf": "^0.1.24",
+    "@llm-tools/embedjs-loader-sitemap": "^0.1.24",
     "@llm-tools/embedjs-loader-web": "^0.1.24",
     "@llm-tools/embedjs-loader-xml": "^0.1.24",
     "@llm-tools/embedjs-ollama": "^0.1.24",
@@ -138,7 +145,6 @@
     "react-dom": "^17.0.0 || ^18.0.0"
   },
   "resolutions": {
-    "@electron/notarize@npm:2.2.1": "patch:@electron/notarize@npm%3A2.3.2#~/.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch",
     "pdf-parse@npm:1.1.1": "patch:pdf-parse@npm%3A1.1.1#~/.yarn/patches/pdf-parse-npm-1.1.1-04a6109b2a.patch"
   },
   "packageManager": "yarn@4.5.0"

src/main/services/KnowledgeService.ts

@@ -7,9 +7,10 @@ import { LanceDb } from '@llm-tools/embedjs-lancedb'
 import { MarkdownLoader } from '@llm-tools/embedjs-loader-markdown'
 import { DocxLoader } from '@llm-tools/embedjs-loader-msoffice'
 import { PdfLoader } from '@llm-tools/embedjs-loader-pdf'
+import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap'
 import { WebLoader } from '@llm-tools/embedjs-loader-web'
 import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'
-import { FileType, RagAppRequestParams } from '@types'
+import { FileType, KnowledgeBaseParams, KnowledgeItem } from '@types'
 import { app } from 'electron'
 
 class KnowledgeService {
@@ -25,7 +26,7 @@ class KnowledgeService {
     }
   }
 
-  private getRagApplication = async ({ id, model, apiKey, baseURL }: RagAppRequestParams): Promise<RAGApplication> => {
+  private getRagApplication = async ({ id, model, apiKey, baseURL }: KnowledgeBaseParams): Promise<RAGApplication> => {
     return new RAGApplicationBuilder()
       .setModel('NO_MODEL')
       .setEmbeddingModel(
@@ -42,13 +43,13 @@ class KnowledgeService {
 
   public create = async (
     _: Electron.IpcMainInvokeEvent,
-    { id, model, apiKey, baseURL }: RagAppRequestParams
+    { id, model, apiKey, baseURL }: KnowledgeBaseParams
   ): Promise<void> => {
     this.getRagApplication({ id, model, apiKey, baseURL })
   }
 
-  public reset = async (_: Electron.IpcMainInvokeEvent, { config }: { config: RagAppRequestParams }): Promise<void> => {
-    const ragApplication = await this.getRagApplication(config)
+  public reset = async (_: Electron.IpcMainInvokeEvent, { base }: { base: KnowledgeBaseParams }): Promise<void> => {
+    const ragApplication = await this.getRagApplication(base)
     await ragApplication.reset()
   }
 
@@ -61,45 +62,59 @@ class KnowledgeService {
 
   public add = async (
     _: Electron.IpcMainInvokeEvent,
-    { data, config }: { data: string | FileType; config: RagAppRequestParams }
+    { base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem }
   ): Promise<AddLoaderReturn> => {
-    const ragApplication = await this.getRagApplication(config)
+    const ragApplication = await this.getRagApplication(base)
 
-    if (typeof data === 'string') {
-      if (data.startsWith('http')) {
-        return await ragApplication.addLoader(new WebLoader({ urlOrContent: data }))
+    if (item.type === 'url') {
+      const content = item.content as string
+      if (content.startsWith('http')) {
+        return await ragApplication.addLoader(new WebLoader({ urlOrContent: content }))
       }
-      return await ragApplication.addLoader(new TextLoader({ text: data }))
     }
 
-    if (data.ext === '.pdf') {
-      return await ragApplication.addLoader(new PdfLoader({ filePathOrUrl: data.path }) as any)
+    if (item.type === 'sitemap') {
+      const content = item.content as string
+      return await ragApplication.addLoader(new SitemapLoader({ url: content }))
     }
 
-    if (data.ext === '.docx') {
-      return await ragApplication.addLoader(new DocxLoader({ filePathOrUrl: data.path }) as any)
+    if (item.type === 'note') {
+      const content = item.content as string
+      return await ragApplication.addLoader(new TextLoader({ text: content }))
     }
 
-    if (data.ext === '.md' || data.ext === '.mdx') {
-      return await ragApplication.addLoader(new MarkdownLoader({ filePathOrUrl: data.path }) as any)
+    if (item.type === 'file') {
+      const file = item.content as FileType
+
+      if (file.ext === '.pdf') {
+        return await ragApplication.addLoader(new PdfLoader({ filePathOrUrl: file.path }) as any)
+      }
+
+      if (file.ext === '.docx') {
+        return await ragApplication.addLoader(new DocxLoader({ filePathOrUrl: file.path }) as any)
+      }
+
+      if (file.ext.startsWith('.md')) {
+        return await ragApplication.addLoader(new MarkdownLoader({ filePathOrUrl: file.path }) as any)
+      }
     }
 
     return { entriesAdded: 0, uniqueId: '', loaderType: '' }
   }
 
   public remove = async (
     _: Electron.IpcMainInvokeEvent,
-    { uniqueId, config }: { uniqueId: string; config: RagAppRequestParams }
+    { uniqueId, base }: { uniqueId: string; base: KnowledgeBaseParams }
   ): Promise<void> => {
-    const ragApplication = await this.getRagApplication(config)
+    const ragApplication = await this.getRagApplication(base)
     await ragApplication.deleteLoader(uniqueId)
   }
 
   public search = async (
     _: Electron.IpcMainInvokeEvent,
-    { search, config }: { search: string; config: RagAppRequestParams }
+    { search, base }: { search: string; base: KnowledgeBaseParams }
   ): Promise<ExtractChunkData[]> => {
-    const ragApplication = await this.getRagApplication(config)
+    const ragApplication = await this.getRagApplication(base)
     return await ragApplication.search(search)
   }
 }