Merge pull request #615 from sugarforever/feature/chunk-setting

Feature/chunk setting

Author: sugarforever

components/KnowledgeBaseForm.vue

@@ -9,7 +9,15 @@ const props = defineProps<{
   title: string
   type: OperateType
   data?: KnowledgeBase
-  embeddings: string[]
+  embeddings: string[],
+  chunking: {
+    parentChunkSize: number,
+    parentChunkOverlap: number,
+    childChunkSize: number,
+    childChunkOverlap: number,
+    parentK: number,
+    childK: number,
+  }
   onSuccess: () => void
   onClose: () => void
 }>()
@@ -28,6 +36,14 @@ const state = reactive({
   pageParser: 'default' as PageParser,
   maxDepth: 0,
   excludeGlobs: '',
+  chunking: {
+    parentChunkSize: props.chunking.parentChunkSize,
+    parentChunkOverlap: props.chunking.parentChunkOverlap,
+    childChunkSize: props.chunking.childChunkSize,
+    childChunkOverlap: props.chunking.childChunkOverlap,
+    parentK: props.chunking.parentK,
+    childK: props.chunking.childK,
+  }
 })
 const loading = ref(false)
 const isModify = computed(() => props.type === 'update')
@@ -81,6 +97,7 @@ async function onSubmit() {
   formData.append("pageParser", state.pageParser)
   formData.append("maxDepth", String(state.maxDepth))
   formData.append("excludeGlobs", state.excludeGlobs)
+  formData.append("chunking", JSON.stringify(state.chunking))
 
   if (isModify.value) {
     formData.append('knowledgeBaseId', String(props.data!.id))
@@ -168,6 +185,29 @@ function generateEmbeddingData(groupName: string, list: string[], slotName: stri
           <UTextarea autoresize :maxrows="4" v-model="state.description" />
         </UFormGroup>
 
+        <UFormGroup :label="t('knowledgeBases.chunking')" name="chunking" class="mb-4">
+          <div class="grid grid-cols-2 gap-4">
+            <UFormGroup label="Parent Chunk Size" name="parentChunkSize">
+              <UInput v-model="state.chunking.parentChunkSize" type="number" />
+            </UFormGroup>
+            <UFormGroup label="Parent Chunk Overlap" name="parentChunkOverlap">
+              <UInput v-model="state.chunking.parentChunkOverlap" type="number" />
+            </UFormGroup>
+            <UFormGroup label="Child Chunk Size" name="childChunkSize">
+              <UInput v-model="state.chunking.childChunkSize" type="number" />
+            </UFormGroup>
+            <UFormGroup label="Child Chunk Overlap" name="childChunkOverlap">
+              <UInput v-model="state.chunking.childChunkOverlap" type="number" />
+            </UFormGroup>
+            <UFormGroup label="Parent K" name="parentK">
+              <UInput v-model="state.chunking.parentK" type="number" />
+            </UFormGroup>
+            <UFormGroup label="Child K" name="childK">
+              <UInput v-model="state.chunking.childK" type="number" />
+            </UFormGroup>
+          </div>
+        </UFormGroup>
+
         <UFormGroup :label="t('knowledgeBases.publicAccessible')" name="public" class="mb-4">
           <p class="text-xs mb-2 text-pink-600 dark:text-pink-300">{{ t("knowledgeBases.publicAccessibleTip") }}</p>
           <UToggle v-model="state.isPublic" />

locales/en-US.json

@@ -48,6 +48,13 @@
     "NoOfFiles": "No. Of Files",
     "description": "Description",
     "embedding": "Embedding",
+    "chunking": "Chunking",
+    "parentChunkSize": "Parent Chunk Size",
+    "parentChunkOverlap": "Parent Chunk Overlap",
+    "childChunkSize": "Child Chunk Size",
+    "childChunkOverlap": "Child Chunk Overlap",
+    "parentK": "Parent K",
+    "childK": "Child K",
     "deleteConfirm": "Are you sure deleting knowledge base {0} ?",
     "deleteTitle": "Delete Knowledge Base",
     "createTitle": "Create a New Knowledge Base",
@@ -186,4 +193,4 @@
     "add": "Add",
     "invalidUrl": "Invalid URL"
   }
-}
+}

locales/zh-CN.json

@@ -48,6 +48,13 @@
     "NoOfFiles": "文件数量",
     "description": "描述",
     "embedding": "嵌入模型",
+    "chunking": "分块",
+    "parentChunkSize": "父级分块大小",
+    "parentChunkOverlap": "父级分块重叠",
+    "childChunkSize": "子级分块大小",
+    "childChunkOverlap": "子级分块重叠",
+    "parentK": "父级K",
+    "childK": "子级K",
     "deleteConfirm": "是否删除知识库：{0}",
     "deleteTitle": "删除知识库",
     "createTitle": "创建新知识库",
@@ -185,4 +192,4 @@
     "selectModels": "选择模型",
     "invalidUrl": "无效的URL"
   }
-}
+}

package.json

@@ -43,7 +43,7 @@
     "@langchain/azure-openai": "^0.0.4",
     "@langchain/cohere": "^0.0.6",
     "@langchain/community": "^0.3.19",
-    "@langchain/core": "^0.3.23",
+    "@langchain/core": "^0.3.37",
     "@langchain/google-genai": "^0.1.5",
     "@langchain/groq": "^0.0.5",
     "@langchain/ollama": "^0.1.3",

pages/knowledgebases/index.vue

@@ -63,6 +63,14 @@ function onShowCreate() {
     type: 'create',
     title: t('knowledgeBases.createTitle'),
     embeddings: embeddings.value,
+    chunking: {
+      parentChunkSize: 3000,
+      parentChunkOverlap: 200,
+      childChunkSize: 1000,
+      childChunkOverlap: 50,
+      parentK: 5,
+      childK: 10,
+    },
     onClose: () => modal.close(),
     onSuccess: () => refresh()
   })
@@ -74,6 +82,14 @@ function onShowUpdate(data: KnowledgeBase) {
     title: t('knowledgeBases.updateTitle'),
     data,
     embeddings: embeddings.value,
+    chunking: {
+      parentChunkSize: data.parentChunkSize || 3000,
+      parentChunkOverlap: data.parentChunkOverlap || 200,
+      childChunkSize: data.childChunkSize || 1000,
+      childChunkOverlap: data.childChunkOverlap || 50,
+      parentK: data.parentK || 5,
+      childK: data.childK || 10,
+    },
     onClose: () => modal.close(),
     onSuccess: () => refresh()
   })