Refactor text processing in BestRAG and update version to 0.3.0

Signed-off-by: samadpls <abdulsamadsid1@gmail.com>
samadpls · Nov 30, 2024 · 31884c0 · 31884c0
1 parent 33d7a2f
commit 31884c0
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 1 deletion.
diff --git a/bestrag/best_rag.py b/bestrag/best_rag.py
@@ -90,6 +90,13 @@ def _clean_text(self, text: str) -> str:
         text = text.replace('\n', ' ')
         text = re.sub(r'\s{2,}', ' ', text)
         text = re.sub(r'(\d+\.)\s', r'\n\1 ', text)
+        text = re.sub(r'[●■○]', '', text)
+        text = re.sub(r'[""''«»]', '"', text)
+        text = re.sub(r'[–—−]', '-', text)
+        text = re.sub(r'[^\x00-\x7F]+', '', text)
+        text = re.sub(r'[\u200B-\u200D\uFEFF]', '', text)
+        text = re.sub(r'\s+', ' ', text)
+
         return text.strip()
 
     def _get_dense_embedding(self, text: str):

diff --git a/setup.py b/setup.py
@@ -5,7 +5,7 @@
 
 setup(
     name="bestrag",
-    version="0.2.1",
+    version="0.3.0",
     description="bestrag: Library for storing and searching document embeddings in a Qdrant vector database using hybrid embedding techniques.",
     author="samadpls",
     author_email="abdulsamadsid1@gmail.com",