feat: ignore irrelevant documents

Author: winstxnhdw

server/config.py

@@ -48,6 +48,7 @@ class Config(BaseSettings):
     document_index_prefix: str = 'doc'
     document_index_tag: str = 'chat_id'
     embedding_dimensions: int = 768
+    minimum_similarity_score: float = 0.3
 
     redis_host: str = 'redis'
     redis_port: int = 6379

server/databases/redis/wrapper.py

@@ -22,11 +22,13 @@ class Mappings(TypedDict):
 
     Attributes
     ----------
+    score (float) : the document's score
     vector (bytes) : the document's vector
     content (str) : the document's content
     chat_id (str) : the document's tag
     """
 
+    score: float
     vector: bytes
     content: str
     chat_id: str
@@ -142,7 +144,11 @@ async def search(self, search_field: str, embedding: NDArray[float32], top_k: in
             redis_query_parameters,  # type: ignore  (this is a bug in the redis-py library)
         )
 
-        return ' '.join(document['content'] for document in search_response.docs)
+        return ' '.join(
+            document['content']
+            for document in search_response.docs
+            if float(document['score']) > Config.minimum_similarity_score
+        )
 
     async def save_messages(
         self,