Remove vectors from Pinecone when deleting a document (#240)

* Remove vectors from Pinecone when deleting a document

* Small tweaks

app/api/documents.py

@@ -7,6 +7,7 @@
 from app.lib.auth.prisma import JWTBearer
 from app.lib.documents import upsert_document, valid_ingestion_types
 from app.lib.models.document import Document
+from app.lib.vectorstores.base import VectorStoreBase
 from app.lib.prisma import prisma
 
 logger = logging.getLogger(__name__)
@@ -50,12 +51,6 @@ async def create_document(body: Document, token=Depends(JWTBearer())):
                 to_page=body.to_page,
                 user_id=token["userId"],
             )
-        else:
-            logger.error("Invalid ingestion type")
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail="Invalid ingestion type",
-            )
         return {"success": True, "data": document}
     except Exception as e:
         logger.error("Couldn't create document", exc_info=e)
@@ -117,6 +112,7 @@ async def delete_document(documentId: str, token=Depends(JWTBearer())):
     """Delete a document"""
     try:
         prisma.document.delete(where={"id": documentId})
+        VectorStoreBase().get_database().delete(namespace=documentId)
         return {"success": True, "data": None}
     except Exception as e:
         logger.error("Couldn't delete document with id {documentId}", exc_info=e)

app/lib/vectorstores/pinecone.py

@@ -7,7 +7,7 @@
     environment=config("PINECONE_ENVIRONMENT"),  # next to api key in console
 )
 
-pinecone.Index("superagent")
+index = pinecone.Index("superagent")
 
 
 class PineconeVectorstore:
@@ -23,3 +23,6 @@ def from_existing_index(self, embeddings, namespace):
         return Pinecone.from_existing_index(
             "superagent", embedding=embeddings, namespace=namespace
         )
+
+    def delete(self, namespace):
+        return index.delete(delete_all=True, namespace=namespace)

pyproject.toml

@@ -37,7 +37,6 @@ transformers = "^4.30.2"
 youtube-transcript-api = "^0.6.1"
 replicate = "^0.8.3"
 python-slugify = "^8.0.1"
-langchain = "^0.0.218"
 google-cloud-firestore = "^2.11.1"
 google-auth = "^2.21.0"
 llama-index = "^0.6.38.post1"
@@ -50,6 +49,7 @@ colorlog = "^6.7.0"
 llama-hub = "^0.0.16"
 pygithub = "^1.59.0"
 gitpython = "^3.1.32"
+langchain = "^0.0.252"
 
 
 [build-system]