fix: Weaviate fix

api/delete.py

@@ -16,4 +16,4 @@ async def delete(payload: RequestPayload):
         encoder=encoder,
     )
     data = await vector_service.delete(file_url=payload.file_url)
-    return {"success": True, "data": data}
+    return ResponsePayload(success=True, data=data)

api/ingest.py

@@ -24,13 +24,13 @@ async def ingest(payload: RequestPayload) -> Dict:
     )
 
     await asyncio.gather(
-        embedding_service.generate_embeddings(
+        embedding_service.generate_and_upsert_embeddings(
             documents=chunks, encoder=encoder, index_name=payload.index_name
         ),
-        embedding_service.generate_embeddings(
+        embedding_service.generate_and_upsert_embeddings(
             documents=summary_documents,
             encoder=encoder,
-            index_name=f"{payload.index_name}-summary",
+            index_name=f"{payload.index_name}summary",  # TODO maybe with '-summary'?
         ),
     )
 

dev/walkthrough.ipynb

@@ -35,7 +35,7 @@
     "    \"files\": [\n",
     "        {\n",
     "            \"type\": \"PDF\",\n",
-    "            \"url\": \"https://arxiv.org/pdf/2210.03629.pdf\"\n",
+    "            \"url\": \"https://arxiv.org/pdf/2402.05131.pdf\"\n",
     "        }\n",
     "    ],\n",
     "    \"vector_database\": {\n",
@@ -54,6 +54,39 @@
     "print(response.json())"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Ingest a file\n",
+    "url = f\"{API_URL}/api/v1/ingest\"\n",
+    "\n",
+    "payload = {\n",
+    "    \"files\": [\n",
+    "        {\n",
+    "            \"type\": \"PDF\",\n",
+    "            \"url\": \"https://arxiv.org/pdf/2402.05131.pdf\"\n",
+    "        }\n",
+    "    ],\n",
+    "    \"vector_database\": {\n",
+    "        \"type\": \"weaviate\",\n",
+    "        \"config\": {\n",
+    "            \"api_key\": \"9eXH8oNR0uqN3GvvzAgaUD11ltPnGqZG2RFQ\",\n",
+    "            \"host\": \"https://superagent-ragas-1575sjfq.weaviate.network\"\n",
+    "        }\n",
+    "    },\n",
+    "    \"index_name\": \"homanp11\",\n",
+    "    \"encoder\": \"cohere\",\n",
+    "    \"webhook_url\": \"https://webhook.site/0e217d1c-49f1-424a-9992-497db09f7793\"\n",
+    "}\n",
+    "\n",
+    "response = requests.post(url, json=payload)\n",
+    "\n",
+    "print(response.json())"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -64,7 +97,7 @@
     "query_url = f\"{API_URL}/api/v1/query\"\n",
     "\n",
     "query_payload = {\n",
-    "    \"input\": \"What is CoT?\",\n",
+    "    \"input\": \"What are the chunking strategies?\",\n",
     "    \"vector_database\": {\n",
     "        \"type\": \"pinecone\",\n",
     "        \"config\": {\n",
@@ -81,6 +114,33 @@
     "print(query_response.json())"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Query the index\n",
+    "query_url = f\"{API_URL}/api/v1/query\"\n",
+    "\n",
+    "query_payload = {\n",
+    "    \"input\": \"What are the chunking strategies?\",\n",
+    "    \"vector_database\": {\n",
+    "        \"type\": \"weaviate\",\n",
+    "        \"config\": {\n",
+    "            \"api_key\": \"9eXH8oNR0uqN3GvvzAgaUD11ltPnGqZG2RFQ\",\n",
+    "            \"host\": \"https://superagent-ragas-1575sjfq.weaviate.network\"\n",
+    "        }\n",
+    "    },\n",
+    "    \"index_name\": \"homanp11\",\n",
+    "    \"encoder\": \"cohere\",\n",
+    "}\n",
+    "\n",
+    "query_response = requests.post(query_url, json=query_payload)\n",
+    "\n",
+    "print(query_response.json())"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -127,6 +187,33 @@
     "\n",
     "print(delete_response.json())"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Delete the index\n",
+    "query_url = f\"{API_URL}/api/v1/delete\"\n",
+    "\n",
+    "delete_payload = {\n",
+    "    \"file_url\": \"https://arxiv.org/pdf/2402.05131.pdf\",\n",
+    "    \"vector_database\": {\n",
+    "        \"type\": \"weaviate\",\n",
+    "        \"config\": {\n",
+    "            \"api_key\": \"9eXH8oNR0uqN3GvvzAgaUD11ltPnGqZG2RFQ\",\n",
+    "            \"host\": \"https://superagent-ragas-1575sjfq.weaviate.network\"\n",
+    "        }\n",
+    "    },\n",
+    "    \"index_name\": \"homanp11\",\n",
+    "    \"encoder\": \"cohere\"\n",
+    "}\n",
+    "\n",
+    "delete_response = requests.delete(query_url, json=delete_payload)\n",
+    "\n",
+    "print(delete_response.json())"
+   ]
   }
  ],
  "metadata": {

models/delete.py

@@ -11,6 +11,10 @@ class RequestPayload(BaseModel):
     encoder: EncoderEnum
 
 
+class DeleteResponse(BaseModel):
+    num_of_deleted_chunks: int
+
+
 class ResponsePayload(BaseModel):
     success: bool
-    data: dict = {}
+    data: DeleteResponse

models/document.py

@@ -1,4 +1,7 @@
-from pydantic import BaseModel
+import uuid
+from typing import List, Optional
+
+from pydantic import BaseModel, validator
 
 
 class BaseDocument(BaseModel):
@@ -8,7 +11,27 @@ class BaseDocument(BaseModel):
     metadata: dict | None = None
 
 
-class BaseDocumentChunk(BaseDocument):
+class BaseDocumentChunk(BaseModel):
+    id: str
     document_id: str
+    content: str
+    doc_url: str
+    metadata: dict | None = None
     page_number: str = ""
-    dense_embedding: list[float] | None = None
+    dense_embedding: Optional[List[float]] = None
+
+    @validator("id")
+    def id_must_be_valid_uuid(cls, v):
+        try:
+            uuid_obj = uuid.UUID(v, version=4)
+            return str(uuid_obj)
+        except ValueError:
+            raise ValueError("id must be a valid UUID")
+
+    @validator("dense_embedding")
+    def embeddings_must_be_list_of_floats(cls, v):
+        if v is None:
+            return v  # Allow None to pass through
+        if not all(isinstance(item, float) for item in v):
+            raise ValueError("embeddings must be a list of floats")
+        return v