Nolan/list collection (#1568)

* Check in

* More

* Fix nested transactions issue in sqlite logger

* Fix update collection return type

js/sdk/examples/data/marmeladov.txt

@@ -19,4 +19,4 @@ that question to him. “Why am I not at my duty? Does not my heart ache
 to think what a useless worm I am? A month ago when Mr. Lebeziatnikov
 beat my wife with his own hands, and I lay drunk, didn’t I suffer?
 Excuse me, young man, has it ever happened to you... hm... well, to
-petition hopelessly for a loan?”
+petition hopelessly for a loan?”

py/core/base/__init__.py

@@ -136,7 +136,6 @@
     "generate_document_id",
     "generate_extraction_id",
     "generate_default_user_collection_id",
-    "generate_id_from_label",
     "generate_user_id",
     "increment_version",
     "EntityType",

py/core/base/api/models/__init__.py

@@ -36,7 +36,6 @@
 from shared.api.models.management.responses import (
     AnalyticsResponse,
     AppSettingsResponse,
-    CollectionOverviewResponse,
     CollectionResponse,
     ConversationOverviewResponse,
     DocumentChunkResponse,
@@ -49,9 +48,8 @@
     WrappedAddUserResponse,
     WrappedAnalyticsResponse,
     WrappedAppSettingsResponse,
-    WrappedCollectionListResponse,
-    WrappedCollectionOverviewResponse,
     WrappedCollectionResponse,
+    WrappedCollectionsResponse,
     WrappedConversationResponse,
     WrappedConversationsOverviewResponse,
     WrappedDeleteResponse,
@@ -117,7 +115,6 @@
     "DocumentOverviewResponse",
     "DocumentChunkResponse",
     "CollectionResponse",
-    "CollectionOverviewResponse",
     "ConversationOverviewResponse",
     "WrappedPromptMessageResponse",
     "WrappedServerStatsResponse",
@@ -130,14 +127,13 @@
     "WrappedDocumentOverviewResponse",
     "WrappedDocumentChunkResponse",
     "WrappedCollectionResponse",
+    "WrappedCollectionsResponse",
     "WrappedDocumentChunkResponse",
-    "WrappedCollectionListResponse",
     "WrappedAddUserResponse",
     "WrappedUsersInCollectionResponse",
     "WrappedGetPromptsResponse",
     "WrappedUserCollectionResponse",
     "WrappedDocumentChunkResponse",
-    "WrappedCollectionOverviewResponse",
     "WrappedDeleteResponse",
     "WrappedConversationsOverviewResponse",
     "WrappedMessageResponse",

py/core/base/providers/database.py

@@ -42,14 +42,12 @@
     VectorTableName,
 )
 from core.base.api.models import (
-    CollectionOverviewResponse,
     CollectionResponse,
     KGCreationEstimationResponse,
     KGDeduplicationEstimationResponse,
     KGEnrichmentEstimationResponse,
     UserResponse,
 )
-from core.base.utils import _decorate_vector_type
 
 from ..logger import RunInfoLog
 from ..logger.base import RunType
@@ -257,29 +255,20 @@ async def get_document_ids_by_status(
 
 
 class CollectionHandler(Handler):
-    @abstractmethod
-    async def create_default_collection(
-        self, user_id: Optional[UUID] = None
-    ) -> CollectionResponse:
-        pass
-
     @abstractmethod
     async def collection_exists(self, collection_id: UUID) -> bool:
         pass
 
     @abstractmethod
     async def create_collection(
         self,
-        name: str,
+        user_id: UUID,
+        name: Optional[str] = None,
         description: str = "",
         collection_id: Optional[UUID] = None,
     ) -> CollectionResponse:
         pass
 
-    @abstractmethod
-    async def get_collection(self, collection_id: UUID) -> CollectionResponse:
-        pass
-
     @abstractmethod
     async def update_collection(
         self,
@@ -293,19 +282,6 @@ async def update_collection(
     async def delete_collection_relational(self, collection_id: UUID) -> None:
         pass
 
-    @abstractmethod
-    async def list_collections(
-        self, offset: int, limit: int
-    ) -> dict[str, Any]:
-        """List collections with pagination."""
-        pass
-
-    @abstractmethod
-    async def get_collections_by_ids(
-        self, collection_ids: list[UUID]
-    ) -> list[CollectionResponse]:
-        pass
-
     @abstractmethod
     async def documents_in_collection(
         self, collection_id: UUID, offset: int, limit: int
@@ -317,13 +293,9 @@ async def get_collections_overview(
         self,
         offset: int,
         limit: int,
-        collection_ids: Optional[list[UUID]] = None,
-    ) -> dict[str, Union[list[CollectionOverviewResponse], int]]:
-        pass
-
-    @abstractmethod
-    async def get_collections_for_user(
-        self, user_id: UUID, offset: int, limit: int
+        filter_user_ids: Optional[list[UUID]] = None,
+        filter_document_ids: Optional[list[UUID]] = None,
+        filter_collection_ids: Optional[list[UUID]] = None,
     ) -> dict[str, Union[list[CollectionResponse], int]]:
         pass
 
@@ -335,12 +307,6 @@ async def assign_document_to_collection_relational(
     ) -> UUID:
         pass
 
-    @abstractmethod
-    async def document_collections(
-        self, document_id: UUID, offset: int, limit: int
-    ) -> dict[str, Union[list[CollectionResponse], int]]:
-        pass
-
     @abstractmethod
     async def remove_document_from_collection_relational(
         self, document_id: UUID, collection_id: UUID
@@ -1158,27 +1124,23 @@ async def get_document_ids_by_status(
         )
 
     # Collection handler methods
-    async def create_default_collection(
-        self, user_id: Optional[UUID] = None
-    ) -> CollectionResponse:
-        return await self.collection_handler.create_default_collection(user_id)
-
     async def collection_exists(self, collection_id: UUID) -> bool:
         return await self.collection_handler.collection_exists(collection_id)
 
     async def create_collection(
         self,
-        name: str,
+        user_id: UUID,
+        name: Optional[str] = None,
         description: str = "",
         collection_id: Optional[UUID] = None,
     ) -> CollectionResponse:
         return await self.collection_handler.create_collection(
-            name, description, collection_id
+            user_id=user_id,
+            name=name,
+            description=description,
+            collection_id=collection_id,
         )
 
-    async def get_collection(self, collection_id: UUID) -> CollectionResponse:
-        return await self.collection_handler.get_collection(collection_id)
-
     async def update_collection(
         self,
         collection_id: UUID,
@@ -1194,18 +1156,6 @@ async def delete_collection_relational(self, collection_id: UUID) -> None:
             collection_id
         )
 
-    async def list_collections(
-        self, offset: int, limit: int
-    ) -> dict[str, Any]:
-        return await self.collection_handler.list_collections(offset, limit)
-
-    async def get_collections_by_ids(
-        self, collection_ids: list[UUID]
-    ) -> list[CollectionResponse]:
-        return await self.collection_handler.get_collections_by_ids(
-            collection_ids
-        )
-
     async def documents_in_collection(
         self, collection_id: UUID, offset: int, limit: int
     ) -> dict[str, Union[list[DocumentInfo], int]]:
@@ -1217,19 +1167,16 @@ async def get_collections_overview(
         self,
         offset: int,
         limit: int,
-        collection_ids: Optional[list[UUID]] = None,
-    ) -> dict[str, Union[list[CollectionOverviewResponse], int]]:
+        filter_user_ids: Optional[list[UUID]] = None,
+        filter_document_ids: Optional[list[UUID]] = None,
+        filter_collection_ids: Optional[list[UUID]] = None,
+    ) -> dict[str, Union[list[CollectionResponse], int]]:
         return await self.collection_handler.get_collections_overview(
             offset=offset,
             limit=limit,
-            collection_ids=collection_ids,
-        )
-
-    async def get_collections_for_user(
-        self, user_id: UUID, offset: int, limit: int
-    ) -> dict[str, Union[list[CollectionResponse], int]]:
-        return await self.collection_handler.get_collections_for_user(
-            user_id, offset, limit
+            filter_user_ids=filter_user_ids,
+            filter_document_ids=filter_document_ids,
+            filter_collection_ids=filter_collection_ids,
         )
 
     async def assign_document_to_collection_relational(
@@ -1241,13 +1188,6 @@ async def assign_document_to_collection_relational(
             document_id, collection_id
         )
 
-    async def document_collections(
-        self, document_id: UUID, offset: int, limit: int
-    ) -> dict[str, Union[list[CollectionResponse], int]]:
-        return await self.collection_handler.document_collections(
-            document_id, offset, limit
-        )
-
     async def remove_document_from_collection_relational(
         self, document_id: UUID, collection_id: UUID
     ) -> None:

py/core/base/utils/__init__.py

@@ -12,7 +12,6 @@
     generate_document_id,
     generate_extraction_id,
     generate_id,
-    generate_id_from_label,
     generate_user_id,
     increment_version,
     llm_cost_per_million_tokens,
@@ -35,7 +34,6 @@
     "generate_document_id",
     "generate_extraction_id",
     "generate_user_id",
-    "generate_id_from_label",
     "generate_default_prompt_id",
     "RecursiveCharacterTextSplitter",
     "TextSplitter",