Skip to content

Commit

Permalink
up
Browse files Browse the repository at this point in the history
  • Loading branch information
emrgnt-cmplxty committed Nov 3, 2024
1 parent 8690ddc commit 33f6b55
Show file tree
Hide file tree
Showing 29 changed files with 1,151 additions and 207 deletions.
6 changes: 4 additions & 2 deletions py/cli/commands/management.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,15 +136,17 @@ async def documents_overview(ctx, document_ids, offset, limit):
help="Should the vector be included in the response chunks",
)
@pass_context
async def document_chunks(ctx, document_id, offset, limit, include_vectors):
async def list_document_chunks(
ctx, document_id, offset, limit, include_vectors
):
"""Get chunks of a specific document."""
client = ctx.obj
if not document_id:
click.echo("Error: Document ID is required.")
return

with timer():
chunks_data = await client.document_chunks(
chunks_data = await client.list_document_chunks(
document_id, offset, limit, include_vectors
)

Expand Down
2 changes: 1 addition & 1 deletion py/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def add_command_with_telemetry(command):
add_command_with_telemetry(management.app_settings)
add_command_with_telemetry(management.users_overview)
add_command_with_telemetry(management.documents_overview)
add_command_with_telemetry(management.document_chunks)
add_command_with_telemetry(management.list_document_chunks)

# Knowledge Graph
add_command_with_telemetry(kg.create_graph)
Expand Down
6 changes: 3 additions & 3 deletions py/core/base/providers/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -548,7 +548,7 @@ async def delete_collection_vector(self, collection_id: UUID) -> None:
pass

@abstractmethod
async def get_document_chunks(
async def list_document_chunks(
self,
document_id: UUID,
offset: int = 0,
Expand Down Expand Up @@ -1442,14 +1442,14 @@ async def delete_collection_vector(self, collection_id: UUID) -> None:
collection_id
)

async def get_document_chunks(
async def list_document_chunks(
self,
document_id: UUID,
offset: int = 0,
limit: int = -1,
include_vectors: bool = False,
) -> dict[str, Any]:
return await self.vector_handler.get_document_chunks(
return await self.vector_handler.list_document_chunks(
document_id, offset, limit, include_vectors
)

Expand Down
16 changes: 8 additions & 8 deletions py/core/main/api/v2/management_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -284,11 +284,11 @@ async def delete_app(
return await self.service.delete(filters=filters_dict)

@self.router.get(
"/download_file/{document_id}", response_class=StreamingResponse
"/download_file/{id}", response_class=StreamingResponse
)
@self.base_endpoint
async def download_file_app(
document_id: str = Path(..., description="Document ID"),
id: str = Path(..., description="Document ID"),
auth_user=Depends(self.service.providers.auth.auth_wrapper),
):
"""
Expand All @@ -297,7 +297,7 @@ async def download_file_app(
# TODO: Add a check to see if the user has access to the file

try:
document_uuid = UUID(document_id)
document_uuid = UUID(id)
except ValueError:
raise R2RException(
status_code=422, message="Invalid document ID format."
Expand Down Expand Up @@ -366,7 +366,7 @@ async def documents_overview_app(
"total_entries": documents_overview_response["total_entries"]
}

@self.router.get("/document_chunks/{document_id}")
@self.router.get("/list_document_chunks/{document_id}")
@self.base_endpoint
async def document_chunks_app(
document_id: str = Path(...),
Expand All @@ -377,11 +377,11 @@ async def document_chunks_app(
) -> WrappedDocumentChunkResponse:
document_uuid = UUID(document_id)

document_chunks = await self.service.document_chunks(
list_document_chunks = await self.service.list_document_chunks(
document_uuid, offset, limit, include_vectors
)

document_chunks_result = document_chunks["results"]
document_chunks_result = list_document_chunks["results"]

if not document_chunks_result:
raise R2RException(
Expand Down Expand Up @@ -411,12 +411,12 @@ async def document_chunks_app(

if not user_has_access and not auth_user.is_superuser:
raise R2RException(
"Only a superuser can arbitrarily call document_chunks.",
"Only a superuser can arbitrarily call list_document_chunks.",
403,
)

return document_chunks_result, { # type: ignore
"total_entries": document_chunks["total_entries"]
"total_entries": list_document_chunks["total_entries"]
}

@self.router.get("/collections_overview")
Expand Down
17 changes: 6 additions & 11 deletions py/core/main/api/v3/chunks_router.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
import logging
from typing import Any, List, Optional, Union
from typing import Any, Optional, Union
from uuid import UUID

from fastapi import Body, Depends, Path, Query
from pydantic import Json
from pydantic import BaseModel, Json

from core.base import (
GenerationConfig,
KGSearchSettings,
Message,
R2RException,
RawChunk,
RunType,
Expand All @@ -26,11 +24,6 @@

from .base_router import BaseRouterV3

from typing import Any, Optional
from uuid import UUID

from pydantic import BaseModel


class ChunkResponse(BaseModel):
document_id: UUID
Expand All @@ -47,6 +40,7 @@ class ChunkIngestionResponse(BaseModel):
message: str
document_id: UUID


logger = logging.getLogger()

MAX_CHUNKS_PER_REQUEST = 1024 * 100
Expand Down Expand Up @@ -111,11 +105,12 @@ def _setup_routes(self):
@self.router.post("/chunks")
@self.base_endpoint
async def create_chunks(
raw_chunks: Json[list[UnprocessedChunk]] = Body(...),
raw_chunks: Json[list[UnprocessedChunk]] = Body(
..., description="List of chunks to create"
),
run_with_orchestration: Optional[bool] = Body(True),
auth_user=Depends(self.providers.auth.auth_wrapper),
) -> ResultsWrapper[list[ChunkIngestionResponse]]:
print("run_with_orchestration = ", run_with_orchestration)
"""
Create multiple chunks and process them through the ingestion pipeline.
"""
Expand Down
182 changes: 182 additions & 0 deletions py/core/main/api/v3/collections_router.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
import logging
from typing import List, Optional, Union
from uuid import UUID

from fastapi import Body, Depends, Path, Query
from pydantic import BaseModel

from core.base import R2RException, RunType
from core.base.api.models import (
WrappedAddUserResponse,
WrappedCollectionListResponse,
WrappedCollectionResponse,
WrappedDeleteResponse,
WrappedDocumentOverviewResponse,
WrappedUsersInCollectionResponse,
)
from core.providers import (
HatchetOrchestrationProvider,
SimpleOrchestrationProvider,
)
from shared.api.models.base import PaginatedResultsWrapper, ResultsWrapper

from .base_router import BaseRouterV3

logger = logging.getLogger()


class CollectionConfig(BaseModel):
name: str
description: Optional[str] = None


class CollectionsRouter(BaseRouterV3):
def __init__(
self,
providers,
services,
orchestration_provider: Union[
HatchetOrchestrationProvider, SimpleOrchestrationProvider
],
run_type: RunType = RunType.MANAGEMENT,
):
super().__init__(providers, services, orchestration_provider, run_type)

def _setup_routes(self):
@self.router.post("/collections")
@self.base_endpoint
async def create_collection(
config: CollectionConfig = Body(...),
auth_user=Depends(self.providers.auth.auth_wrapper),
) -> WrappedCollectionResponse:
"""
Create a new collection and automatically add the creating user to it.
"""
pass

@self.router.get("/collections")
@self.base_endpoint
async def list_collections(
offset: int = Query(0, ge=0),
limit: int = Query(100, ge=1, le=1000),
name: Optional[str] = Query(None),
sort_by: Optional[str] = Query(None),
sort_order: Optional[str] = Query("desc"),
auth_user=Depends(self.providers.auth.auth_wrapper),
) -> WrappedCollectionListResponse:
"""
List collections the user has access to with pagination and filtering options.
"""
pass

@self.router.get("/collections/{id}")
@self.base_endpoint
async def get_collection(
id: UUID = Path(...),
auth_user=Depends(self.providers.auth.auth_wrapper),
) -> WrappedCollectionResponse:
"""
Get details of a specific collection.
"""
pass

@self.router.post("/collections/{id}")
@self.base_endpoint
async def update_collection(
id: UUID = Path(...),
config: CollectionConfig = Body(...),
auth_user=Depends(self.providers.auth.auth_wrapper),
) -> WrappedCollectionResponse:
"""
Update an existing collection's configuration.
"""
pass

@self.router.delete("/collections/{id}")
@self.base_endpoint
async def delete_collection(
id: UUID = Path(...),
auth_user=Depends(self.providers.auth.auth_wrapper),
) -> WrappedDeleteResponse:
"""
Delete an existing collection.
"""
pass

@self.router.get("/collections/{id}/documents")
@self.base_endpoint
async def get_collection_documents(
id: UUID = Path(...),
offset: int = Query(0, ge=0),
limit: int = Query(100, ge=1, le=1000),
sort_by: Optional[str] = Query(None),
sort_order: Optional[str] = Query("desc"),
auth_user=Depends(self.providers.auth.auth_wrapper),
) -> WrappedDocumentOverviewResponse:
"""
Get all documents in a collection with pagination and sorting options.
"""
pass

@self.router.post("/collections/{id}/documents/{document_id}")
@self.base_endpoint
async def add_document_to_collection(
id: UUID = Path(...),
document_id: UUID = Path(...),
auth_user=Depends(self.providers.auth.auth_wrapper),
) -> WrappedAddUserResponse:
"""
Add a document to a collection.
"""
pass

@self.router.delete("/collections/{id}/documents/{document_id}")
@self.base_endpoint
async def remove_document_from_collection(
id: UUID = Path(...),
document_id: UUID = Path(...),
auth_user=Depends(self.providers.auth.auth_wrapper),
) -> WrappedDeleteResponse:
"""
Remove a document from a collection.
"""
pass

@self.router.get("/collections/{id}/users")
@self.base_endpoint
async def get_collection_users(
id: UUID = Path(...),
offset: int = Query(0, ge=0),
limit: int = Query(100, ge=1, le=1000),
sort_by: Optional[str] = Query(None),
sort_order: Optional[str] = Query("desc"),
auth_user=Depends(self.providers.auth.auth_wrapper),
) -> WrappedUsersInCollectionResponse:
"""
Get all users in a collection with pagination and sorting options.
"""
pass

@self.router.post("/collections/{id}/users/{user_id}")
@self.base_endpoint
async def add_user_to_collection(
id: UUID = Path(...),
user_id: UUID = Path(...),
auth_user=Depends(self.providers.auth.auth_wrapper),
) -> WrappedAddUserResponse:
"""
Add a user to a collection.
"""
pass

@self.router.delete("/collections/{id}/users/{user_id}")
@self.base_endpoint
async def remove_user_from_collection(
id: UUID = Path(...),
user_id: UUID = Path(...),
auth_user=Depends(self.providers.auth.auth_wrapper),
) -> WrappedDeleteResponse:
"""
Remove a user from a collection.
"""
pass
Loading

0 comments on commit 33f6b55

Please sign in to comment.