From 9e55051a819d05d01018927110dc71ffef80f91f Mon Sep 17 00:00:00 2001 From: pdmurray Date: Fri, 18 Oct 2024 10:29:39 -0700 Subject: [PATCH] Add fastapi_pagination --- .../_internal/server/app.py | 3 + .../_internal/server/views/api.py | 86 ++++++++++++++----- conda-store-server/pyproject.toml | 2 + 3 files changed, 68 insertions(+), 23 deletions(-) diff --git a/conda-store-server/conda_store_server/_internal/server/app.py b/conda-store-server/conda_store_server/_internal/server/app.py index 7e23361d8..3ff0df544 100644 --- a/conda-store-server/conda_store_server/_internal/server/app.py +++ b/conda-store-server/conda_store_server/_internal/server/app.py @@ -18,6 +18,7 @@ from fastapi.responses import FileResponse, JSONResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates +from fastapi_pagination import add_pagination from sqlalchemy.pool import QueuePool from starlette.middleware.sessions import SessionMiddleware from traitlets import ( @@ -236,6 +237,8 @@ def trim_slash(url): }, ) + add_pagination(app) + app.add_middleware( CORSMiddleware, allow_origins=self.cors_allow_origins, diff --git a/conda-store-server/conda_store_server/_internal/server/views/api.py b/conda-store-server/conda_store_server/_internal/server/views/api.py index 89bf91dd4..79df926a8 100644 --- a/conda-store-server/conda_store_server/_internal/server/views/api.py +++ b/conda-store-server/conda_store_server/_internal/server/views/api.py @@ -4,7 +4,7 @@ import datetime -from typing import Any, Dict, List, Optional, TypedDict +from typing import Any, Dict, List, Optional, Tuple, TypedDict import pydantic import yaml @@ -12,6 +12,11 @@ from celery.result import AsyncResult from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request from fastapi.responses import PlainTextResponse, RedirectResponse +from fastapi_pagination import set_params +from fastapi_pagination.cursor import CursorPage, CursorParams +from fastapi_pagination.ext.sqlalchemy import paginate +from pydantic import BaseModel +from sqlalchemy.orm import Query as SqlQuery from conda_store_server import __version__, api, app from conda_store_server._internal import orm, schema, utils @@ -21,6 +26,9 @@ from conda_store_server.server.auth import Authentication +set_params(CursorParams(size=10, cursor=None)) + + class PaginatedArgs(TypedDict): """Dictionary type holding information about paginated requests.""" @@ -56,19 +64,40 @@ def get_paginated_args( def filter_distinct_on( - query, - distinct_on: List[str] = [], - allowed_distinct_ons: Dict = {}, - default_distinct_on: List[str] = [], -): - distinct_on = distinct_on or default_distinct_on - distinct_on = [ - allowed_distinct_ons[d] for d in distinct_on if d in allowed_distinct_ons - ] + query: SqlQuery, + distinct_on: List[str] | None = None, + allowed_distinct_ons: Dict | None = None, +) -> Tuple[List[str], SqlQuery]: + """Filter the query using the distinct fields. + + Parameters + ---------- + query : SqlQuery + Query to filter + distinct_on : List[str] | None + Parameter to pass to the FILTER DISTINCT statement + allowed_distinct_ons : Dict | None + Allowed values of the parameter + + Returns + ------- + SqlQuery + Query containing filtered results + """ + if distinct_on is None: + distinct_on = [] + + disallowed = set(distinct_on) - set(allowed_distinct_ons) + if disallowed: + raise HTTPException( + status_code=400, + detail=( + f"Requested distinct_on terms ({disallowed}) are not allowed. " + f"Valid terms are {set(allowed_distinct_ons)}" + ), + ) - if distinct_on: - return distinct_on, query.distinct(*distinct_on) - return distinct_on, query + return query.distinct(*[allowed_distinct_ons[item] for item in distinct_on]) def get_sorts( @@ -95,6 +124,15 @@ def get_sorts( return [order_mapping[order](k) for k in sort_by] +def paginate_response( + query: SqlQuery, + obj_schema: BaseModel, + order: str = "asc", + sort_by: List[str] = None, +) -> CursorPage: + return + + def paginated_api_response( query, paginated_args, @@ -105,7 +143,7 @@ def paginated_api_response( required_sort_bys: List = [], default_sort_by: List = [], default_order: str = "asc", -): +) -> CursorPage: sorts = get_sorts( order=paginated_args["order"], sort_by=paginated_args["sort_by"], @@ -115,15 +153,17 @@ def paginated_api_response( default_order=default_order, ) - count = query.count() - query = ( - query.order_by(*sorts) - .limit(paginated_args["limit"]) - .offset(paginated_args["offset"]) + print( + query, + paginated_args, + object_schema, + sorts, ) + + count = query.count() return { "status": "ok", - "data": [object_schema.from_orm(_).dict(exclude=exclude) for _ in query.all()], + "data": paginate(query.order_by(*sorts)), "page": (paginated_args["offset"] // paginated_args["limit"]) + 1, "size": paginated_args["limit"], "count": count, @@ -714,9 +754,9 @@ async def api_list_environments( paginated_args, schema.Environment, exclude={"current_build"}, - allowed_sort_bys={ - "scheduled_on": orm.Environment.current_build.scheduled_on, - }, + # allowed_sort_bys={ + # "scheduled_on": orm.Environment.current_build.scheduled_on, + # }, default_sort_by=["scheduled_on"], default_order="asc", ) diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index 477d3ec77..2ecd8b134 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -52,6 +52,7 @@ dependencies = [ "alembic", "celery", "fastapi", + "fastapi_pagination", "filelock", "flower", "itsdangerous", @@ -67,6 +68,7 @@ dependencies = [ "pydantic >=1.10.16,<2.0a0", "python-multipart", "sqlalchemy<2.0a0", + "sqlakeyset", "traitlets", "uvicorn", "yarl",