Pagination computation is elegant with tuple_()

conda-store-server/conda_store_server/_internal/server/views/api.py

@@ -11,23 +11,21 @@
 from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request
 from fastapi.responses import JSONResponse, PlainTextResponse, RedirectResponse
 
-from sqlalchemy.orm import Query as SqlQuery
-
 from conda_store_server import __version__, api, app
 from conda_store_server._internal import orm, schema
 from conda_store_server._internal.environment import filter_environments
-from conda_store_server._internal.schema import AuthenticationToken, Permissions
+from conda_store_server._internal.schema import (
+    AuthenticationToken,
+    Permissions,
+)
 from conda_store_server._internal.server import dependencies
+from conda_store_server._internal.server.views.pagination import Cursor, paginate
 from conda_store_server.exception import CondaStoreError
 from conda_store_server.server.auth import Authentication
 
 
-def paginate(
-    query: SqlQuery,
-    next_token: str | None = None,
-    sort_by: List[str] | None = None,
-    order: str = 'asc',
-)
+def get_cursor(cursor: Optional[str] = None) -> Cursor:
+    return Cursor.load(cursor)
 
 
 class PaginatedArgs(TypedDict):
@@ -646,6 +644,7 @@ async def api_list_environments(
     conda_store: app.CondaStore = Depends(dependencies.get_conda_store),
     entity: AuthenticationToken = Depends(dependencies.get_entity),
     paginated_args: PaginatedArgs = Depends(get_paginated_args),
+    cursor: Cursor = Depends(get_cursor),
     artifact: Optional[schema.BuildArtifactType] = None,
     jwt: Optional[str] = None,
     name: Optional[str] = None,
@@ -705,7 +704,7 @@ async def api_list_environments(
         else:
             role_bindings = None
 
-        orm_environments = api.list_environments(
+        query = api.list_environments(
             db,
             search=search,
             namespace=namespace,
@@ -718,20 +717,52 @@ async def api_list_environments(
         )
 
         # Filter by environments that the user who made the query has access to
-        orm_environments = filter_environments(
-            query=orm_environments,
+        query = filter_environments(
+            query=query,
             role_bindings=auth.entity_bindings(entity),
         )
 
+        sorts = get_sorts(
+            order=paginated_args["order"],
+            sort_by=paginated_args["sort_by"],
+            allowed_sort_bys={
+                "namespace": orm.Namespace.name,
+                "name": orm.Environment.name,
+            },
+            default_sort_by=["namespace", "name"],
+            default_order="asc",
+        )
+
+        # query = (
+        #     query
+        #     .filter(
+        #         or_(
+        #             orm.Namespace.name > cursor.order_by['namespace'],
+        #             orm.Namespace.name == cursor.order_by['namespace']
+        #         )
+        #     )
+        #     .order_by(
+        #         *sorts,
+        #         orm.Environment.id.asc()
+        #     )
+        # )
+
+        return paginate(
+            query=query,
+            cursor=cursor,
+            sort_by=paginated_args["sort_by"],
+        )
+
         return paginated_api_response(
-            orm_environments,
+            query,
             paginated_args,
             schema.Environment,
             exclude={"current_build"},
             allowed_sort_bys={
-                "scheduled_on": orm.Environment.current_build.scheduled_on,
+                "namespace": orm.Namespace.name,
+                "name": orm.Environment.name,
             },
-            default_sort_by=["scheduled_on"],
+            default_sort_by=["namespace", "name"],
             default_order="asc",
         )
 

conda-store-server/conda_store_server/_internal/server/views/pagination.py

@@ -0,0 +1,60 @@
+import base64
+from typing import Dict, List, Optional
+
+import pydantic
+from sqlalchemy import tuple_
+from sqlalchemy.orm import Query as SqlQuery
+
+
+class Cursor(pydantic.BaseModel):
+    last_id: Optional[int] = 1
+
+    # List of names of attributes to order by, and the last value of the ordered attribute
+    # {
+    #   'namespace': 'foo',
+    #   'environment': 'bar',
+    # }
+    last_value: Optional[Dict[str, str]] = {}
+
+    def dump(self):
+        return base64.b64encode(self.model_dump_json())
+
+    @classmethod
+    def load(cls, data: str):
+        return cls.from_json(base64.b64decode(data))
+
+
+def paginate(
+    query: SqlQuery,
+    cursor: Cursor,
+    sort_by: List[str],
+    valid_sort_by: Dict[str, object],
+) -> SqlQuery:
+    """Paginate the query using the cursor and the requested sort_bys.
+
+    With cursor pagination, all keys used to order must be included in
+    the call to query.filter().
+
+    https://medium.com/@george_16060/cursor-based-pagination-with-arbitrary-ordering-b4af6d5e22db
+
+    Parameters
+    ----------
+    query : SqlQuery
+        Query containing database results to paginate
+    cursor : Cursor
+        Cursor object containing information about the last item
+        on the previous page
+    sort_by : List[str]
+        List of sort_by query parameters
+    valid_sort_by : Dict[str, object]
+        Mapping between query parameter names and the orm object they apply to
+    """
+    objects = []
+    last_values = []
+    for obj in sort_by:
+        objects.append(valid_sort_by[obj])
+        last_values.append(cursor.last_value[obj])
+
+    return query.filter(
+        tuple_(*objects, object.id) > (*last_values, cursor.last_id)
+    ).order_by(sorts)

conda-store-server/pyproject.toml

@@ -125,9 +125,6 @@ user-journey-test = ["pytest -m user_journey"]
 conda-store-server = "conda_store_server._internal.server.__main__:main"
 conda-store-worker = "conda_store_server._internal.worker.__main__:main"
 
-[tool.black]
-line-length = 88
-
 [tool.isort]
 lines_between_types = 1
 lines_after_imports = 2
@@ -142,7 +139,6 @@ exclude = [
 [tool.ruff.lint]
 ignore = [
     "E501", # line-length
-
     "ANN001",	# missing-type-function-argument
     "ANN002",	# missing-type-args
     "ANN003",	# missing-type-kwargs