feat(api): add configurable page sizes

cmd/serve.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"github.com/formancehq/go-libs/v2/logging"
+	"github.com/formancehq/ledger/internal/api/common"
 	systemstore "github.com/formancehq/ledger/internal/storage/system"
 	"net/http"
 	"net/http/pprof"
@@ -43,6 +44,8 @@ const (
 	BulkMaxSizeFlag            = "bulk-max-size"
 	BulkParallelFlag           = "bulk-parallel"
 	NumscriptInterpreterFlag   = "experimental-numscript-interpreter"
+	DefaultPageSizeFlag        = "default-page-size"
+	MaxPageSizeFlag            = "max-page-size"
 )
 
 func NewServeCommand() *cobra.Command {
@@ -73,6 +76,16 @@ func NewServeCommand() *cobra.Command {
 				return err
 			}
 
+			maxPageSize, err := cmd.Flags().GetUint64(MaxPageSizeFlag)
+			if err != nil {
+				return err
+			}
+
+			defaultPageSize, err := cmd.Flags().GetUint64(DefaultPageSizeFlag)
+			if err != nil {
+				return err
+			}
+
 			options := []fx.Option{
 				fx.NopLogger,
 				otlp.FXModuleFromFlags(cmd),
@@ -102,18 +115,22 @@ func NewServeCommand() *cobra.Command {
 						MaxSize:  bulkMaxSize,
 						Parallel: bulkParallel,
 					},
+					Pagination: common.PaginationConfig{
+						MaxPageSize:     maxPageSize,
+						DefaultPageSize: defaultPageSize,
+					},
 				}),
 				fx.Decorate(func(
 					params struct {
-						fx.In
+					fx.In
 
-						Handler          chi.Router
-						HealthController *health.HealthController
-						Logger           logging.Logger
+					Handler          chi.Router
+					HealthController *health.HealthController
+					Logger           logging.Logger
 
-						MeterProvider *metric.MeterProvider         `optional:"true"`
-						Exporter      *otlpmetrics.InMemoryExporter `optional:"true"`
-					},
+					MeterProvider *metric.MeterProvider         `optional:"true"`
+					Exporter      *otlpmetrics.InMemoryExporter `optional:"true"`
+				},
 				) chi.Router {
 					return assembleFinalRouter(
 						service.IsDebug(cmd),
@@ -140,6 +157,8 @@ func NewServeCommand() *cobra.Command {
 	cmd.Flags().Int(BulkMaxSizeFlag, api.DefaultBulkMaxSize, "Bulk max size (default 100)")
 	cmd.Flags().Int(BulkParallelFlag, 10, "Bulk max parallelism")
 	cmd.Flags().Bool(NumscriptInterpreterFlag, false, "Enable experimental numscript rewrite")
+	cmd.Flags().Uint64(MaxPageSizeFlag, 100, "Max page size")
+	cmd.Flags().Uint64(DefaultPageSizeFlag, 15, "Default page size")
 
 	service.AddFlags(cmd.Flags())
 	bunconnect.AddFlags(cmd.Flags())

internal/api/common/pagination.go

@@ -0,0 +1,6 @@
+package common
+
+type PaginationConfig struct {
+	MaxPageSize     uint64
+	DefaultPageSize uint64
+}

internal/api/module.go

@@ -5,6 +5,7 @@ import (
 	"github.com/formancehq/go-libs/v2/auth"
 	"github.com/formancehq/go-libs/v2/health"
 	"github.com/formancehq/ledger/internal/api/bulking"
+	"github.com/formancehq/ledger/internal/api/common"
 	"github.com/formancehq/ledger/internal/controller/system"
 	"github.com/go-chi/chi/v5"
 	"go.opentelemetry.io/otel/trace"
@@ -17,9 +18,10 @@ type BulkConfig struct {
 }
 
 type Config struct {
-	Version string
-	Debug   bool
-	Bulk    BulkConfig
+	Version    string
+	Debug      bool
+	Bulk       BulkConfig
+	Pagination common.PaginationConfig
 }
 
 func Module(cfg Config) fx.Option {
@@ -40,6 +42,7 @@ func Module(cfg Config) fx.Option {
 					bulking.WithParallelism(cfg.Bulk.Parallel),
 					bulking.WithTracer(tracerProvider.Tracer("api.bulking")),
 				)),
+				WithPaginationConfiguration(cfg.Pagination),
 			)
 		}),
 		health.Module(),

internal/api/router.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"github.com/formancehq/go-libs/v2/api"
+	"github.com/formancehq/go-libs/v2/bun/bunpaginate"
 	"github.com/formancehq/ledger/internal/api/bulking"
 	"github.com/formancehq/ledger/internal/controller/system"
 	"go.opentelemetry.io/otel/attribute"
@@ -71,6 +72,7 @@ func NewRouter(
 			"application/json": bulking.NewJSONBulkHandlerFactory(routerOptions.bulkMaxSize),
 			"application/vnd.formance.ledger.api.v2.bulk+script-stream": bulking.NewScriptStreamBulkHandlerFactory(),
 		}),
+		v2.WithPaginationConfig(routerOptions.paginationConfig),
 	)
 	mux.Handle("/v2*", http.StripPrefix("/v2", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		chi.RouteContext(r.Context()).Reset()
@@ -91,7 +93,8 @@ func NewRouter(
 type routerOptions struct {
 	tracer        trace.Tracer
 	bulkMaxSize   int
-	bulkerFactory bulking.BulkerFactory
+	bulkerFactory    bulking.BulkerFactory
+	paginationConfig common.PaginationConfig
 }
 
 type RouterOption func(ro *routerOptions)
@@ -114,9 +117,19 @@ func WithBulkerFactory(bf bulking.BulkerFactory) RouterOption {
 	}
 }
 
+func WithPaginationConfiguration(paginationConfig common.PaginationConfig) RouterOption {
+	return func(ro *routerOptions) {
+		ro.paginationConfig = paginationConfig
+	}
+}
+
 var defaultRouterOptions = []RouterOption{
 	WithTracer(nooptracer.Tracer{}),
 	WithBulkMaxSize(DefaultBulkMaxSize),
+	WithPaginationConfiguration(common.PaginationConfig{
+		MaxPageSize:     bunpaginate.MaxPageSize,
+		DefaultPageSize: bunpaginate.QueryDefaultPageSize,
+	}),
 }
 
 const DefaultBulkMaxSize = 100

internal/api/v2/common.go

@@ -2,6 +2,7 @@ package v2
 
 import (
 	. "github.com/formancehq/go-libs/v2/collectionutils"
+	"github.com/formancehq/ledger/internal/api/common"
 	ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger"
 	"io"
 	"net/http"
@@ -61,14 +62,18 @@ func getExpand(r *http.Request) []string {
 	)
 }
 
-func getOffsetPaginatedQuery[v any](r *http.Request, modifiers ...func(*v) error) (*ledgercontroller.OffsetPaginatedQuery[v], error) {
+func getOffsetPaginatedQuery[v any](r *http.Request, paginationConfig common.PaginationConfig, modifiers ...func(*v) error) (*ledgercontroller.OffsetPaginatedQuery[v], error) {
 	return bunpaginate.Extract[ledgercontroller.OffsetPaginatedQuery[v]](r, func() (*ledgercontroller.OffsetPaginatedQuery[v], error) {
 		rq, err := getResourceQuery[v](r, modifiers...)
 		if err != nil {
 			return nil, err
 		}
 
-		pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize))
+		pageSize, err := bunpaginate.GetPageSize(
+			r,
+			bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize),
+			bunpaginate.WithDefaultPageSize(paginationConfig.DefaultPageSize),
+		)
 		if err != nil {
 			return nil, err
 		}
@@ -80,14 +85,18 @@ func getOffsetPaginatedQuery[v any](r *http.Request, modifiers ...func(*v) error
 	})
 }
 
-func getColumnPaginatedQuery[v any](r *http.Request, defaultPaginationColumn string, order bunpaginate.Order, modifiers ...func(*v) error) (*ledgercontroller.ColumnPaginatedQuery[v], error) {
+func getColumnPaginatedQuery[v any](r *http.Request, paginationConfig common.PaginationConfig, defaultPaginationColumn string, order bunpaginate.Order, modifiers ...func(*v) error) (*ledgercontroller.ColumnPaginatedQuery[v], error) {
 	return bunpaginate.Extract[ledgercontroller.ColumnPaginatedQuery[v]](r, func() (*ledgercontroller.ColumnPaginatedQuery[v], error) {
 		rq, err := getResourceQuery[v](r, modifiers...)
 		if err != nil {
 			return nil, err
 		}
 
-		pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize))
+		pageSize, err := bunpaginate.GetPageSize(
+			r,
+			bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize),
+			bunpaginate.WithDefaultPageSize(paginationConfig.DefaultPageSize),
+		)
 		if err != nil {
 			return nil, err
 		}

internal/api/v2/controllers_accounts_list.go

@@ -1,33 +1,34 @@
 package v2
 
 import (
-	"net/http"
-
 	"errors"
 	"github.com/formancehq/go-libs/v2/api"
 	"github.com/formancehq/ledger/internal/api/common"
 	ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger"
+	"net/http"
 )
 
-func listAccounts(w http.ResponseWriter, r *http.Request) {
-	l := common.LedgerFromContext(r.Context())
-
-	query, err := getOffsetPaginatedQuery[any](r)
-	if err != nil {
-		api.BadRequest(w, common.ErrValidation, err)
-		return
-	}
+func listAccounts(paginationConfig common.PaginationConfig) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		l := common.LedgerFromContext(r.Context())
 
-	cursor, err := l.ListAccounts(r.Context(), *query)
-	if err != nil {
-		switch {
-		case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}):
+		query, err := getOffsetPaginatedQuery[any](r, paginationConfig)
+		if err != nil {
 			api.BadRequest(w, common.ErrValidation, err)
-		default:
-			common.HandleCommonErrors(w, r, err)
+			return
+		}
+
+		cursor, err := l.ListAccounts(r.Context(), *query)
+		if err != nil {
+			switch {
+			case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}):
+				api.BadRequest(w, common.ErrValidation, err)
+			default:
+				common.HandleCommonErrors(w, r, err)
+			}
+			return
 		}
-		return
-	}
 
-	api.RenderCursor(w, *cursor)
+		api.RenderCursor(w, *cursor)
+	}
 }

internal/api/v2/controllers_accounts_list_test.go

@@ -41,7 +41,7 @@ func TestAccountsList(t *testing.T) {
 		{
 			name: "nominal",
 			expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
-				PageSize: DefaultPageSize,
+				PageSize: bunpaginate.QueryDefaultPageSize,
 				Options: ledgercontroller.ResourceQuery[any]{
 					PIT:    &before,
 					Expand: make([]string, 0),
@@ -54,7 +54,7 @@ func TestAccountsList(t *testing.T) {
 			body:              `{"$match": { "metadata[roles]": "admin" }}`,
 			expectBackendCall: true,
 			expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
-				PageSize: DefaultPageSize,
+				PageSize: bunpaginate.QueryDefaultPageSize,
 				Options: ledgercontroller.ResourceQuery[any]{
 					PIT:     &before,
 					Builder: query.Match("metadata[roles]", "admin"),
@@ -67,7 +67,7 @@ func TestAccountsList(t *testing.T) {
 			body:              `{"$match": { "address": "foo" }}`,
 			expectBackendCall: true,
 			expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
-				PageSize: DefaultPageSize,
+				PageSize: bunpaginate.QueryDefaultPageSize,
 				Options: ledgercontroller.ResourceQuery[any]{
 					PIT:     &before,
 					Builder: query.Match("address", "foo"),
@@ -80,12 +80,12 @@ func TestAccountsList(t *testing.T) {
 			expectBackendCall: true,
 			queryParams: url.Values{
 				"cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.OffsetPaginatedQuery[any]{
-					PageSize: DefaultPageSize,
+					PageSize: bunpaginate.QueryDefaultPageSize,
 					Options:  ledgercontroller.ResourceQuery[any]{},
 				})},
 			},
 			expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
-				PageSize: DefaultPageSize,
+				PageSize: bunpaginate.QueryDefaultPageSize,
 				Options:  ledgercontroller.ResourceQuery[any]{},
 			},
 		},
@@ -112,7 +112,7 @@ func TestAccountsList(t *testing.T) {
 				"pageSize": []string{"1000000"},
 			},
 			expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
-				PageSize: MaxPageSize,
+				PageSize: bunpaginate.MaxPageSize,
 				Options: ledgercontroller.ResourceQuery[any]{
 					PIT:    &before,
 					Expand: make([]string, 0),
@@ -124,7 +124,7 @@ func TestAccountsList(t *testing.T) {
 			expectBackendCall: true,
 			body:              `{"$lt": { "balance[USD/2]": 100 }}`,
 			expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
-				PageSize: DefaultPageSize,
+				PageSize: bunpaginate.QueryDefaultPageSize,
 				Options: ledgercontroller.ResourceQuery[any]{
 					PIT:     &before,
 					Builder: query.Lt("balance[USD/2]", float64(100)),
@@ -137,7 +137,7 @@ func TestAccountsList(t *testing.T) {
 			expectBackendCall: true,
 			body:              `{"$exists": { "metadata": "foo" }}`,
 			expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
-				PageSize: DefaultPageSize,
+				PageSize: bunpaginate.QueryDefaultPageSize,
 				Options: ledgercontroller.ResourceQuery[any]{
 					PIT:     &before,
 					Builder: query.Exists("metadata", "foo"),
@@ -158,7 +158,7 @@ func TestAccountsList(t *testing.T) {
 			expectBackendCall: true,
 			returnErr:         ledgercontroller.ErrInvalidQuery{},
 			expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
-				PageSize: DefaultPageSize,
+				PageSize: bunpaginate.QueryDefaultPageSize,
 				Options: ledgercontroller.ResourceQuery[any]{
 					PIT:    &before,
 					Expand: make([]string, 0),
@@ -172,7 +172,7 @@ func TestAccountsList(t *testing.T) {
 			expectBackendCall: true,
 			returnErr:         ledgercontroller.ErrMissingFeature{},
 			expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
-				PageSize: DefaultPageSize,
+				PageSize: bunpaginate.QueryDefaultPageSize,
 				Options: ledgercontroller.ResourceQuery[any]{
 					PIT:    &before,
 					Expand: make([]string, 0),
@@ -186,7 +186,7 @@ func TestAccountsList(t *testing.T) {
 			expectBackendCall: true,
 			returnErr:         errors.New("undefined error"),
 			expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{
-				PageSize: DefaultPageSize,
+				PageSize: bunpaginate.QueryDefaultPageSize,
 				Options: ledgercontroller.ResourceQuery[any]{
 					PIT:    &before,
 					Expand: make([]string, 0),

internal/api/v2/controllers_ledgers_list.go

@@ -12,11 +12,14 @@ import (
 	"github.com/formancehq/ledger/internal/controller/system"
 )
 
-func listLedgers(b system.Controller) http.HandlerFunc {
+func listLedgers(b system.Controller, paginationConfig common.PaginationConfig) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 
 		query, err := bunpaginate.Extract[ledgercontroller.ListLedgersQuery](r, func() (*ledgercontroller.ListLedgersQuery, error) {
-			pageSize, err := bunpaginate.GetPageSize(r)
+			pageSize, err := bunpaginate.GetPageSize(r,
+				bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize),
+				bunpaginate.WithDefaultPageSize(paginationConfig.DefaultPageSize),
+			)
 			if err != nil {
 				return nil, err
 			}

internal/api/v2/controllers_ledgers_list_test.go

@@ -71,15 +71,15 @@ func TestListLedgers(t *testing.T) {
 			expectedErrorCode:  common.ErrValidation,
 			expectBackendCall:  true,
 			returnErr:          ledgercontroller.ErrInvalidQuery{},
-			expectQuery:        ledgercontroller.NewListLedgersQuery(DefaultPageSize),
+			expectQuery:        ledgercontroller.NewListLedgersQuery(bunpaginate.QueryDefaultPageSize),
 		},
 		{
 			name:               "with missing feature",
 			expectedStatusCode: http.StatusBadRequest,
 			expectedErrorCode:  common.ErrValidation,
 			expectBackendCall:  true,
 			returnErr:          ledgercontroller.ErrMissingFeature{},
-			expectQuery:        ledgercontroller.NewListLedgersQuery(DefaultPageSize),
+			expectQuery:        ledgercontroller.NewListLedgersQuery(bunpaginate.QueryDefaultPageSize),
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {