From dea8d3f0cc822f03d1d4a83b1df59d0549b195f2 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Mon, 30 Dec 2024 13:10:06 +0100 Subject: [PATCH] feat(api): add configurable page sizes --- cmd/serve.go | 33 ++++++-- internal/api/common/pagination.go | 6 ++ internal/api/module.go | 9 +- internal/api/router.go | 15 +++- internal/api/v2/common.go | 17 +++- internal/api/v2/controllers_accounts_list.go | 39 ++++----- .../api/v2/controllers_accounts_list_test.go | 22 ++--- internal/api/v2/controllers_ledgers_list.go | 7 +- .../api/v2/controllers_ledgers_list_test.go | 4 +- internal/api/v2/controllers_logs_list.go | 42 +++++----- internal/api/v2/controllers_logs_list_test.go | 14 ++-- .../api/v2/controllers_transactions_list.go | 47 ++++++----- .../v2/controllers_transactions_list_test.go | 26 +++--- internal/api/v2/controllers_volumes.go | 83 +++++++++---------- internal/api/v2/controllers_volumes_test.go | 12 +-- internal/api/v2/query.go | 8 -- internal/api/v2/routes.go | 22 +++-- pkg/testserver/server.go | 9 +- test/e2e/api_ledgers_list_test.go | 9 +- 19 files changed, 247 insertions(+), 177 deletions(-) create mode 100644 internal/api/common/pagination.go diff --git a/cmd/serve.go b/cmd/serve.go index c4a050763..a1f6431ef 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -2,6 +2,7 @@ package cmd import ( "github.com/formancehq/go-libs/v2/logging" + "github.com/formancehq/ledger/internal/api/common" systemstore "github.com/formancehq/ledger/internal/storage/system" "net/http" "net/http/pprof" @@ -43,6 +44,8 @@ const ( BulkMaxSizeFlag = "bulk-max-size" BulkParallelFlag = "bulk-parallel" NumscriptInterpreterFlag = "experimental-numscript-interpreter" + DefaultPageSizeFlag = "default-page-size" + MaxPageSizeFlag = "max-page-size" ) func NewServeCommand() *cobra.Command { @@ -73,6 +76,16 @@ func NewServeCommand() *cobra.Command { return err } + maxPageSize, err := cmd.Flags().GetUint64(MaxPageSizeFlag) + if err != nil { + return err + } + + defaultPageSize, err := cmd.Flags().GetUint64(DefaultPageSizeFlag) + if err != nil { + return err + } + options := []fx.Option{ fx.NopLogger, otlp.FXModuleFromFlags(cmd), @@ -102,18 +115,22 @@ func NewServeCommand() *cobra.Command { MaxSize: bulkMaxSize, Parallel: bulkParallel, }, + Pagination: common.PaginationConfig{ + MaxPageSize: maxPageSize, + DefaultPageSize: defaultPageSize, + }, }), fx.Decorate(func( params struct { - fx.In + fx.In - Handler chi.Router - HealthController *health.HealthController - Logger logging.Logger + Handler chi.Router + HealthController *health.HealthController + Logger logging.Logger - MeterProvider *metric.MeterProvider `optional:"true"` - Exporter *otlpmetrics.InMemoryExporter `optional:"true"` - }, + MeterProvider *metric.MeterProvider `optional:"true"` + Exporter *otlpmetrics.InMemoryExporter `optional:"true"` + }, ) chi.Router { return assembleFinalRouter( service.IsDebug(cmd), @@ -140,6 +157,8 @@ func NewServeCommand() *cobra.Command { cmd.Flags().Int(BulkMaxSizeFlag, api.DefaultBulkMaxSize, "Bulk max size (default 100)") cmd.Flags().Int(BulkParallelFlag, 10, "Bulk max parallelism") cmd.Flags().Bool(NumscriptInterpreterFlag, false, "Enable experimental numscript rewrite") + cmd.Flags().Uint64(MaxPageSizeFlag, 100, "Max page size") + cmd.Flags().Uint64(DefaultPageSizeFlag, 15, "Default page size") service.AddFlags(cmd.Flags()) bunconnect.AddFlags(cmd.Flags()) diff --git a/internal/api/common/pagination.go b/internal/api/common/pagination.go new file mode 100644 index 000000000..163a29616 --- /dev/null +++ b/internal/api/common/pagination.go @@ -0,0 +1,6 @@ +package common + +type PaginationConfig struct { + MaxPageSize uint64 + DefaultPageSize uint64 +} diff --git a/internal/api/module.go b/internal/api/module.go index d3e54c6b9..d9e6bad7f 100644 --- a/internal/api/module.go +++ b/internal/api/module.go @@ -5,6 +5,7 @@ import ( "github.com/formancehq/go-libs/v2/auth" "github.com/formancehq/go-libs/v2/health" "github.com/formancehq/ledger/internal/api/bulking" + "github.com/formancehq/ledger/internal/api/common" "github.com/formancehq/ledger/internal/controller/system" "github.com/go-chi/chi/v5" "go.opentelemetry.io/otel/trace" @@ -17,9 +18,10 @@ type BulkConfig struct { } type Config struct { - Version string - Debug bool - Bulk BulkConfig + Version string + Debug bool + Bulk BulkConfig + Pagination common.PaginationConfig } func Module(cfg Config) fx.Option { @@ -40,6 +42,7 @@ func Module(cfg Config) fx.Option { bulking.WithParallelism(cfg.Bulk.Parallel), bulking.WithTracer(tracerProvider.Tracer("api.bulking")), )), + WithPaginationConfiguration(cfg.Pagination), ) }), health.Module(), diff --git a/internal/api/router.go b/internal/api/router.go index 940ac8bf4..220339515 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,6 +2,7 @@ package api import ( "github.com/formancehq/go-libs/v2/api" + "github.com/formancehq/go-libs/v2/bun/bunpaginate" "github.com/formancehq/ledger/internal/api/bulking" "github.com/formancehq/ledger/internal/controller/system" "go.opentelemetry.io/otel/attribute" @@ -71,6 +72,7 @@ func NewRouter( "application/json": bulking.NewJSONBulkHandlerFactory(routerOptions.bulkMaxSize), "application/vnd.formance.ledger.api.v2.bulk+script-stream": bulking.NewScriptStreamBulkHandlerFactory(), }), + v2.WithPaginationConfig(routerOptions.paginationConfig), ) mux.Handle("/v2*", http.StripPrefix("/v2", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { chi.RouteContext(r.Context()).Reset() @@ -91,7 +93,8 @@ func NewRouter( type routerOptions struct { tracer trace.Tracer bulkMaxSize int - bulkerFactory bulking.BulkerFactory + bulkerFactory bulking.BulkerFactory + paginationConfig common.PaginationConfig } type RouterOption func(ro *routerOptions) @@ -114,9 +117,19 @@ func WithBulkerFactory(bf bulking.BulkerFactory) RouterOption { } } +func WithPaginationConfiguration(paginationConfig common.PaginationConfig) RouterOption { + return func(ro *routerOptions) { + ro.paginationConfig = paginationConfig + } +} + var defaultRouterOptions = []RouterOption{ WithTracer(nooptracer.Tracer{}), WithBulkMaxSize(DefaultBulkMaxSize), + WithPaginationConfiguration(common.PaginationConfig{ + MaxPageSize: bunpaginate.MaxPageSize, + DefaultPageSize: bunpaginate.QueryDefaultPageSize, + }), } const DefaultBulkMaxSize = 100 diff --git a/internal/api/v2/common.go b/internal/api/v2/common.go index 6d7c94185..18b90c97e 100644 --- a/internal/api/v2/common.go +++ b/internal/api/v2/common.go @@ -2,6 +2,7 @@ package v2 import ( . "github.com/formancehq/go-libs/v2/collectionutils" + "github.com/formancehq/ledger/internal/api/common" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" "io" "net/http" @@ -61,14 +62,18 @@ func getExpand(r *http.Request) []string { ) } -func getOffsetPaginatedQuery[v any](r *http.Request, modifiers ...func(*v) error) (*ledgercontroller.OffsetPaginatedQuery[v], error) { +func getOffsetPaginatedQuery[v any](r *http.Request, paginationConfig common.PaginationConfig, modifiers ...func(*v) error) (*ledgercontroller.OffsetPaginatedQuery[v], error) { return bunpaginate.Extract[ledgercontroller.OffsetPaginatedQuery[v]](r, func() (*ledgercontroller.OffsetPaginatedQuery[v], error) { rq, err := getResourceQuery[v](r, modifiers...) if err != nil { return nil, err } - pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize)) + pageSize, err := bunpaginate.GetPageSize( + r, + bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize), + bunpaginate.WithDefaultPageSize(paginationConfig.DefaultPageSize), + ) if err != nil { return nil, err } @@ -80,14 +85,18 @@ func getOffsetPaginatedQuery[v any](r *http.Request, modifiers ...func(*v) error }) } -func getColumnPaginatedQuery[v any](r *http.Request, defaultPaginationColumn string, order bunpaginate.Order, modifiers ...func(*v) error) (*ledgercontroller.ColumnPaginatedQuery[v], error) { +func getColumnPaginatedQuery[v any](r *http.Request, paginationConfig common.PaginationConfig, defaultPaginationColumn string, order bunpaginate.Order, modifiers ...func(*v) error) (*ledgercontroller.ColumnPaginatedQuery[v], error) { return bunpaginate.Extract[ledgercontroller.ColumnPaginatedQuery[v]](r, func() (*ledgercontroller.ColumnPaginatedQuery[v], error) { rq, err := getResourceQuery[v](r, modifiers...) if err != nil { return nil, err } - pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize)) + pageSize, err := bunpaginate.GetPageSize( + r, + bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize), + bunpaginate.WithDefaultPageSize(paginationConfig.DefaultPageSize), + ) if err != nil { return nil, err } diff --git a/internal/api/v2/controllers_accounts_list.go b/internal/api/v2/controllers_accounts_list.go index 0c1e171fd..190f7f5d1 100644 --- a/internal/api/v2/controllers_accounts_list.go +++ b/internal/api/v2/controllers_accounts_list.go @@ -1,33 +1,34 @@ package v2 import ( - "net/http" - "errors" "github.com/formancehq/go-libs/v2/api" "github.com/formancehq/ledger/internal/api/common" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "net/http" ) -func listAccounts(w http.ResponseWriter, r *http.Request) { - l := common.LedgerFromContext(r.Context()) - - query, err := getOffsetPaginatedQuery[any](r) - if err != nil { - api.BadRequest(w, common.ErrValidation, err) - return - } +func listAccounts(paginationConfig common.PaginationConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := common.LedgerFromContext(r.Context()) - cursor, err := l.ListAccounts(r.Context(), *query) - if err != nil { - switch { - case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): + query, err := getOffsetPaginatedQuery[any](r, paginationConfig) + if err != nil { api.BadRequest(w, common.ErrValidation, err) - default: - common.HandleCommonErrors(w, r, err) + return + } + + cursor, err := l.ListAccounts(r.Context(), *query) + if err != nil { + switch { + case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): + api.BadRequest(w, common.ErrValidation, err) + default: + common.HandleCommonErrors(w, r, err) + } + return } - return - } - api.RenderCursor(w, *cursor) + api.RenderCursor(w, *cursor) + } } diff --git a/internal/api/v2/controllers_accounts_list_test.go b/internal/api/v2/controllers_accounts_list_test.go index 3dff1b1de..ae0252e64 100644 --- a/internal/api/v2/controllers_accounts_list_test.go +++ b/internal/api/v2/controllers_accounts_list_test.go @@ -41,7 +41,7 @@ func TestAccountsList(t *testing.T) { { name: "nominal", expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[any]{ PIT: &before, Expand: make([]string, 0), @@ -54,7 +54,7 @@ func TestAccountsList(t *testing.T) { body: `{"$match": { "metadata[roles]": "admin" }}`, expectBackendCall: true, expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[any]{ PIT: &before, Builder: query.Match("metadata[roles]", "admin"), @@ -67,7 +67,7 @@ func TestAccountsList(t *testing.T) { body: `{"$match": { "address": "foo" }}`, expectBackendCall: true, expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[any]{ PIT: &before, Builder: query.Match("address", "foo"), @@ -80,12 +80,12 @@ func TestAccountsList(t *testing.T) { expectBackendCall: true, queryParams: url.Values{ "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.OffsetPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[any]{}, })}, }, expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[any]{}, }, }, @@ -112,7 +112,7 @@ func TestAccountsList(t *testing.T) { "pageSize": []string{"1000000"}, }, expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ - PageSize: MaxPageSize, + PageSize: bunpaginate.MaxPageSize, Options: ledgercontroller.ResourceQuery[any]{ PIT: &before, Expand: make([]string, 0), @@ -124,7 +124,7 @@ func TestAccountsList(t *testing.T) { expectBackendCall: true, body: `{"$lt": { "balance[USD/2]": 100 }}`, expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[any]{ PIT: &before, Builder: query.Lt("balance[USD/2]", float64(100)), @@ -137,7 +137,7 @@ func TestAccountsList(t *testing.T) { expectBackendCall: true, body: `{"$exists": { "metadata": "foo" }}`, expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[any]{ PIT: &before, Builder: query.Exists("metadata", "foo"), @@ -158,7 +158,7 @@ func TestAccountsList(t *testing.T) { expectBackendCall: true, returnErr: ledgercontroller.ErrInvalidQuery{}, expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[any]{ PIT: &before, Expand: make([]string, 0), @@ -172,7 +172,7 @@ func TestAccountsList(t *testing.T) { expectBackendCall: true, returnErr: ledgercontroller.ErrMissingFeature{}, expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[any]{ PIT: &before, Expand: make([]string, 0), @@ -186,7 +186,7 @@ func TestAccountsList(t *testing.T) { expectBackendCall: true, returnErr: errors.New("undefined error"), expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[any]{ PIT: &before, Expand: make([]string, 0), diff --git a/internal/api/v2/controllers_ledgers_list.go b/internal/api/v2/controllers_ledgers_list.go index 8f2a70343..dd0bd3d30 100644 --- a/internal/api/v2/controllers_ledgers_list.go +++ b/internal/api/v2/controllers_ledgers_list.go @@ -12,11 +12,14 @@ import ( "github.com/formancehq/ledger/internal/controller/system" ) -func listLedgers(b system.Controller) http.HandlerFunc { +func listLedgers(b system.Controller, paginationConfig common.PaginationConfig) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { query, err := bunpaginate.Extract[ledgercontroller.ListLedgersQuery](r, func() (*ledgercontroller.ListLedgersQuery, error) { - pageSize, err := bunpaginate.GetPageSize(r) + pageSize, err := bunpaginate.GetPageSize(r, + bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize), + bunpaginate.WithDefaultPageSize(paginationConfig.DefaultPageSize), + ) if err != nil { return nil, err } diff --git a/internal/api/v2/controllers_ledgers_list_test.go b/internal/api/v2/controllers_ledgers_list_test.go index b72785af5..4ffbadd27 100644 --- a/internal/api/v2/controllers_ledgers_list_test.go +++ b/internal/api/v2/controllers_ledgers_list_test.go @@ -71,7 +71,7 @@ func TestListLedgers(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrInvalidQuery{}, - expectQuery: ledgercontroller.NewListLedgersQuery(DefaultPageSize), + expectQuery: ledgercontroller.NewListLedgersQuery(bunpaginate.QueryDefaultPageSize), }, { name: "with missing feature", @@ -79,7 +79,7 @@ func TestListLedgers(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrMissingFeature{}, - expectQuery: ledgercontroller.NewListLedgersQuery(DefaultPageSize), + expectQuery: ledgercontroller.NewListLedgersQuery(bunpaginate.QueryDefaultPageSize), }, } { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/api/v2/controllers_logs_list.go b/internal/api/v2/controllers_logs_list.go index 56c82236a..439b0bdb9 100644 --- a/internal/api/v2/controllers_logs_list.go +++ b/internal/api/v2/controllers_logs_list.go @@ -1,35 +1,35 @@ package v2 import ( - "net/http" - "errors" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" - "github.com/formancehq/go-libs/v2/api" "github.com/formancehq/go-libs/v2/bun/bunpaginate" "github.com/formancehq/ledger/internal/api/common" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "net/http" ) -func listLogs(w http.ResponseWriter, r *http.Request) { - l := common.LedgerFromContext(r.Context()) +func listLogs(paginationConfig common.PaginationConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := common.LedgerFromContext(r.Context()) - rq, err := getColumnPaginatedQuery[any](r, "id", bunpaginate.OrderDesc) - if err != nil { - api.BadRequest(w, common.ErrValidation, err) - return - } - - cursor, err := l.ListLogs(r.Context(), *rq) - if err != nil { - switch { - case errors.Is(err, ledgercontroller.ErrInvalidQuery{}): + rq, err := getColumnPaginatedQuery[any](r, paginationConfig, "id", bunpaginate.OrderDesc) + if err != nil { api.BadRequest(w, common.ErrValidation, err) - default: - common.HandleCommonErrors(w, r, err) + return } - return - } - api.RenderCursor(w, *cursor) + cursor, err := l.ListLogs(r.Context(), *rq) + if err != nil { + switch { + case errors.Is(err, ledgercontroller.ErrInvalidQuery{}): + api.BadRequest(w, common.ErrValidation, err) + default: + common.HandleCommonErrors(w, r, err) + } + return + } + + api.RenderCursor(w, *cursor) + } } diff --git a/internal/api/v2/controllers_logs_list_test.go b/internal/api/v2/controllers_logs_list_test.go index 2e931f7f3..340f84b2d 100644 --- a/internal/api/v2/controllers_logs_list_test.go +++ b/internal/api/v2/controllers_logs_list_test.go @@ -43,7 +43,7 @@ func TestGetLogs(t *testing.T) { { name: "nominal", expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -56,7 +56,7 @@ func TestGetLogs(t *testing.T) { name: "using start time", body: fmt.Sprintf(`{"$gte": {"date": "%s"}}`, now.Format(time.DateFormat)), expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -70,7 +70,7 @@ func TestGetLogs(t *testing.T) { name: "using end time", body: fmt.Sprintf(`{"$lt": {"date": "%s"}}`, now.Format(time.DateFormat)), expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -84,13 +84,13 @@ func TestGetLogs(t *testing.T) { name: "using empty cursor", queryParams: url.Values{ "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), })}, }, expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, @@ -122,7 +122,7 @@ func TestGetLogs(t *testing.T) { name: "with invalid query", expectStatusCode: http.StatusBadRequest, expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -137,7 +137,7 @@ func TestGetLogs(t *testing.T) { name: "with unexpected error", expectStatusCode: http.StatusInternalServerError, expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ diff --git a/internal/api/v2/controllers_transactions_list.go b/internal/api/v2/controllers_transactions_list.go index 2705a514e..c11baa859 100644 --- a/internal/api/v2/controllers_transactions_list.go +++ b/internal/api/v2/controllers_transactions_list.go @@ -1,39 +1,40 @@ package v2 import ( - "net/http" - "errors" "github.com/formancehq/go-libs/v2/api" "github.com/formancehq/go-libs/v2/bun/bunpaginate" "github.com/formancehq/ledger/internal/api/common" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "net/http" ) -func listTransactions(w http.ResponseWriter, r *http.Request) { - l := common.LedgerFromContext(r.Context()) +func listTransactions(paginationConfig common.PaginationConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := common.LedgerFromContext(r.Context()) - paginationColumn := "id" - if r.URL.Query().Get("order") == "effective" { - paginationColumn = "timestamp" - } - - rq, err := getColumnPaginatedQuery[any](r, paginationColumn, bunpaginate.OrderDesc) - if err != nil { - api.BadRequest(w, common.ErrValidation, err) - return - } + paginationColumn := "id" + if r.URL.Query().Get("order") == "effective" { + paginationColumn = "timestamp" + } - cursor, err := l.ListTransactions(r.Context(), *rq) - if err != nil { - switch { - case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): + rq, err := getColumnPaginatedQuery[any](r, paginationConfig, paginationColumn, bunpaginate.OrderDesc) + if err != nil { api.BadRequest(w, common.ErrValidation, err) - default: - common.HandleCommonErrors(w, r, err) + return } - return - } - api.RenderCursor(w, *cursor) + cursor, err := l.ListTransactions(r.Context(), *rq) + if err != nil { + switch { + case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): + api.BadRequest(w, common.ErrValidation, err) + default: + common.HandleCommonErrors(w, r, err) + } + return + } + + api.RenderCursor(w, *cursor) + } } diff --git a/internal/api/v2/controllers_transactions_list_test.go b/internal/api/v2/controllers_transactions_list_test.go index c64489bb7..087faf521 100644 --- a/internal/api/v2/controllers_transactions_list_test.go +++ b/internal/api/v2/controllers_transactions_list_test.go @@ -40,7 +40,7 @@ func TestTransactionsList(t *testing.T) { { name: "nominal", expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -53,7 +53,7 @@ func TestTransactionsList(t *testing.T) { name: "using metadata", body: `{"$match": {"metadata[roles]": "admin"}}`, expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -67,7 +67,7 @@ func TestTransactionsList(t *testing.T) { name: "using startTime", body: fmt.Sprintf(`{"$gte": {"start_time": "%s"}}`, now.Format(time.DateFormat)), expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -81,7 +81,7 @@ func TestTransactionsList(t *testing.T) { name: "using endTime", body: fmt.Sprintf(`{"$lte": {"end_time": "%s"}}`, now.Format(time.DateFormat)), expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -95,7 +95,7 @@ func TestTransactionsList(t *testing.T) { name: "using account", body: `{"$match": {"account": "xxx"}}`, expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -109,7 +109,7 @@ func TestTransactionsList(t *testing.T) { name: "using reference", body: `{"$match": {"reference": "xxx"}}`, expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -123,7 +123,7 @@ func TestTransactionsList(t *testing.T) { name: "using destination", body: `{"$match": {"destination": "xxx"}}`, expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -137,7 +137,7 @@ func TestTransactionsList(t *testing.T) { name: "using source", body: `{"$match": {"source": "xxx"}}`, expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -176,7 +176,7 @@ func TestTransactionsList(t *testing.T) { "pageSize": []string{"1000000"}, }, expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: MaxPageSize, + PageSize: bunpaginate.MaxPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -190,7 +190,7 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "cursor": []string{func() string { return bunpaginate.EncodeCursor(ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -200,7 +200,7 @@ func TestTransactionsList(t *testing.T) { }()}, }, expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -212,7 +212,7 @@ func TestTransactionsList(t *testing.T) { name: "using $exists metadata filter", body: `{"$exists": {"metadata": "foo"}}`, expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ @@ -226,7 +226,7 @@ func TestTransactionsList(t *testing.T) { name: "paginate using effective order", queryParams: map[string][]string{"order": {"effective"}}, expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Column: "timestamp", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Options: ledgercontroller.ResourceQuery[any]{ diff --git a/internal/api/v2/controllers_volumes.go b/internal/api/v2/controllers_volumes.go index 8c94765cb..e5b443e75 100644 --- a/internal/api/v2/controllers_volumes.go +++ b/internal/api/v2/controllers_volumes.go @@ -1,66 +1,65 @@ package v2 import ( - "net/http" - "strconv" - "errors" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" - "github.com/formancehq/go-libs/v2/api" "github.com/formancehq/ledger/internal/api/common" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "net/http" + "strconv" ) -func readVolumes(w http.ResponseWriter, r *http.Request) { +func readVolumes(paginationConfig common.PaginationConfig) http.HandlerFunc { - l := common.LedgerFromContext(r.Context()) + return func(w http.ResponseWriter, r *http.Request) { + l := common.LedgerFromContext(r.Context()) - rq, err := getOffsetPaginatedQuery[ledgercontroller.GetVolumesOptions](r, func(opts *ledgercontroller.GetVolumesOptions) error { - groupBy := r.URL.Query().Get("groupBy") - if groupBy != "" { - v, err := strconv.ParseInt(groupBy, 10, 64) - if err != nil { - return err + rq, err := getOffsetPaginatedQuery[ledgercontroller.GetVolumesOptions](r, paginationConfig, func(opts *ledgercontroller.GetVolumesOptions) error { + groupBy := r.URL.Query().Get("groupBy") + if groupBy != "" { + v, err := strconv.ParseInt(groupBy, 10, 64) + if err != nil { + return err + } + opts.GroupLvl = int(v) } - opts.GroupLvl = int(v) - } - opts.UseInsertionDate = api.QueryParamBool(r, "insertionDate") + opts.UseInsertionDate = api.QueryParamBool(r, "insertionDate") - return nil - }) - if err != nil { - api.BadRequest(w, common.ErrValidation, err) - return - } - - if r.URL.Query().Get("endTime") != "" { - rq.Options.PIT, err = getDate(r, "endTime") + return nil + }) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - } - if r.URL.Query().Get("startTime") != "" { - rq.Options.OOT, err = getDate(r, "startTime") - if err != nil { - api.BadRequest(w, common.ErrValidation, err) - return + if r.URL.Query().Get("endTime") != "" { + rq.Options.PIT, err = getDate(r, "endTime") + if err != nil { + api.BadRequest(w, common.ErrValidation, err) + return + } } - } - cursor, err := l.GetVolumesWithBalances(r.Context(), *rq) - if err != nil { - switch { - case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): - api.BadRequest(w, common.ErrValidation, err) - default: - common.HandleCommonErrors(w, r, err) + if r.URL.Query().Get("startTime") != "" { + rq.Options.OOT, err = getDate(r, "startTime") + if err != nil { + api.BadRequest(w, common.ErrValidation, err) + return + } } - return - } - api.RenderCursor(w, *cursor) + cursor, err := l.GetVolumesWithBalances(r.Context(), *rq) + if err != nil { + switch { + case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): + api.BadRequest(w, common.ErrValidation, err) + default: + common.HandleCommonErrors(w, r, err) + } + return + } + api.RenderCursor(w, *cursor) + } } diff --git a/internal/api/v2/controllers_volumes_test.go b/internal/api/v2/controllers_volumes_test.go index b570bae9b..0b9a06593 100644 --- a/internal/api/v2/controllers_volumes_test.go +++ b/internal/api/v2/controllers_volumes_test.go @@ -41,7 +41,7 @@ func TestGetVolumes(t *testing.T) { { name: "basic", expectQuery: ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &before, Expand: make([]string, 0), @@ -52,7 +52,7 @@ func TestGetVolumes(t *testing.T) { name: "using metadata", body: `{"$match": { "metadata[roles]": "admin" }}`, expectQuery: ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &before, Builder: query.Match("metadata[roles]", "admin"), @@ -64,7 +64,7 @@ func TestGetVolumes(t *testing.T) { name: "using account", body: `{"$match": { "account": "foo" }}`, expectQuery: ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &before, Builder: query.Match("account", "foo"), @@ -85,7 +85,7 @@ func TestGetVolumes(t *testing.T) { "groupBy": []string{"3"}, }, expectQuery: ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &before, Expand: make([]string, 0), @@ -99,7 +99,7 @@ func TestGetVolumes(t *testing.T) { name: "using Exists metadata filter", body: `{"$exists": { "metadata": "foo" }}`, expectQuery: ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &before, Builder: query.Exists("metadata", "foo"), @@ -111,7 +111,7 @@ func TestGetVolumes(t *testing.T) { name: "using balance filter", body: `{"$gte": { "balance[EUR]": 50 }}`, expectQuery: ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ - PageSize: DefaultPageSize, + PageSize: bunpaginate.QueryDefaultPageSize, Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &before, Builder: query.Gte("balance[EUR]", float64(50)), diff --git a/internal/api/v2/query.go b/internal/api/v2/query.go index 22cc3b868..913231aaf 100644 --- a/internal/api/v2/query.go +++ b/internal/api/v2/query.go @@ -6,14 +6,6 @@ import ( "github.com/formancehq/ledger/internal/controller/ledger" "github.com/formancehq/go-libs/v2/api" - "github.com/formancehq/go-libs/v2/bun/bunpaginate" -) - -const ( - MaxPageSize = bunpaginate.MaxPageSize - DefaultPageSize = bunpaginate.QueryDefaultPageSize - - QueryKeyCursor = "cursor" ) func getCommandParameters[INPUT any](r *http.Request, input INPUT) ledger.Parameters[INPUT] { diff --git a/internal/api/v2/routes.go b/internal/api/v2/routes.go index 5cd635fbc..22b93e4ac 100644 --- a/internal/api/v2/routes.go +++ b/internal/api/v2/routes.go @@ -1,6 +1,7 @@ package v2 import ( + "github.com/formancehq/go-libs/v2/bun/bunpaginate" "github.com/formancehq/ledger/internal/api/bulking" nooptracer "go.opentelemetry.io/otel/trace/noop" "net/http" @@ -35,7 +36,7 @@ func NewRouter( router.Use(auth.Middleware(authenticator)) router.Use(service.OTLPMiddleware("ledger", debug)) - router.Get("/", listLedgers(systemController)) + router.Get("/", listLedgers(systemController, routerOptions.paginationConfig)) router.Route("/{ledger}", func(router chi.Router) { router.Use(func(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -61,19 +62,19 @@ func NewRouter( // LedgerController router.Get("/_info", getLedgerInfo) router.Get("/stats", readStats) - router.Get("/logs", listLogs) + router.Get("/logs", listLogs(routerOptions.paginationConfig)) router.Post("/logs/import", importLogs) router.Post("/logs/export", exportLogs) // AccountController - router.Get("/accounts", listAccounts) + router.Get("/accounts", listAccounts(routerOptions.paginationConfig)) router.Head("/accounts", countAccounts) router.Get("/accounts/{address}", readAccount) router.Post("/accounts/{address}/metadata", addAccountMetadata) router.Delete("/accounts/{address}/metadata/{key}", deleteAccountMetadata) // TransactionController - router.Get("/transactions", listTransactions) + router.Get("/transactions", listTransactions(routerOptions.paginationConfig)) router.Head("/transactions", countTransactions) router.Post("/transactions", createTransaction) @@ -85,7 +86,7 @@ func NewRouter( router.Get("/aggregate/balances", readBalancesAggregated) - router.Get("/volumes", readVolumes) + router.Get("/volumes", readVolumes(routerOptions.paginationConfig)) }) }) }) @@ -98,6 +99,7 @@ type routerOptions struct { middlewares []func(http.Handler) http.Handler bulkerFactory bulking.BulkerFactory bulkHandlerFactories map[string]bulking.HandlerFactory + paginationConfig common.PaginationConfig } type RouterOption func(ro *routerOptions) @@ -126,6 +128,12 @@ func WithBulkerFactory(bulkerFactory bulking.BulkerFactory) RouterOption { } } +func WithPaginationConfig(paginationConfig common.PaginationConfig) RouterOption { + return func(ro *routerOptions) { + ro.paginationConfig = paginationConfig + } +} + var defaultRouterOptions = []RouterOption{ WithTracer(nooptracer.Tracer{}), WithBulkerFactory(bulking.NewDefaultBulkerFactory()), @@ -133,4 +141,8 @@ var defaultRouterOptions = []RouterOption{ "application/json": bulking.NewJSONBulkHandlerFactory(100), "application/vnd.formance.ledger.api.v2.bulk+script-stream": bulking.NewScriptStreamBulkHandlerFactory(), }), + WithPaginationConfig(common.PaginationConfig{ + DefaultPageSize: bunpaginate.QueryDefaultPageSize, + MaxPageSize: bunpaginate.MaxPageSize, + }), } diff --git a/pkg/testserver/server.go b/pkg/testserver/server.go index d6dae85ab..d35cd7441 100644 --- a/pkg/testserver/server.go +++ b/pkg/testserver/server.go @@ -48,6 +48,8 @@ type Configuration struct { DisableAutoUpgrade bool BulkMaxSize int ExperimentalNumscriptRewrite bool + MaxPageSize uint64 + DefaultPageSize uint64 } type Logger interface { @@ -177,7 +179,12 @@ func (s *Server) Start() error { args = append(args, "--"+otlp.OtelServiceNameFlag, s.configuration.OTLPConfig.BaseConfig.ServiceName) } } - + if s.configuration.MaxPageSize != 0 { + args = append(args, "--"+cmd.MaxPageSizeFlag, fmt.Sprint(s.configuration.MaxPageSize)) + } + if s.configuration.DefaultPageSize != 0 { + args = append(args, "--"+cmd.DefaultPageSizeFlag, fmt.Sprint(s.configuration.DefaultPageSize)) + } if s.configuration.Debug { args = append(args, "--"+service.DebugFlag) } diff --git a/test/e2e/api_ledgers_list_test.go b/test/e2e/api_ledgers_list_test.go index 34a473118..1bd3deb34 100644 --- a/test/e2e/api_ledgers_list_test.go +++ b/test/e2e/api_ledgers_list_test.go @@ -5,6 +5,7 @@ package test_suite import ( "fmt" "github.com/formancehq/go-libs/v2/logging" + "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/ledger/pkg/client/models/operations" . "github.com/formancehq/ledger/pkg/testserver" . "github.com/onsi/ginkgo/v2" @@ -23,6 +24,8 @@ var _ = Context("Ledger engine tests", func() { Output: GinkgoWriter, Debug: debug, NatsURL: natsServer.GetValue().ClientURL(), + MaxPageSize: 5, + DefaultPageSize: 5, } }) @@ -36,9 +39,11 @@ var _ = Context("Ledger engine tests", func() { } }) It("should be listable", func() { - ledgers, err := ListLedgers(ctx, testServer.GetValue(), operations.V2ListLedgersRequest{}) + ledgers, err := ListLedgers(ctx, testServer.GetValue(), operations.V2ListLedgersRequest{ + PageSize: pointer.For(int64(100)), + }) Expect(err).To(BeNil()) - Expect(ledgers.Data).To(HaveLen(10)) + Expect(ledgers.Data).To(HaveLen(5)) }) }) })