From a2d52cd5a7199e58b54f5e7f83f1384f67606b91 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Tue, 3 Oct 2023 16:12:40 +0300 Subject: [PATCH 01/27] Added Access API event encoding parameter. --- access/api.go | 11 +- access/handler.go | 36 +++++- access/legacy/handler.go | 7 +- access/mock/api.go | 92 ++++++------- .../rest/apiproxy/rest_proxy_handler.go | 38 ++++-- engine/access/rest/routes/events.go | 6 +- engine/access/rest/routes/transactions.go | 6 +- engine/access/rpc/backend/backend_events.go | 9 +- engine/access/rpc/backend/backend_test.go | 40 +++--- .../rpc/backend/backend_transactions.go | 15 ++- .../rpc/backend/backend_transactions_test.go | 17 +-- .../rpc/backend/historical_access_test.go | 3 +- engine/access/rpc/backend/retry_test.go | 2 +- .../state/mock/script_execution_state.go | 88 +++++++++++++ go.mod | 2 + go.sum | 4 +- integration/go.mod | 2 + integration/go.sum | 4 +- storage/mock/light_transaction_results.go | 122 ++++++++++++++++++ 19 files changed, 389 insertions(+), 115 deletions(-) create mode 100644 engine/execution/state/mock/script_execution_state.go create mode 100644 storage/mock/light_transaction_results.go diff --git a/access/api.go b/access/api.go index 933dbe3cd05..24e2c53c5a1 100644 --- a/access/api.go +++ b/access/api.go @@ -5,6 +5,7 @@ import ( "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" + execproto "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" @@ -29,9 +30,9 @@ type API interface { SendTransaction(ctx context.Context, tx *flow.TransactionBody) error GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error) GetTransactionsByBlockID(ctx context.Context, blockID flow.Identifier) ([]*flow.TransactionBody, error) - GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*TransactionResult, error) - GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32) (*TransactionResult, error) - GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier) ([]*TransactionResult, error) + GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, eventEncodingVersion execproto.EventEncodingVersion) (*TransactionResult, error) + GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, eventEncodingVersion execproto.EventEncodingVersion) (*TransactionResult, error) + GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, eventEncodingVersion execproto.EventEncodingVersion) ([]*TransactionResult, error) GetAccount(ctx context.Context, address flow.Address) (*flow.Account, error) GetAccountAtLatestBlock(ctx context.Context, address flow.Address) (*flow.Account, error) @@ -41,8 +42,8 @@ type API interface { ExecuteScriptAtBlockHeight(ctx context.Context, blockHeight uint64, script []byte, arguments [][]byte) ([]byte, error) ExecuteScriptAtBlockID(ctx context.Context, blockID flow.Identifier, script []byte, arguments [][]byte) ([]byte, error) - GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64) ([]flow.BlockEvents, error) - GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier) ([]flow.BlockEvents, error) + GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64, eventEncodingVersion execproto.EventEncodingVersion) ([]flow.BlockEvents, error) + GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersion execproto.EventEncodingVersion) ([]flow.BlockEvents, error) GetLatestProtocolStateSnapshot(ctx context.Context) ([]byte, error) diff --git a/access/handler.go b/access/handler.go index 80f7a96fdf3..7180236b97a 100644 --- a/access/handler.go +++ b/access/handler.go @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" + execproto "github.com/onflow/flow/protobuf/go/flow/execution" ) type Handler struct { @@ -274,7 +275,12 @@ func (h *Handler) GetTransactionResult( } } - result, err := h.api.GetTransactionResult(ctx, transactionID, blockId, collectionId) + eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + if requestEEV := req.GetEventEncodingVersion(); requestEEV != nil { + eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + } + + result, err := h.api.GetTransactionResult(ctx, transactionID, blockId, collectionId, eventEncodingVersion) if err != nil { return nil, err } @@ -296,7 +302,12 @@ func (h *Handler) GetTransactionResultsByBlockID( return nil, err } - results, err := h.api.GetTransactionResultsByBlockID(ctx, id) + eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + if requestEEV := req.GetEventEncodingVersion(); requestEEV != nil { + eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + } + + results, err := h.api.GetTransactionResultsByBlockID(ctx, id, eventEncodingVersion) if err != nil { return nil, err } @@ -342,7 +353,12 @@ func (h *Handler) GetTransactionResultByIndex( return nil, err } - result, err := h.api.GetTransactionResultByIndex(ctx, blockID, req.GetIndex()) + eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + if requestEEV := req.GetEventEncodingVersion(); requestEEV != nil { + eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + } + + result, err := h.api.GetTransactionResultByIndex(ctx, blockID, req.GetIndex(), eventEncodingVersion) if err != nil { return nil, err } @@ -513,7 +529,12 @@ func (h *Handler) GetEventsForHeightRange( startHeight := req.GetStartHeight() endHeight := req.GetEndHeight() - results, err := h.api.GetEventsForHeightRange(ctx, eventType, startHeight, endHeight) + eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + if requestEEV := req.GetEventEncodingVersion(); requestEEV != nil { + eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + } + + results, err := h.api.GetEventsForHeightRange(ctx, eventType, startHeight, endHeight, eventEncodingVersion) if err != nil { return nil, err } @@ -545,7 +566,12 @@ func (h *Handler) GetEventsForBlockIDs( return nil, err } - results, err := h.api.GetEventsForBlockIDs(ctx, eventType, blockIDs) + eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + if requestEEV := req.GetEventEncodingVersion(); requestEEV != nil { + eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + } + + results, err := h.api.GetEventsForBlockIDs(ctx, eventType, blockIDs, eventEncodingVersion) if err != nil { return nil, err } diff --git a/access/legacy/handler.go b/access/legacy/handler.go index 48f4efc911d..24218615489 100644 --- a/access/legacy/handler.go +++ b/access/legacy/handler.go @@ -3,6 +3,7 @@ package handler import ( "context" + execproto "github.com/onflow/flow/protobuf/go/flow/execution" accessproto "github.com/onflow/flow/protobuf/go/flow/legacy/access" entitiesproto "github.com/onflow/flow/protobuf/go/flow/legacy/entities" "google.golang.org/grpc/codes" @@ -189,7 +190,7 @@ func (h *Handler) GetTransactionResult( ) (*accessproto.TransactionResultResponse, error) { id := convert.MessageToIdentifier(req.GetId()) - result, err := h.api.GetTransactionResult(ctx, id, flow.ZeroID, flow.ZeroID) + result, err := h.api.GetTransactionResult(ctx, id, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) if err != nil { return nil, err } @@ -313,7 +314,7 @@ func (h *Handler) GetEventsForHeightRange( startHeight := req.GetStartHeight() endHeight := req.GetEndHeight() - results, err := h.api.GetEventsForHeightRange(ctx, eventType, startHeight, endHeight) + results, err := h.api.GetEventsForHeightRange(ctx, eventType, startHeight, endHeight, execproto.EventEncodingVersion_JSON_CDC_V0) if err != nil { return nil, err } @@ -331,7 +332,7 @@ func (h *Handler) GetEventsForBlockIDs( eventType := req.GetType() blockIDs := convert.MessagesToIdentifiers(req.GetBlockIds()) - results, err := h.api.GetEventsForBlockIDs(ctx, eventType, blockIDs) + results, err := h.api.GetEventsForBlockIDs(ctx, eventType, blockIDs, execproto.EventEncodingVersion_JSON_CDC_V0) if err != nil { return nil, err } diff --git a/access/mock/api.go b/access/mock/api.go index b3a91590f80..4065590969a 100644 --- a/access/mock/api.go +++ b/access/mock/api.go @@ -7,6 +7,8 @@ import ( access "github.com/onflow/flow-go/access" + execution "github.com/onflow/flow/protobuf/go/flow/execution" + flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" @@ -331,25 +333,25 @@ func (_m *API) GetCollectionByID(ctx context.Context, id flow.Identifier) (*flow return r0, r1 } -// GetEventsForBlockIDs provides a mock function with given fields: ctx, eventType, blockIDs -func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier) ([]flow.BlockEvents, error) { - ret := _m.Called(ctx, eventType, blockIDs) +// GetEventsForBlockIDs provides a mock function with given fields: ctx, eventType, blockIDs, eventEncodingVersion +func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersion execution.EventEncodingVersion) ([]flow.BlockEvents, error) { + ret := _m.Called(ctx, eventType, blockIDs, eventEncodingVersion) var r0 []flow.BlockEvents var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier) ([]flow.BlockEvents, error)); ok { - return rf(ctx, eventType, blockIDs) + if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier, execution.EventEncodingVersion) ([]flow.BlockEvents, error)); ok { + return rf(ctx, eventType, blockIDs, eventEncodingVersion) } - if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier) []flow.BlockEvents); ok { - r0 = rf(ctx, eventType, blockIDs) + if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier, execution.EventEncodingVersion) []flow.BlockEvents); ok { + r0 = rf(ctx, eventType, blockIDs, eventEncodingVersion) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]flow.BlockEvents) } } - if rf, ok := ret.Get(1).(func(context.Context, string, []flow.Identifier) error); ok { - r1 = rf(ctx, eventType, blockIDs) + if rf, ok := ret.Get(1).(func(context.Context, string, []flow.Identifier, execution.EventEncodingVersion) error); ok { + r1 = rf(ctx, eventType, blockIDs, eventEncodingVersion) } else { r1 = ret.Error(1) } @@ -357,25 +359,25 @@ func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, block return r0, r1 } -// GetEventsForHeightRange provides a mock function with given fields: ctx, eventType, startHeight, endHeight -func (_m *API) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight uint64, endHeight uint64) ([]flow.BlockEvents, error) { - ret := _m.Called(ctx, eventType, startHeight, endHeight) +// GetEventsForHeightRange provides a mock function with given fields: ctx, eventType, startHeight, endHeight, eventEncodingVersion +func (_m *API) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight uint64, endHeight uint64, eventEncodingVersion execution.EventEncodingVersion) ([]flow.BlockEvents, error) { + ret := _m.Called(ctx, eventType, startHeight, endHeight, eventEncodingVersion) var r0 []flow.BlockEvents var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64) ([]flow.BlockEvents, error)); ok { - return rf(ctx, eventType, startHeight, endHeight) + if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64, execution.EventEncodingVersion) ([]flow.BlockEvents, error)); ok { + return rf(ctx, eventType, startHeight, endHeight, eventEncodingVersion) } - if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64) []flow.BlockEvents); ok { - r0 = rf(ctx, eventType, startHeight, endHeight) + if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64, execution.EventEncodingVersion) []flow.BlockEvents); ok { + r0 = rf(ctx, eventType, startHeight, endHeight, eventEncodingVersion) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]flow.BlockEvents) } } - if rf, ok := ret.Get(1).(func(context.Context, string, uint64, uint64) error); ok { - r1 = rf(ctx, eventType, startHeight, endHeight) + if rf, ok := ret.Get(1).(func(context.Context, string, uint64, uint64, execution.EventEncodingVersion) error); ok { + r1 = rf(ctx, eventType, startHeight, endHeight, eventEncodingVersion) } else { r1 = ret.Error(1) } @@ -593,25 +595,25 @@ func (_m *API) GetTransaction(ctx context.Context, id flow.Identifier) (*flow.Tr return r0, r1 } -// GetTransactionResult provides a mock function with given fields: ctx, id, blockID, collectionID -func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*access.TransactionResult, error) { - ret := _m.Called(ctx, id, blockID, collectionID) +// GetTransactionResult provides a mock function with given fields: ctx, id, blockID, collectionID, eventEncodingVersion +func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, eventEncodingVersion execution.EventEncodingVersion) (*access.TransactionResult, error) { + ret := _m.Called(ctx, id, blockID, collectionID, eventEncodingVersion) var r0 *access.TransactionResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier) (*access.TransactionResult, error)); ok { - return rf(ctx, id, blockID, collectionID) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, execution.EventEncodingVersion) (*access.TransactionResult, error)); ok { + return rf(ctx, id, blockID, collectionID, eventEncodingVersion) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier) *access.TransactionResult); ok { - r0 = rf(ctx, id, blockID, collectionID) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, execution.EventEncodingVersion) *access.TransactionResult); ok { + r0 = rf(ctx, id, blockID, collectionID, eventEncodingVersion) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*access.TransactionResult) } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier) error); ok { - r1 = rf(ctx, id, blockID, collectionID) + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, execution.EventEncodingVersion) error); ok { + r1 = rf(ctx, id, blockID, collectionID, eventEncodingVersion) } else { r1 = ret.Error(1) } @@ -619,25 +621,25 @@ func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blo return r0, r1 } -// GetTransactionResultByIndex provides a mock function with given fields: ctx, blockID, index -func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32) (*access.TransactionResult, error) { - ret := _m.Called(ctx, blockID, index) +// GetTransactionResultByIndex provides a mock function with given fields: ctx, blockID, index, eventEncodingVersion +func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, eventEncodingVersion execution.EventEncodingVersion) (*access.TransactionResult, error) { + ret := _m.Called(ctx, blockID, index, eventEncodingVersion) var r0 *access.TransactionResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32) (*access.TransactionResult, error)); ok { - return rf(ctx, blockID, index) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32, execution.EventEncodingVersion) (*access.TransactionResult, error)); ok { + return rf(ctx, blockID, index, eventEncodingVersion) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32) *access.TransactionResult); ok { - r0 = rf(ctx, blockID, index) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32, execution.EventEncodingVersion) *access.TransactionResult); ok { + r0 = rf(ctx, blockID, index, eventEncodingVersion) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*access.TransactionResult) } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, uint32) error); ok { - r1 = rf(ctx, blockID, index) + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, uint32, execution.EventEncodingVersion) error); ok { + r1 = rf(ctx, blockID, index, eventEncodingVersion) } else { r1 = ret.Error(1) } @@ -645,25 +647,25 @@ func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Ide return r0, r1 } -// GetTransactionResultsByBlockID provides a mock function with given fields: ctx, blockID -func (_m *API) GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier) ([]*access.TransactionResult, error) { - ret := _m.Called(ctx, blockID) +// GetTransactionResultsByBlockID provides a mock function with given fields: ctx, blockID, eventEncodingVersion +func (_m *API) GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, eventEncodingVersion execution.EventEncodingVersion) ([]*access.TransactionResult, error) { + ret := _m.Called(ctx, blockID, eventEncodingVersion) var r0 []*access.TransactionResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) ([]*access.TransactionResult, error)); ok { - return rf(ctx, blockID) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, execution.EventEncodingVersion) ([]*access.TransactionResult, error)); ok { + return rf(ctx, blockID, eventEncodingVersion) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) []*access.TransactionResult); ok { - r0 = rf(ctx, blockID) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, execution.EventEncodingVersion) []*access.TransactionResult); ok { + r0 = rf(ctx, blockID, eventEncodingVersion) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*access.TransactionResult) } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { - r1 = rf(ctx, blockID) + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, execution.EventEncodingVersion) error); ok { + r1 = rf(ctx, blockID, eventEncodingVersion) } else { r1 = ret.Error(1) } diff --git a/engine/access/rest/apiproxy/rest_proxy_handler.go b/engine/access/rest/apiproxy/rest_proxy_handler.go index 01e7b56724d..b19e4ea9428 100644 --- a/engine/access/rest/apiproxy/rest_proxy_handler.go +++ b/engine/access/rest/apiproxy/rest_proxy_handler.go @@ -16,6 +16,8 @@ import ( "github.com/onflow/flow-go/module/metrics" accessproto "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/onflow/flow/protobuf/go/flow/entities" + execproto "github.com/onflow/flow/protobuf/go/flow/execution" ) // RestProxyHandler is a structure that represents the proxy algorithm for observer node. @@ -147,7 +149,13 @@ func (r *RestProxyHandler) GetTransaction(ctx context.Context, id flow.Identifie } // GetTransactionResult returns transaction result by the transaction ID. -func (r *RestProxyHandler) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*access.TransactionResult, error) { +func (r *RestProxyHandler) GetTransactionResult( + ctx context.Context, + id flow.Identifier, + blockID flow.Identifier, + collectionID flow.Identifier, + eventEncodingVersion execproto.EventEncodingVersion, +) (*access.TransactionResult, error) { upstream, err := r.FaultTolerantClient() if err != nil { @@ -155,9 +163,10 @@ func (r *RestProxyHandler) GetTransactionResult(ctx context.Context, id flow.Ide } getTransactionResultRequest := &accessproto.GetTransactionRequest{ - Id: id[:], - BlockId: blockID[:], - CollectionId: collectionID[:], + Id: id[:], + BlockId: blockID[:], + CollectionId: collectionID[:], + EventEncodingVersion: &entities.EventEncodingVersionValue{Value: entities.EventEncodingVersion(eventEncodingVersion)}, } transactionResultResponse, err := upstream.GetTransactionResult(ctx, getTransactionResultRequest) @@ -258,16 +267,22 @@ func (r *RestProxyHandler) ExecuteScriptAtBlockID(ctx context.Context, blockID f } // GetEventsForHeightRange returns events by their name in the specified blocks heights. -func (r *RestProxyHandler) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64) ([]flow.BlockEvents, error) { +func (r *RestProxyHandler) GetEventsForHeightRange( + ctx context.Context, + eventType string, + startHeight, endHeight uint64, + eventEncodingVersion execproto.EventEncodingVersion, +) ([]flow.BlockEvents, error) { upstream, err := r.FaultTolerantClient() if err != nil { return nil, err } getEventsForHeightRangeRequest := &accessproto.GetEventsForHeightRangeRequest{ - Type: eventType, - StartHeight: startHeight, - EndHeight: endHeight, + Type: eventType, + StartHeight: startHeight, + EndHeight: endHeight, + EventEncodingVersion: &entities.EventEncodingVersionValue{Value: entities.EventEncodingVersion(eventEncodingVersion)}, } eventsResponse, err := upstream.GetEventsForHeightRange(ctx, getEventsForHeightRangeRequest) r.log("upstream", "GetEventsForHeightRange", err) @@ -280,7 +295,7 @@ func (r *RestProxyHandler) GetEventsForHeightRange(ctx context.Context, eventTyp } // GetEventsForBlockIDs returns events by their name in the specified block IDs. -func (r *RestProxyHandler) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier) ([]flow.BlockEvents, error) { +func (r *RestProxyHandler) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersion execproto.EventEncodingVersion) ([]flow.BlockEvents, error) { upstream, err := r.FaultTolerantClient() if err != nil { return nil, err @@ -289,8 +304,9 @@ func (r *RestProxyHandler) GetEventsForBlockIDs(ctx context.Context, eventType s blockIds := convert.IdentifiersToMessages(blockIDs) getEventsForBlockIDsRequest := &accessproto.GetEventsForBlockIDsRequest{ - Type: eventType, - BlockIds: blockIds, + Type: eventType, + BlockIds: blockIds, + EventEncodingVersion: &entities.EventEncodingVersionValue{Value: entities.EventEncodingVersion(eventEncodingVersion)}, } eventsResponse, err := upstream.GetEventsForBlockIDs(ctx, getEventsForBlockIDsRequest) r.log("upstream", "GetEventsForBlockIDs", err) diff --git a/engine/access/rest/routes/events.go b/engine/access/rest/routes/events.go index 4f03624c768..86e63b8fed1 100644 --- a/engine/access/rest/routes/events.go +++ b/engine/access/rest/routes/events.go @@ -3,6 +3,8 @@ package routes import ( "fmt" + execproto "github.com/onflow/flow/protobuf/go/flow/execution" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" @@ -21,7 +23,7 @@ func GetEvents(r *request.Request, backend access.API, _ models.LinkGenerator) ( // if the request has block IDs provided then return events for block IDs var blocksEvents models.BlocksEvents if len(req.BlockIDs) > 0 { - events, err := backend.GetEventsForBlockIDs(r.Context(), req.Type, req.BlockIDs) + events, err := backend.GetEventsForBlockIDs(r.Context(), req.Type, req.BlockIDs, execproto.EventEncodingVersion_JSON_CDC_V0) if err != nil { return nil, err } @@ -45,7 +47,7 @@ func GetEvents(r *request.Request, backend access.API, _ models.LinkGenerator) ( } // if request provided block height range then return events for that range - events, err := backend.GetEventsForHeightRange(r.Context(), req.Type, req.StartHeight, req.EndHeight) + events, err := backend.GetEventsForHeightRange(r.Context(), req.Type, req.StartHeight, req.EndHeight, execproto.EventEncodingVersion_JSON_CDC_V0) if err != nil { return nil, err } diff --git a/engine/access/rest/routes/transactions.go b/engine/access/rest/routes/transactions.go index b77aead82b4..922fabaa958 100644 --- a/engine/access/rest/routes/transactions.go +++ b/engine/access/rest/routes/transactions.go @@ -4,6 +4,8 @@ import ( "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" + + execproto "github.com/onflow/flow/protobuf/go/flow/execution" ) // GetTransactionByID gets a transaction by requested ID. @@ -21,7 +23,7 @@ func GetTransactionByID(r *request.Request, backend access.API, link models.Link var txr *access.TransactionResult // only lookup result if transaction result is to be expanded if req.ExpandsResult { - txr, err = backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID) + txr, err = backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID, execproto.EventEncodingVersion_JSON_CDC_V0) if err != nil { return nil, err } @@ -39,7 +41,7 @@ func GetTransactionResultByID(r *request.Request, backend access.API, link model return nil, models.NewBadRequestError(err) } - txr, err := backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID) + txr, err := backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID, execproto.EventEncodingVersion_JSON_CDC_V0) if err != nil { return nil, err } diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index 40541ceb060..f40ec7d172b 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -36,6 +36,7 @@ func (b *backendEvents) GetEventsForHeightRange( ctx context.Context, eventType string, startHeight, endHeight uint64, + eventEncodingVersion execproto.EventEncodingVersion, ) ([]flow.BlockEvents, error) { if endHeight < startHeight { @@ -77,7 +78,7 @@ func (b *backendEvents) GetEventsForHeightRange( blockHeaders = append(blockHeaders, header) } - return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType) + return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, eventEncodingVersion) } // GetEventsForBlockIDs retrieves events for all the specified block IDs that have the given type @@ -85,6 +86,7 @@ func (b *backendEvents) GetEventsForBlockIDs( ctx context.Context, eventType string, blockIDs []flow.Identifier, + eventEncodingVersion execproto.EventEncodingVersion, ) ([]flow.BlockEvents, error) { if uint(len(blockIDs)) > b.maxHeightRange { @@ -103,13 +105,14 @@ func (b *backendEvents) GetEventsForBlockIDs( } // forward the request to the execution node - return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType) + return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, eventEncodingVersion) } func (b *backendEvents) getBlockEventsFromExecutionNode( ctx context.Context, blockHeaders []*flow.Header, eventType string, + eventEncodingVersion execproto.EventEncodingVersion, ) ([]flow.BlockEvents, error) { // create an execution API request for events at block ID @@ -149,7 +152,7 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( Msg("successfully got events") // convert execution node api result to access node api result - results, err := verifyAndConvertToAccessEvents(resp.GetResults(), blockHeaders, resp.GetEventEncodingVersion()) + results, err := verifyAndConvertToAccessEvents(resp.GetResults(), blockHeaders, eventEncodingVersion) if err != nil { return nil, status.Errorf(codes.Internal, "failed to verify retrieved events from execution node: %v", err) } diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 8df1d48ba94..c0f99e4af14 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -510,7 +510,7 @@ func (suite *Suite) TestGetTransactionResultByIndex() { Return(exeEventResp, nil). Once() - result, err := backend.GetTransactionResultByIndex(ctx, blockId, index) + result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(result.BlockHeight, block.Header.Height) @@ -560,7 +560,7 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { Return(exeEventResp, nil). Once() - result, err := backend.GetTransactionResultsByBlockID(ctx, blockId) + result, err := backend.GetTransactionResultsByBlockID(ctx, blockId, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.assertAllExpectations() @@ -639,7 +639,7 @@ func (suite *Suite) TestTransactionStatusTransition() { Times(len(fixedENIDs)) // should call each EN once // first call - when block under test is greater height than the sealed head, but execution node does not know about Tx - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) // status should be finalized since the sealed Blocks is smaller in height @@ -654,7 +654,7 @@ func (suite *Suite) TestTransactionStatusTransition() { Return(exeEventResp, nil) // second call - when block under test's height is greater height than the sealed head - result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) // status should be executed since no `NotFound` error in the `GetTransactionResult` call @@ -664,7 +664,7 @@ func (suite *Suite) TestTransactionStatusTransition() { headBlock.Header.Height = block.Header.Height + 1 // third call - when block under test's height is less than sealed head's height - result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) // status should be sealed since the sealed Blocks is greater in height @@ -675,7 +675,7 @@ func (suite *Suite) TestTransactionStatusTransition() { // fourth call - when block under test's height so much less than the head's height that it's considered expired, // but since there is a execution result, means it should retain it's sealed status - result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) // status should be expired since @@ -738,7 +738,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // should return pending status when we have not observed an expiry block suite.Run("pending", func() { // referenced block isn't known yet, so should return pending status - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) @@ -754,7 +754,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // we have NOT observed all intermediary Collections fullHeight = block.Header.Height + flow.DefaultTransactionExpiry/2 - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) }) @@ -764,7 +764,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // we have observed all intermediary Collections fullHeight = block.Header.Height + flow.DefaultTransactionExpiry + 1 - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) }) @@ -779,7 +779,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // we have observed all intermediary Collections fullHeight = block.Header.Height + flow.DefaultTransactionExpiry + 1 - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusExpired, result.Status) }) @@ -892,7 +892,7 @@ func (suite *Suite) TestTransactionPendingToFinalizedStatusTransition() { // should return pending status when we have not observed collection for the transaction suite.Run("pending", func() { currentState = flow.TransactionStatusPending - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) // assert that no call to an execution node is made @@ -903,7 +903,7 @@ func (suite *Suite) TestTransactionPendingToFinalizedStatusTransition() { // preceding sealed refBlock) suite.Run("finalized", func() { currentState = flow.TransactionStatusFinalized - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusFinalized, result.Status) }) @@ -929,7 +929,7 @@ func (suite *Suite) TestTransactionResultUnknown() { suite.Require().NoError(err) // first call - when block under test is greater height than the sealed head, but execution node does not know about Tx - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) // status should be reported as unknown @@ -1082,7 +1082,7 @@ func (suite *Suite) TestGetEventsForBlockIDs() { suite.Require().NoError(err) // execute request - actual, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), blockIDs) + actual, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), blockIDs, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(actual, err) suite.Require().Equal(expected, actual) @@ -1100,7 +1100,7 @@ func (suite *Suite) TestGetEventsForBlockIDs() { suite.Require().NoError(err) // execute request with an empty block id list and expect an empty list of events and no error - resp, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), []flow.Identifier{}) + resp, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), []flow.Identifier{}, execproto.EventEncodingVersion_JSON_CDC_V0) require.NoError(suite.T(), err) require.Empty(suite.T(), resp) }) @@ -1354,7 +1354,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), maxHeight, minHeight) + _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), maxHeight, minHeight, execproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) suite.assertAllExpectations() // assert that request was not sent to execution node @@ -1384,7 +1384,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { suite.Require().NoError(err) // execute request - actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight) + actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, execproto.EventEncodingVersion_JSON_CDC_V0) // check response suite.checkResponse(actualResp, err) @@ -1412,7 +1412,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight) + actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(actualResp, err) suite.assertAllExpectations() @@ -1434,7 +1434,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, minHeight+1) + _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, minHeight+1, execproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) }) @@ -1456,7 +1456,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight) + _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, execproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) }) diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 38944173b69..c6f104e839b 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -228,6 +228,7 @@ func (b *backendTransactions) GetTransactionResult( txID flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, + eventEncodingVersion execproto.EventEncodingVersion, ) (*access.TransactionResult, error) { // look up transaction from storage start := time.Now() @@ -286,7 +287,7 @@ func (b *backendTransactions) GetTransactionResult( // access node may not have the block if it hasn't yet been finalized, hence block can be nil at this point if block != nil { foundBlockID := block.ID() - transactionWasExecuted, events, statusCode, txError, err = b.lookupTransactionResult(ctx, txID, foundBlockID) + transactionWasExecuted, events, statusCode, txError, err = b.lookupTransactionResult(ctx, txID, foundBlockID, eventEncodingVersion) if err != nil { return nil, rpc.ConvertError(err, "failed to retrieve result from any execution node", codes.Internal) } @@ -385,6 +386,7 @@ func (b *backendTransactions) retrieveBlock( func (b *backendTransactions) GetTransactionResultsByBlockID( ctx context.Context, blockID flow.Identifier, + eventEncodingVersion execproto.EventEncodingVersion, ) ([]*access.TransactionResult, error) { // TODO: consider using storage.Index.ByBlockID, the index contains collection id and seals ID block, err := b.blocks.ByID(blockID) @@ -435,7 +437,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( return nil, rpc.ConvertStorageError(err) } - events, err := convert.MessagesToEventsFromVersion(txResult.GetEvents(), resp.GetEventEncodingVersion()) + events, err := convert.MessagesToEventsFromVersion(txResult.GetEvents(), eventEncodingVersion) if err != nil { return nil, status.Errorf(codes.Internal, "failed to convert events to message in txID %x: %v", txID, err) @@ -515,6 +517,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( ctx context.Context, blockID flow.Identifier, index uint32, + eventEncodingVersion execproto.EventEncodingVersion, ) (*access.TransactionResult, error) { // TODO: https://github.com/onflow/flow-go/issues/2175 so caching doesn't cause a circular dependency block, err := b.blocks.ByID(blockID) @@ -547,7 +550,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( return nil, rpc.ConvertStorageError(err) } - events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), resp.GetEventEncodingVersion()) + events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) if err != nil { return nil, status.Errorf(codes.Internal, "failed to convert events in blockID %x: %v", blockID, err) } @@ -664,9 +667,10 @@ func (b *backendTransactions) lookupTransactionResult( ctx context.Context, txID flow.Identifier, blockID flow.Identifier, + eventEncodingVersion execproto.EventEncodingVersion, ) (bool, []flow.Event, uint32, string, error) { - events, txStatus, message, err := b.getTransactionResultFromExecutionNode(ctx, blockID, txID[:]) + events, txStatus, message, err := b.getTransactionResultFromExecutionNode(ctx, blockID, txID[:], eventEncodingVersion) if err != nil { // if either the execution node reported no results or the execution node could not be chosen if status.Code(err) == codes.NotFound { @@ -749,6 +753,7 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( ctx context.Context, blockID flow.Identifier, transactionID []byte, + eventEncodingVersion execproto.EventEncodingVersion, ) ([]flow.Event, uint32, string, error) { // create an execution API request for events at blockID and transactionID @@ -771,7 +776,7 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return nil, 0, "", err } - events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), resp.GetEventEncodingVersion()) + events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) if err != nil { return nil, 0, "", rpc.ConvertError(err, "failed to convert events to message", codes.Internal) } diff --git a/engine/access/rpc/backend/backend_transactions_test.go b/engine/access/rpc/backend/backend_transactions_test.go index ffe4d4a3734..3761ba272d5 100644 --- a/engine/access/rpc/backend/backend_transactions_test.go +++ b/engine/access/rpc/backend/backend_transactions_test.go @@ -7,6 +7,7 @@ import ( "github.com/dgraph-io/badger/v2" "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" + execproto "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" @@ -74,7 +75,7 @@ func (suite *Suite) TestGetTransactionResultReturnsUnknown() { backend, err := New(params) suite.Require().NoError(err) - res, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID()) + res, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().NoError(err) suite.Require().Equal(res.Status, flow.TransactionStatusUnknown) }) @@ -107,7 +108,7 @@ func (suite *Suite) TestGetTransactionResultReturnsTransactionError() { backend, err := New(params) suite.Require().NoError(err) - _, err = backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID()) + _, err = backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Equal(err, status.Errorf(codes.Internal, "failed to find: %v", fmt.Errorf("some other error"))) }) @@ -145,7 +146,7 @@ func (suite *Suite) TestGetTransactionResultReturnsValidTransactionResultFromHis backend, err := New(params) suite.Require().NoError(err) - resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID()) + resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusExecuted, resp.Status) suite.Require().Equal(uint(flow.TransactionStatusExecuted), resp.StatusCode) @@ -191,12 +192,12 @@ func (suite *Suite) TestGetTransactionResultFromCache() { coll := flow.CollectionFromTransactions([]*flow.Transaction{tx}) - resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID()) + resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusExecuted, resp.Status) suite.Require().Equal(uint(flow.TransactionStatusExecuted), resp.StatusCode) - resp2, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID()) + resp2, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusExecuted, resp2.Status) suite.Require().Equal(uint(flow.TransactionStatusExecuted), resp2.StatusCode) @@ -223,7 +224,7 @@ func (suite *Suite) TestGetTransactionResultCacheNonExistent() { coll := flow.CollectionFromTransactions([]*flow.Transaction{tx}) - resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID()) + resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusUnknown, resp.Status) suite.Require().Equal(uint(flow.TransactionStatusUnknown), resp.StatusCode) @@ -258,7 +259,7 @@ func (suite *Suite) TestGetTransactionResultUnknownFromCache() { coll := flow.CollectionFromTransactions([]*flow.Transaction{tx}) - resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID()) + resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusUnknown, resp.Status) suite.Require().Equal(uint(flow.TransactionStatusUnknown), resp.StatusCode) @@ -272,7 +273,7 @@ func (suite *Suite) TestGetTransactionResultUnknownFromCache() { StatusCode: uint(txStatus), }) - resp2, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID()) + resp2, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusUnknown, resp2.Status) suite.Require().Equal(uint(flow.TransactionStatusUnknown), resp2.StatusCode) diff --git a/engine/access/rpc/backend/historical_access_test.go b/engine/access/rpc/backend/historical_access_test.go index 9593462339c..78682c8758b 100644 --- a/engine/access/rpc/backend/historical_access_test.go +++ b/engine/access/rpc/backend/historical_access_test.go @@ -5,6 +5,7 @@ import ( accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" + execproto "github.com/onflow/flow/protobuf/go/flow/execution" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -48,7 +49,7 @@ func (suite *Suite) TestHistoricalTransactionResult() { Once() // Make the call for the transaction result - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) // status should be sealed diff --git a/engine/access/rpc/backend/retry_test.go b/engine/access/rpc/backend/retry_test.go index 28de0341b67..7936cdfacc7 100644 --- a/engine/access/rpc/backend/retry_test.go +++ b/engine/access/rpc/backend/retry_test.go @@ -130,7 +130,7 @@ func (suite *Suite) TestSuccessfulTransactionsDontRetry() { Times(len(enIDs)) // should call each EN once // first call - when block under test is greater height than the sealed head, but execution node does not know about Tx - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execution.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) // status should be finalized since the sealed Blocks is smaller in height diff --git a/engine/execution/state/mock/script_execution_state.go b/engine/execution/state/mock/script_execution_state.go new file mode 100644 index 00000000000..904defab7fa --- /dev/null +++ b/engine/execution/state/mock/script_execution_state.go @@ -0,0 +1,88 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + context "context" + + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" + + snapshot "github.com/onflow/flow-go/fvm/storage/snapshot" +) + +// ScriptExecutionState is an autogenerated mock type for the ScriptExecutionState type +type ScriptExecutionState struct { + mock.Mock +} + +// HasState provides a mock function with given fields: _a0 +func (_m *ScriptExecutionState) HasState(_a0 flow.StateCommitment) bool { + ret := _m.Called(_a0) + + var r0 bool + if rf, ok := ret.Get(0).(func(flow.StateCommitment) bool); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// NewStorageSnapshot provides a mock function with given fields: _a0 +func (_m *ScriptExecutionState) NewStorageSnapshot(_a0 flow.StateCommitment) snapshot.StorageSnapshot { + ret := _m.Called(_a0) + + var r0 snapshot.StorageSnapshot + if rf, ok := ret.Get(0).(func(flow.StateCommitment) snapshot.StorageSnapshot); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(snapshot.StorageSnapshot) + } + } + + return r0 +} + +// StateCommitmentByBlockID provides a mock function with given fields: _a0, _a1 +func (_m *ScriptExecutionState) StateCommitmentByBlockID(_a0 context.Context, _a1 flow.Identifier) (flow.StateCommitment, error) { + ret := _m.Called(_a0, _a1) + + var r0 flow.StateCommitment + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (flow.StateCommitment, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) flow.StateCommitment); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.StateCommitment) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewScriptExecutionState interface { + mock.TestingT + Cleanup(func()) +} + +// NewScriptExecutionState creates a new instance of ScriptExecutionState. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewScriptExecutionState(t mockConstructorTestingTNewScriptExecutionState) *ScriptExecutionState { + mock := &ScriptExecutionState{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/go.mod b/go.mod index 458eb924f3f..48066538584 100644 --- a/go.mod +++ b/go.mod @@ -294,3 +294,5 @@ require ( lukechampine.com/blake3 v1.2.1 // indirect nhooyr.io/websocket v1.8.7 // indirect ) + +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231003102535-25fed8fe4f6a diff --git a/go.sum b/go.sum index 601b1843764..639e16b7a50 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231003102535-25fed8fe4f6a h1:HsCZqXvGHxJ1GZPgf2anx87MHoGUMe4BP2MmLpSENdc= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231003102535-25fed8fe4f6a/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -1327,8 +1329,6 @@ github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 h1:IGDANryEVnuVAlDZDzNLMMUui9lbwS04KE70C0HXkFw= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= diff --git a/integration/go.mod b/integration/go.mod index 7528980db5b..129b62e709a 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -341,3 +341,5 @@ require ( replace github.com/onflow/flow-go => ../ replace github.com/onflow/flow-go/insecure => ../insecure + +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231003102535-25fed8fe4f6a diff --git a/integration/go.sum b/integration/go.sum index f44ac8f6da8..0f4d3c79c65 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -100,6 +100,8 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231003102535-25fed8fe4f6a h1:HsCZqXvGHxJ1GZPgf2anx87MHoGUMe4BP2MmLpSENdc= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231003102535-25fed8fe4f6a/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= @@ -1444,8 +1446,6 @@ github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 h1:IGDANryEVnuVAlDZDzNLMMUui9lbwS04KE70C0HXkFw= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/nft-storefront/lib/go/contracts v0.0.0-20221222181731-14b90207cead h1:2j1Unqs76Z1b95Gu4C3Y28hzNUHBix7wL490e61SMSw= diff --git a/storage/mock/light_transaction_results.go b/storage/mock/light_transaction_results.go new file mode 100644 index 00000000000..4b8f1f6980e --- /dev/null +++ b/storage/mock/light_transaction_results.go @@ -0,0 +1,122 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" + + storage "github.com/onflow/flow-go/storage" +) + +// LightTransactionResults is an autogenerated mock type for the LightTransactionResults type +type LightTransactionResults struct { + mock.Mock +} + +// BatchStore provides a mock function with given fields: blockID, transactionResults, batch +func (_m *LightTransactionResults) BatchStore(blockID flow.Identifier, transactionResults []flow.LightTransactionResult, batch storage.BatchStorage) error { + ret := _m.Called(blockID, transactionResults, batch) + + var r0 error + if rf, ok := ret.Get(0).(func(flow.Identifier, []flow.LightTransactionResult, storage.BatchStorage) error); ok { + r0 = rf(blockID, transactionResults, batch) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ByBlockID provides a mock function with given fields: id +func (_m *LightTransactionResults) ByBlockID(id flow.Identifier) ([]flow.LightTransactionResult, error) { + ret := _m.Called(id) + + var r0 []flow.LightTransactionResult + var r1 error + if rf, ok := ret.Get(0).(func(flow.Identifier) ([]flow.LightTransactionResult, error)); ok { + return rf(id) + } + if rf, ok := ret.Get(0).(func(flow.Identifier) []flow.LightTransactionResult); ok { + r0 = rf(id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]flow.LightTransactionResult) + } + } + + if rf, ok := ret.Get(1).(func(flow.Identifier) error); ok { + r1 = rf(id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ByBlockIDTransactionID provides a mock function with given fields: blockID, transactionID +func (_m *LightTransactionResults) ByBlockIDTransactionID(blockID flow.Identifier, transactionID flow.Identifier) (*flow.LightTransactionResult, error) { + ret := _m.Called(blockID, transactionID) + + var r0 *flow.LightTransactionResult + var r1 error + if rf, ok := ret.Get(0).(func(flow.Identifier, flow.Identifier) (*flow.LightTransactionResult, error)); ok { + return rf(blockID, transactionID) + } + if rf, ok := ret.Get(0).(func(flow.Identifier, flow.Identifier) *flow.LightTransactionResult); ok { + r0 = rf(blockID, transactionID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.LightTransactionResult) + } + } + + if rf, ok := ret.Get(1).(func(flow.Identifier, flow.Identifier) error); ok { + r1 = rf(blockID, transactionID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ByBlockIDTransactionIndex provides a mock function with given fields: blockID, txIndex +func (_m *LightTransactionResults) ByBlockIDTransactionIndex(blockID flow.Identifier, txIndex uint32) (*flow.LightTransactionResult, error) { + ret := _m.Called(blockID, txIndex) + + var r0 *flow.LightTransactionResult + var r1 error + if rf, ok := ret.Get(0).(func(flow.Identifier, uint32) (*flow.LightTransactionResult, error)); ok { + return rf(blockID, txIndex) + } + if rf, ok := ret.Get(0).(func(flow.Identifier, uint32) *flow.LightTransactionResult); ok { + r0 = rf(blockID, txIndex) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.LightTransactionResult) + } + } + + if rf, ok := ret.Get(1).(func(flow.Identifier, uint32) error); ok { + r1 = rf(blockID, txIndex) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewLightTransactionResults interface { + mock.TestingT + Cleanup(func()) +} + +// NewLightTransactionResults creates a new instance of LightTransactionResults. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewLightTransactionResults(t mockConstructorTestingTNewLightTransactionResults) *LightTransactionResults { + mock := &LightTransactionResults{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From e6465e0a27193df95ca797fea01013cc99b93a8c Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Tue, 3 Oct 2023 16:49:53 +0300 Subject: [PATCH 02/27] Make event encoding version optional. --- access/api.go | 11 +-- access/handler.go | 26 +----- access/legacy/handler.go | 7 +- access/mock/api.go | 92 +++++++++---------- .../rest/apiproxy/rest_proxy_handler.go | 18 ++-- engine/access/rest/routes/events.go | 6 +- engine/access/rest/routes/transactions.go | 6 +- engine/access/rpc/backend/backend_events.go | 17 +++- engine/access/rpc/backend/backend_test.go | 40 ++++---- .../rpc/backend/backend_transactions.go | 32 +++++-- .../rpc/backend/backend_transactions_test.go | 17 ++-- .../rpc/backend/historical_access_test.go | 3 +- engine/access/rpc/backend/retry_test.go | 2 +- 13 files changed, 139 insertions(+), 138 deletions(-) diff --git a/access/api.go b/access/api.go index 24e2c53c5a1..66ee14ae1d5 100644 --- a/access/api.go +++ b/access/api.go @@ -5,7 +5,6 @@ import ( "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" - execproto "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" @@ -30,9 +29,9 @@ type API interface { SendTransaction(ctx context.Context, tx *flow.TransactionBody) error GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error) GetTransactionsByBlockID(ctx context.Context, blockID flow.Identifier) ([]*flow.TransactionBody, error) - GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, eventEncodingVersion execproto.EventEncodingVersion) (*TransactionResult, error) - GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, eventEncodingVersion execproto.EventEncodingVersion) (*TransactionResult, error) - GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, eventEncodingVersion execproto.EventEncodingVersion) ([]*TransactionResult, error) + GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, eventEncodingVersionValue *entities.EventEncodingVersionValue) (*TransactionResult, error) + GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, eventEncodingVersionValue *entities.EventEncodingVersionValue) (*TransactionResult, error) + GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, eventEncodingVersionValue *entities.EventEncodingVersionValue) ([]*TransactionResult, error) GetAccount(ctx context.Context, address flow.Address) (*flow.Account, error) GetAccountAtLatestBlock(ctx context.Context, address flow.Address) (*flow.Account, error) @@ -42,8 +41,8 @@ type API interface { ExecuteScriptAtBlockHeight(ctx context.Context, blockHeight uint64, script []byte, arguments [][]byte) ([]byte, error) ExecuteScriptAtBlockID(ctx context.Context, blockID flow.Identifier, script []byte, arguments [][]byte) ([]byte, error) - GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64, eventEncodingVersion execproto.EventEncodingVersion) ([]flow.BlockEvents, error) - GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersion execproto.EventEncodingVersion) ([]flow.BlockEvents, error) + GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64, eventEncodingVersionValue *entities.EventEncodingVersionValue) ([]flow.BlockEvents, error) + GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersionValue *entities.EventEncodingVersionValue) ([]flow.BlockEvents, error) GetLatestProtocolStateSnapshot(ctx context.Context) ([]byte, error) diff --git a/access/handler.go b/access/handler.go index 7180236b97a..05773c2202d 100644 --- a/access/handler.go +++ b/access/handler.go @@ -14,7 +14,6 @@ import ( "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" - execproto "github.com/onflow/flow/protobuf/go/flow/execution" ) type Handler struct { @@ -275,10 +274,7 @@ func (h *Handler) GetTransactionResult( } } - eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 - if requestEEV := req.GetEventEncodingVersion(); requestEEV != nil { - eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) - } + eventEncodingVersion := req.GetEventEncodingVersion() result, err := h.api.GetTransactionResult(ctx, transactionID, blockId, collectionId, eventEncodingVersion) if err != nil { @@ -302,10 +298,7 @@ func (h *Handler) GetTransactionResultsByBlockID( return nil, err } - eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 - if requestEEV := req.GetEventEncodingVersion(); requestEEV != nil { - eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) - } + eventEncodingVersion := req.GetEventEncodingVersion() results, err := h.api.GetTransactionResultsByBlockID(ctx, id, eventEncodingVersion) if err != nil { @@ -353,10 +346,7 @@ func (h *Handler) GetTransactionResultByIndex( return nil, err } - eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 - if requestEEV := req.GetEventEncodingVersion(); requestEEV != nil { - eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) - } + eventEncodingVersion := req.GetEventEncodingVersion() result, err := h.api.GetTransactionResultByIndex(ctx, blockID, req.GetIndex(), eventEncodingVersion) if err != nil { @@ -529,10 +519,7 @@ func (h *Handler) GetEventsForHeightRange( startHeight := req.GetStartHeight() endHeight := req.GetEndHeight() - eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 - if requestEEV := req.GetEventEncodingVersion(); requestEEV != nil { - eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) - } + eventEncodingVersion := req.GetEventEncodingVersion() results, err := h.api.GetEventsForHeightRange(ctx, eventType, startHeight, endHeight, eventEncodingVersion) if err != nil { @@ -566,10 +553,7 @@ func (h *Handler) GetEventsForBlockIDs( return nil, err } - eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 - if requestEEV := req.GetEventEncodingVersion(); requestEEV != nil { - eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) - } + eventEncodingVersion := req.GetEventEncodingVersion() results, err := h.api.GetEventsForBlockIDs(ctx, eventType, blockIDs, eventEncodingVersion) if err != nil { diff --git a/access/legacy/handler.go b/access/legacy/handler.go index 24218615489..f81090daab0 100644 --- a/access/legacy/handler.go +++ b/access/legacy/handler.go @@ -3,7 +3,6 @@ package handler import ( "context" - execproto "github.com/onflow/flow/protobuf/go/flow/execution" accessproto "github.com/onflow/flow/protobuf/go/flow/legacy/access" entitiesproto "github.com/onflow/flow/protobuf/go/flow/legacy/entities" "google.golang.org/grpc/codes" @@ -190,7 +189,7 @@ func (h *Handler) GetTransactionResult( ) (*accessproto.TransactionResultResponse, error) { id := convert.MessageToIdentifier(req.GetId()) - result, err := h.api.GetTransactionResult(ctx, id, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err := h.api.GetTransactionResult(ctx, id, flow.ZeroID, flow.ZeroID, nil) if err != nil { return nil, err } @@ -314,7 +313,7 @@ func (h *Handler) GetEventsForHeightRange( startHeight := req.GetStartHeight() endHeight := req.GetEndHeight() - results, err := h.api.GetEventsForHeightRange(ctx, eventType, startHeight, endHeight, execproto.EventEncodingVersion_JSON_CDC_V0) + results, err := h.api.GetEventsForHeightRange(ctx, eventType, startHeight, endHeight, nil) if err != nil { return nil, err } @@ -332,7 +331,7 @@ func (h *Handler) GetEventsForBlockIDs( eventType := req.GetType() blockIDs := convert.MessagesToIdentifiers(req.GetBlockIds()) - results, err := h.api.GetEventsForBlockIDs(ctx, eventType, blockIDs, execproto.EventEncodingVersion_JSON_CDC_V0) + results, err := h.api.GetEventsForBlockIDs(ctx, eventType, blockIDs, nil) if err != nil { return nil, err } diff --git a/access/mock/api.go b/access/mock/api.go index 4065590969a..9a3f2a8db08 100644 --- a/access/mock/api.go +++ b/access/mock/api.go @@ -7,7 +7,7 @@ import ( access "github.com/onflow/flow-go/access" - execution "github.com/onflow/flow/protobuf/go/flow/execution" + entities "github.com/onflow/flow/protobuf/go/flow/entities" flow "github.com/onflow/flow-go/model/flow" @@ -333,25 +333,25 @@ func (_m *API) GetCollectionByID(ctx context.Context, id flow.Identifier) (*flow return r0, r1 } -// GetEventsForBlockIDs provides a mock function with given fields: ctx, eventType, blockIDs, eventEncodingVersion -func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersion execution.EventEncodingVersion) ([]flow.BlockEvents, error) { - ret := _m.Called(ctx, eventType, blockIDs, eventEncodingVersion) +// GetEventsForBlockIDs provides a mock function with given fields: ctx, eventType, blockIDs, eventEncodingVersionValue +func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersionValue *entities.EventEncodingVersionValue) ([]flow.BlockEvents, error) { + ret := _m.Called(ctx, eventType, blockIDs, eventEncodingVersionValue) var r0 []flow.BlockEvents var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier, execution.EventEncodingVersion) ([]flow.BlockEvents, error)); ok { - return rf(ctx, eventType, blockIDs, eventEncodingVersion) + if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier, *entities.EventEncodingVersionValue) ([]flow.BlockEvents, error)); ok { + return rf(ctx, eventType, blockIDs, eventEncodingVersionValue) } - if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier, execution.EventEncodingVersion) []flow.BlockEvents); ok { - r0 = rf(ctx, eventType, blockIDs, eventEncodingVersion) + if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier, *entities.EventEncodingVersionValue) []flow.BlockEvents); ok { + r0 = rf(ctx, eventType, blockIDs, eventEncodingVersionValue) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]flow.BlockEvents) } } - if rf, ok := ret.Get(1).(func(context.Context, string, []flow.Identifier, execution.EventEncodingVersion) error); ok { - r1 = rf(ctx, eventType, blockIDs, eventEncodingVersion) + if rf, ok := ret.Get(1).(func(context.Context, string, []flow.Identifier, *entities.EventEncodingVersionValue) error); ok { + r1 = rf(ctx, eventType, blockIDs, eventEncodingVersionValue) } else { r1 = ret.Error(1) } @@ -359,25 +359,25 @@ func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, block return r0, r1 } -// GetEventsForHeightRange provides a mock function with given fields: ctx, eventType, startHeight, endHeight, eventEncodingVersion -func (_m *API) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight uint64, endHeight uint64, eventEncodingVersion execution.EventEncodingVersion) ([]flow.BlockEvents, error) { - ret := _m.Called(ctx, eventType, startHeight, endHeight, eventEncodingVersion) +// GetEventsForHeightRange provides a mock function with given fields: ctx, eventType, startHeight, endHeight, eventEncodingVersionValue +func (_m *API) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight uint64, endHeight uint64, eventEncodingVersionValue *entities.EventEncodingVersionValue) ([]flow.BlockEvents, error) { + ret := _m.Called(ctx, eventType, startHeight, endHeight, eventEncodingVersionValue) var r0 []flow.BlockEvents var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64, execution.EventEncodingVersion) ([]flow.BlockEvents, error)); ok { - return rf(ctx, eventType, startHeight, endHeight, eventEncodingVersion) + if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64, *entities.EventEncodingVersionValue) ([]flow.BlockEvents, error)); ok { + return rf(ctx, eventType, startHeight, endHeight, eventEncodingVersionValue) } - if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64, execution.EventEncodingVersion) []flow.BlockEvents); ok { - r0 = rf(ctx, eventType, startHeight, endHeight, eventEncodingVersion) + if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64, *entities.EventEncodingVersionValue) []flow.BlockEvents); ok { + r0 = rf(ctx, eventType, startHeight, endHeight, eventEncodingVersionValue) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]flow.BlockEvents) } } - if rf, ok := ret.Get(1).(func(context.Context, string, uint64, uint64, execution.EventEncodingVersion) error); ok { - r1 = rf(ctx, eventType, startHeight, endHeight, eventEncodingVersion) + if rf, ok := ret.Get(1).(func(context.Context, string, uint64, uint64, *entities.EventEncodingVersionValue) error); ok { + r1 = rf(ctx, eventType, startHeight, endHeight, eventEncodingVersionValue) } else { r1 = ret.Error(1) } @@ -595,25 +595,25 @@ func (_m *API) GetTransaction(ctx context.Context, id flow.Identifier) (*flow.Tr return r0, r1 } -// GetTransactionResult provides a mock function with given fields: ctx, id, blockID, collectionID, eventEncodingVersion -func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, eventEncodingVersion execution.EventEncodingVersion) (*access.TransactionResult, error) { - ret := _m.Called(ctx, id, blockID, collectionID, eventEncodingVersion) +// GetTransactionResult provides a mock function with given fields: ctx, id, blockID, collectionID, eventEncodingVersionValue +func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, eventEncodingVersionValue *entities.EventEncodingVersionValue) (*access.TransactionResult, error) { + ret := _m.Called(ctx, id, blockID, collectionID, eventEncodingVersionValue) var r0 *access.TransactionResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, execution.EventEncodingVersion) (*access.TransactionResult, error)); ok { - return rf(ctx, id, blockID, collectionID, eventEncodingVersion) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, *entities.EventEncodingVersionValue) (*access.TransactionResult, error)); ok { + return rf(ctx, id, blockID, collectionID, eventEncodingVersionValue) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, execution.EventEncodingVersion) *access.TransactionResult); ok { - r0 = rf(ctx, id, blockID, collectionID, eventEncodingVersion) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, *entities.EventEncodingVersionValue) *access.TransactionResult); ok { + r0 = rf(ctx, id, blockID, collectionID, eventEncodingVersionValue) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*access.TransactionResult) } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, execution.EventEncodingVersion) error); ok { - r1 = rf(ctx, id, blockID, collectionID, eventEncodingVersion) + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, *entities.EventEncodingVersionValue) error); ok { + r1 = rf(ctx, id, blockID, collectionID, eventEncodingVersionValue) } else { r1 = ret.Error(1) } @@ -621,25 +621,25 @@ func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blo return r0, r1 } -// GetTransactionResultByIndex provides a mock function with given fields: ctx, blockID, index, eventEncodingVersion -func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, eventEncodingVersion execution.EventEncodingVersion) (*access.TransactionResult, error) { - ret := _m.Called(ctx, blockID, index, eventEncodingVersion) +// GetTransactionResultByIndex provides a mock function with given fields: ctx, blockID, index, eventEncodingVersionValue +func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, eventEncodingVersionValue *entities.EventEncodingVersionValue) (*access.TransactionResult, error) { + ret := _m.Called(ctx, blockID, index, eventEncodingVersionValue) var r0 *access.TransactionResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32, execution.EventEncodingVersion) (*access.TransactionResult, error)); ok { - return rf(ctx, blockID, index, eventEncodingVersion) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32, *entities.EventEncodingVersionValue) (*access.TransactionResult, error)); ok { + return rf(ctx, blockID, index, eventEncodingVersionValue) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32, execution.EventEncodingVersion) *access.TransactionResult); ok { - r0 = rf(ctx, blockID, index, eventEncodingVersion) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32, *entities.EventEncodingVersionValue) *access.TransactionResult); ok { + r0 = rf(ctx, blockID, index, eventEncodingVersionValue) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*access.TransactionResult) } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, uint32, execution.EventEncodingVersion) error); ok { - r1 = rf(ctx, blockID, index, eventEncodingVersion) + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, uint32, *entities.EventEncodingVersionValue) error); ok { + r1 = rf(ctx, blockID, index, eventEncodingVersionValue) } else { r1 = ret.Error(1) } @@ -647,25 +647,25 @@ func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Ide return r0, r1 } -// GetTransactionResultsByBlockID provides a mock function with given fields: ctx, blockID, eventEncodingVersion -func (_m *API) GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, eventEncodingVersion execution.EventEncodingVersion) ([]*access.TransactionResult, error) { - ret := _m.Called(ctx, blockID, eventEncodingVersion) +// GetTransactionResultsByBlockID provides a mock function with given fields: ctx, blockID, eventEncodingVersionValue +func (_m *API) GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, eventEncodingVersionValue *entities.EventEncodingVersionValue) ([]*access.TransactionResult, error) { + ret := _m.Called(ctx, blockID, eventEncodingVersionValue) var r0 []*access.TransactionResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, execution.EventEncodingVersion) ([]*access.TransactionResult, error)); ok { - return rf(ctx, blockID, eventEncodingVersion) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, *entities.EventEncodingVersionValue) ([]*access.TransactionResult, error)); ok { + return rf(ctx, blockID, eventEncodingVersionValue) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, execution.EventEncodingVersion) []*access.TransactionResult); ok { - r0 = rf(ctx, blockID, eventEncodingVersion) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, *entities.EventEncodingVersionValue) []*access.TransactionResult); ok { + r0 = rf(ctx, blockID, eventEncodingVersionValue) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*access.TransactionResult) } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, execution.EventEncodingVersion) error); ok { - r1 = rf(ctx, blockID, eventEncodingVersion) + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, *entities.EventEncodingVersionValue) error); ok { + r1 = rf(ctx, blockID, eventEncodingVersionValue) } else { r1 = ret.Error(1) } diff --git a/engine/access/rest/apiproxy/rest_proxy_handler.go b/engine/access/rest/apiproxy/rest_proxy_handler.go index b19e4ea9428..e65eb96be5e 100644 --- a/engine/access/rest/apiproxy/rest_proxy_handler.go +++ b/engine/access/rest/apiproxy/rest_proxy_handler.go @@ -17,7 +17,6 @@ import ( accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" - execproto "github.com/onflow/flow/protobuf/go/flow/execution" ) // RestProxyHandler is a structure that represents the proxy algorithm for observer node. @@ -154,7 +153,7 @@ func (r *RestProxyHandler) GetTransactionResult( id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, - eventEncodingVersion execproto.EventEncodingVersion, + eventEncodingVersionValue *entities.EventEncodingVersionValue, ) (*access.TransactionResult, error) { upstream, err := r.FaultTolerantClient() if err != nil { @@ -166,7 +165,7 @@ func (r *RestProxyHandler) GetTransactionResult( Id: id[:], BlockId: blockID[:], CollectionId: collectionID[:], - EventEncodingVersion: &entities.EventEncodingVersionValue{Value: entities.EventEncodingVersion(eventEncodingVersion)}, + EventEncodingVersion: eventEncodingVersionValue, } transactionResultResponse, err := upstream.GetTransactionResult(ctx, getTransactionResultRequest) @@ -271,7 +270,7 @@ func (r *RestProxyHandler) GetEventsForHeightRange( ctx context.Context, eventType string, startHeight, endHeight uint64, - eventEncodingVersion execproto.EventEncodingVersion, + eventEncodingVersionValue *entities.EventEncodingVersionValue, ) ([]flow.BlockEvents, error) { upstream, err := r.FaultTolerantClient() if err != nil { @@ -282,7 +281,7 @@ func (r *RestProxyHandler) GetEventsForHeightRange( Type: eventType, StartHeight: startHeight, EndHeight: endHeight, - EventEncodingVersion: &entities.EventEncodingVersionValue{Value: entities.EventEncodingVersion(eventEncodingVersion)}, + EventEncodingVersion: eventEncodingVersionValue, } eventsResponse, err := upstream.GetEventsForHeightRange(ctx, getEventsForHeightRangeRequest) r.log("upstream", "GetEventsForHeightRange", err) @@ -295,7 +294,12 @@ func (r *RestProxyHandler) GetEventsForHeightRange( } // GetEventsForBlockIDs returns events by their name in the specified block IDs. -func (r *RestProxyHandler) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersion execproto.EventEncodingVersion) ([]flow.BlockEvents, error) { +func (r *RestProxyHandler) GetEventsForBlockIDs( + ctx context.Context, + eventType string, + blockIDs []flow.Identifier, + eventEncodingVersionValue *entities.EventEncodingVersionValue, +) ([]flow.BlockEvents, error) { upstream, err := r.FaultTolerantClient() if err != nil { return nil, err @@ -306,7 +310,7 @@ func (r *RestProxyHandler) GetEventsForBlockIDs(ctx context.Context, eventType s getEventsForBlockIDsRequest := &accessproto.GetEventsForBlockIDsRequest{ Type: eventType, BlockIds: blockIds, - EventEncodingVersion: &entities.EventEncodingVersionValue{Value: entities.EventEncodingVersion(eventEncodingVersion)}, + EventEncodingVersion: eventEncodingVersionValue, } eventsResponse, err := upstream.GetEventsForBlockIDs(ctx, getEventsForBlockIDsRequest) r.log("upstream", "GetEventsForBlockIDs", err) diff --git a/engine/access/rest/routes/events.go b/engine/access/rest/routes/events.go index 86e63b8fed1..b5ba02e34de 100644 --- a/engine/access/rest/routes/events.go +++ b/engine/access/rest/routes/events.go @@ -3,8 +3,6 @@ package routes import ( "fmt" - execproto "github.com/onflow/flow/protobuf/go/flow/execution" - "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" @@ -23,7 +21,7 @@ func GetEvents(r *request.Request, backend access.API, _ models.LinkGenerator) ( // if the request has block IDs provided then return events for block IDs var blocksEvents models.BlocksEvents if len(req.BlockIDs) > 0 { - events, err := backend.GetEventsForBlockIDs(r.Context(), req.Type, req.BlockIDs, execproto.EventEncodingVersion_JSON_CDC_V0) + events, err := backend.GetEventsForBlockIDs(r.Context(), req.Type, req.BlockIDs, nil) if err != nil { return nil, err } @@ -47,7 +45,7 @@ func GetEvents(r *request.Request, backend access.API, _ models.LinkGenerator) ( } // if request provided block height range then return events for that range - events, err := backend.GetEventsForHeightRange(r.Context(), req.Type, req.StartHeight, req.EndHeight, execproto.EventEncodingVersion_JSON_CDC_V0) + events, err := backend.GetEventsForHeightRange(r.Context(), req.Type, req.StartHeight, req.EndHeight, nil) if err != nil { return nil, err } diff --git a/engine/access/rest/routes/transactions.go b/engine/access/rest/routes/transactions.go index 922fabaa958..52dfa81a2e3 100644 --- a/engine/access/rest/routes/transactions.go +++ b/engine/access/rest/routes/transactions.go @@ -4,8 +4,6 @@ import ( "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" - - execproto "github.com/onflow/flow/protobuf/go/flow/execution" ) // GetTransactionByID gets a transaction by requested ID. @@ -23,7 +21,7 @@ func GetTransactionByID(r *request.Request, backend access.API, link models.Link var txr *access.TransactionResult // only lookup result if transaction result is to be expanded if req.ExpandsResult { - txr, err = backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID, execproto.EventEncodingVersion_JSON_CDC_V0) + txr, err = backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID, nil) if err != nil { return nil, err } @@ -41,7 +39,7 @@ func GetTransactionResultByID(r *request.Request, backend access.API, link model return nil, models.NewBadRequestError(err) } - txr, err := backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID, execproto.EventEncodingVersion_JSON_CDC_V0) + txr, err := backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID, nil) if err != nil { return nil, err } diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index f40ec7d172b..f34448be161 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -7,6 +7,8 @@ import ( "fmt" "time" + "github.com/onflow/flow/protobuf/go/flow/entities" + execproto "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/rs/zerolog" "google.golang.org/grpc/codes" @@ -36,7 +38,7 @@ func (b *backendEvents) GetEventsForHeightRange( ctx context.Context, eventType string, startHeight, endHeight uint64, - eventEncodingVersion execproto.EventEncodingVersion, + eventEncodingVersionValue *entities.EventEncodingVersionValue, ) ([]flow.BlockEvents, error) { if endHeight < startHeight { @@ -78,7 +80,7 @@ func (b *backendEvents) GetEventsForHeightRange( blockHeaders = append(blockHeaders, header) } - return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, eventEncodingVersion) + return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, eventEncodingVersionValue) } // GetEventsForBlockIDs retrieves events for all the specified block IDs that have the given type @@ -86,7 +88,7 @@ func (b *backendEvents) GetEventsForBlockIDs( ctx context.Context, eventType string, blockIDs []flow.Identifier, - eventEncodingVersion execproto.EventEncodingVersion, + eventEncodingVersionValue *entities.EventEncodingVersionValue, ) ([]flow.BlockEvents, error) { if uint(len(blockIDs)) > b.maxHeightRange { @@ -105,14 +107,14 @@ func (b *backendEvents) GetEventsForBlockIDs( } // forward the request to the execution node - return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, eventEncodingVersion) + return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, eventEncodingVersionValue) } func (b *backendEvents) getBlockEventsFromExecutionNode( ctx context.Context, blockHeaders []*flow.Header, eventType string, - eventEncodingVersion execproto.EventEncodingVersion, + eventEncodingVersionValue *entities.EventEncodingVersionValue, ) ([]flow.BlockEvents, error) { // create an execution API request for events at block ID @@ -151,6 +153,11 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( Str("last_block_id", lastBlockID.String()). Msg("successfully got events") + eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + if requestEEV := eventEncodingVersionValue; requestEEV != nil { + eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + } + // convert execution node api result to access node api result results, err := verifyAndConvertToAccessEvents(resp.GetResults(), blockHeaders, eventEncodingVersion) if err != nil { diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index c0f99e4af14..97ead1534a6 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -510,7 +510,7 @@ func (suite *Suite) TestGetTransactionResultByIndex() { Return(exeEventResp, nil). Once() - result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, nil) suite.checkResponse(result, err) suite.Assert().Equal(result.BlockHeight, block.Header.Height) @@ -560,7 +560,7 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { Return(exeEventResp, nil). Once() - result, err := backend.GetTransactionResultsByBlockID(ctx, blockId, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err := backend.GetTransactionResultsByBlockID(ctx, blockId, nil) suite.checkResponse(result, err) suite.assertAllExpectations() @@ -639,7 +639,7 @@ func (suite *Suite) TestTransactionStatusTransition() { Times(len(fixedENIDs)) // should call each EN once // first call - when block under test is greater height than the sealed head, but execution node does not know about Tx - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) // status should be finalized since the sealed Blocks is smaller in height @@ -654,7 +654,7 @@ func (suite *Suite) TestTransactionStatusTransition() { Return(exeEventResp, nil) // second call - when block under test's height is greater height than the sealed head - result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) // status should be executed since no `NotFound` error in the `GetTransactionResult` call @@ -664,7 +664,7 @@ func (suite *Suite) TestTransactionStatusTransition() { headBlock.Header.Height = block.Header.Height + 1 // third call - when block under test's height is less than sealed head's height - result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) // status should be sealed since the sealed Blocks is greater in height @@ -675,7 +675,7 @@ func (suite *Suite) TestTransactionStatusTransition() { // fourth call - when block under test's height so much less than the head's height that it's considered expired, // but since there is a execution result, means it should retain it's sealed status - result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) // status should be expired since @@ -738,7 +738,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // should return pending status when we have not observed an expiry block suite.Run("pending", func() { // referenced block isn't known yet, so should return pending status - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) @@ -754,7 +754,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // we have NOT observed all intermediary Collections fullHeight = block.Header.Height + flow.DefaultTransactionExpiry/2 - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) }) @@ -764,7 +764,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // we have observed all intermediary Collections fullHeight = block.Header.Height + flow.DefaultTransactionExpiry + 1 - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) }) @@ -779,7 +779,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // we have observed all intermediary Collections fullHeight = block.Header.Height + flow.DefaultTransactionExpiry + 1 - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusExpired, result.Status) }) @@ -892,7 +892,7 @@ func (suite *Suite) TestTransactionPendingToFinalizedStatusTransition() { // should return pending status when we have not observed collection for the transaction suite.Run("pending", func() { currentState = flow.TransactionStatusPending - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) // assert that no call to an execution node is made @@ -903,7 +903,7 @@ func (suite *Suite) TestTransactionPendingToFinalizedStatusTransition() { // preceding sealed refBlock) suite.Run("finalized", func() { currentState = flow.TransactionStatusFinalized - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusFinalized, result.Status) }) @@ -929,7 +929,7 @@ func (suite *Suite) TestTransactionResultUnknown() { suite.Require().NoError(err) // first call - when block under test is greater height than the sealed head, but execution node does not know about Tx - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) // status should be reported as unknown @@ -1082,7 +1082,7 @@ func (suite *Suite) TestGetEventsForBlockIDs() { suite.Require().NoError(err) // execute request - actual, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), blockIDs, execproto.EventEncodingVersion_JSON_CDC_V0) + actual, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), blockIDs, nil) suite.checkResponse(actual, err) suite.Require().Equal(expected, actual) @@ -1100,7 +1100,7 @@ func (suite *Suite) TestGetEventsForBlockIDs() { suite.Require().NoError(err) // execute request with an empty block id list and expect an empty list of events and no error - resp, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), []flow.Identifier{}, execproto.EventEncodingVersion_JSON_CDC_V0) + resp, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), []flow.Identifier{}, nil) require.NoError(suite.T(), err) require.Empty(suite.T(), resp) }) @@ -1354,7 +1354,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), maxHeight, minHeight, execproto.EventEncodingVersion_JSON_CDC_V0) + _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), maxHeight, minHeight, nil) suite.Require().Error(err) suite.assertAllExpectations() // assert that request was not sent to execution node @@ -1384,7 +1384,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { suite.Require().NoError(err) // execute request - actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, execproto.EventEncodingVersion_JSON_CDC_V0) + actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, nil) // check response suite.checkResponse(actualResp, err) @@ -1412,7 +1412,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, execproto.EventEncodingVersion_JSON_CDC_V0) + actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, nil) suite.checkResponse(actualResp, err) suite.assertAllExpectations() @@ -1434,7 +1434,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, minHeight+1, execproto.EventEncodingVersion_JSON_CDC_V0) + _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, minHeight+1, nil) suite.Require().Error(err) }) @@ -1456,7 +1456,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, execproto.EventEncodingVersion_JSON_CDC_V0) + _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, nil) suite.Require().Error(err) }) diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index c6f104e839b..e1c13bc29ea 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -228,7 +228,7 @@ func (b *backendTransactions) GetTransactionResult( txID flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, - eventEncodingVersion execproto.EventEncodingVersion, + eventEncodingVersionValue *entities.EventEncodingVersionValue, ) (*access.TransactionResult, error) { // look up transaction from storage start := time.Now() @@ -287,7 +287,7 @@ func (b *backendTransactions) GetTransactionResult( // access node may not have the block if it hasn't yet been finalized, hence block can be nil at this point if block != nil { foundBlockID := block.ID() - transactionWasExecuted, events, statusCode, txError, err = b.lookupTransactionResult(ctx, txID, foundBlockID, eventEncodingVersion) + transactionWasExecuted, events, statusCode, txError, err = b.lookupTransactionResult(ctx, txID, foundBlockID, eventEncodingVersionValue) if err != nil { return nil, rpc.ConvertError(err, "failed to retrieve result from any execution node", codes.Internal) } @@ -386,7 +386,7 @@ func (b *backendTransactions) retrieveBlock( func (b *backendTransactions) GetTransactionResultsByBlockID( ctx context.Context, blockID flow.Identifier, - eventEncodingVersion execproto.EventEncodingVersion, + eventEncodingVersionValue *entities.EventEncodingVersionValue, ) ([]*access.TransactionResult, error) { // TODO: consider using storage.Index.ByBlockID, the index contains collection id and seals ID block, err := b.blocks.ByID(blockID) @@ -418,6 +418,11 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( "number of transaction results returned by execution node is less than the number of transactions in the block", ) + eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + if requestEEV := eventEncodingVersionValue; requestEEV != nil { + eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + } + for _, guarantee := range block.Payload.Guarantees { collection, err := b.collections.LightByID(guarantee.CollectionID) if err != nil { @@ -492,7 +497,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( return nil, rpc.ConvertStorageError(err) } - events, err := convert.MessagesToEventsFromVersion(systemTxResult.GetEvents(), resp.GetEventEncodingVersion()) + events, err := convert.MessagesToEventsFromVersion(systemTxResult.GetEvents(), eventEncodingVersion) if err != nil { return nil, rpc.ConvertError(err, "failed to convert events from system tx result", codes.Internal) } @@ -517,7 +522,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( ctx context.Context, blockID flow.Identifier, index uint32, - eventEncodingVersion execproto.EventEncodingVersion, + eventEncodingVersionValue *entities.EventEncodingVersionValue, ) (*access.TransactionResult, error) { // TODO: https://github.com/onflow/flow-go/issues/2175 so caching doesn't cause a circular dependency block, err := b.blocks.ByID(blockID) @@ -550,6 +555,11 @@ func (b *backendTransactions) GetTransactionResultByIndex( return nil, rpc.ConvertStorageError(err) } + eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + if requestEEV := eventEncodingVersionValue; requestEEV != nil { + eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + } + events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) if err != nil { return nil, status.Errorf(codes.Internal, "failed to convert events in blockID %x: %v", blockID, err) @@ -667,10 +677,9 @@ func (b *backendTransactions) lookupTransactionResult( ctx context.Context, txID flow.Identifier, blockID flow.Identifier, - eventEncodingVersion execproto.EventEncodingVersion, + eventEncodingVersionValue *entities.EventEncodingVersionValue, ) (bool, []flow.Event, uint32, string, error) { - - events, txStatus, message, err := b.getTransactionResultFromExecutionNode(ctx, blockID, txID[:], eventEncodingVersion) + events, txStatus, message, err := b.getTransactionResultFromExecutionNode(ctx, blockID, txID[:], eventEncodingVersionValue) if err != nil { // if either the execution node reported no results or the execution node could not be chosen if status.Code(err) == codes.NotFound { @@ -753,7 +762,7 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( ctx context.Context, blockID flow.Identifier, transactionID []byte, - eventEncodingVersion execproto.EventEncodingVersion, + eventEncodingVersionValue *entities.EventEncodingVersionValue, ) ([]flow.Event, uint32, string, error) { // create an execution API request for events at blockID and transactionID @@ -776,6 +785,11 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return nil, 0, "", err } + eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + if requestEEV := eventEncodingVersionValue; requestEEV != nil { + eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + } + events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) if err != nil { return nil, 0, "", rpc.ConvertError(err, "failed to convert events to message", codes.Internal) diff --git a/engine/access/rpc/backend/backend_transactions_test.go b/engine/access/rpc/backend/backend_transactions_test.go index 3761ba272d5..4d2f0f2d10a 100644 --- a/engine/access/rpc/backend/backend_transactions_test.go +++ b/engine/access/rpc/backend/backend_transactions_test.go @@ -7,7 +7,6 @@ import ( "github.com/dgraph-io/badger/v2" "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" - execproto "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" @@ -75,7 +74,7 @@ func (suite *Suite) TestGetTransactionResultReturnsUnknown() { backend, err := New(params) suite.Require().NoError(err) - res, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) + res, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), nil) suite.Require().NoError(err) suite.Require().Equal(res.Status, flow.TransactionStatusUnknown) }) @@ -108,7 +107,7 @@ func (suite *Suite) TestGetTransactionResultReturnsTransactionError() { backend, err := New(params) suite.Require().NoError(err) - _, err = backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) + _, err = backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), nil) suite.Require().Equal(err, status.Errorf(codes.Internal, "failed to find: %v", fmt.Errorf("some other error"))) }) @@ -146,7 +145,7 @@ func (suite *Suite) TestGetTransactionResultReturnsValidTransactionResultFromHis backend, err := New(params) suite.Require().NoError(err) - resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) + resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), nil) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusExecuted, resp.Status) suite.Require().Equal(uint(flow.TransactionStatusExecuted), resp.StatusCode) @@ -192,12 +191,12 @@ func (suite *Suite) TestGetTransactionResultFromCache() { coll := flow.CollectionFromTransactions([]*flow.Transaction{tx}) - resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) + resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), nil) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusExecuted, resp.Status) suite.Require().Equal(uint(flow.TransactionStatusExecuted), resp.StatusCode) - resp2, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) + resp2, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), nil) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusExecuted, resp2.Status) suite.Require().Equal(uint(flow.TransactionStatusExecuted), resp2.StatusCode) @@ -224,7 +223,7 @@ func (suite *Suite) TestGetTransactionResultCacheNonExistent() { coll := flow.CollectionFromTransactions([]*flow.Transaction{tx}) - resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) + resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), nil) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusUnknown, resp.Status) suite.Require().Equal(uint(flow.TransactionStatusUnknown), resp.StatusCode) @@ -259,7 +258,7 @@ func (suite *Suite) TestGetTransactionResultUnknownFromCache() { coll := flow.CollectionFromTransactions([]*flow.Transaction{tx}) - resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) + resp, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), nil) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusUnknown, resp.Status) suite.Require().Equal(uint(flow.TransactionStatusUnknown), resp.StatusCode) @@ -273,7 +272,7 @@ func (suite *Suite) TestGetTransactionResultUnknownFromCache() { StatusCode: uint(txStatus), }) - resp2, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), execproto.EventEncodingVersion_JSON_CDC_V0) + resp2, err := backend.GetTransactionResult(context.Background(), tx.ID(), block.ID(), coll.ID(), nil) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusUnknown, resp2.Status) suite.Require().Equal(uint(flow.TransactionStatusUnknown), resp2.StatusCode) diff --git a/engine/access/rpc/backend/historical_access_test.go b/engine/access/rpc/backend/historical_access_test.go index 78682c8758b..e48daac2161 100644 --- a/engine/access/rpc/backend/historical_access_test.go +++ b/engine/access/rpc/backend/historical_access_test.go @@ -5,7 +5,6 @@ import ( accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" - execproto "github.com/onflow/flow/protobuf/go/flow/execution" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -49,7 +48,7 @@ func (suite *Suite) TestHistoricalTransactionResult() { Once() // Make the call for the transaction result - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execproto.EventEncodingVersion_JSON_CDC_V0) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) // status should be sealed diff --git a/engine/access/rpc/backend/retry_test.go b/engine/access/rpc/backend/retry_test.go index 7936cdfacc7..6c9c850b79c 100644 --- a/engine/access/rpc/backend/retry_test.go +++ b/engine/access/rpc/backend/retry_test.go @@ -130,7 +130,7 @@ func (suite *Suite) TestSuccessfulTransactionsDontRetry() { Times(len(enIDs)) // should call each EN once // first call - when block under test is greater height than the sealed head, but execution node does not know about Tx - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, execution.EventEncodingVersion_JSON_CDC_V0) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, nil) suite.checkResponse(result, err) // status should be finalized since the sealed Blocks is smaller in height From 01208ea95be9b29110aedd3ce16917833c9c2cbf Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Wed, 4 Oct 2023 13:01:16 +0300 Subject: [PATCH 03/27] Fixed package for event version enum --- engine/access/rpc/backend/backend_events.go | 6 +++--- engine/access/rpc/backend/backend_transactions.go | 13 ++++++------- engine/common/rpc/convert/events.go | 9 ++++----- engine/common/rpc/convert/events_test.go | 11 ++++++----- engine/execution/rpc/engine.go | 10 ++++++---- go.mod | 2 +- go.sum | 4 ++-- integration/go.mod | 2 +- integration/go.sum | 4 ++-- 9 files changed, 31 insertions(+), 30 deletions(-) diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index f34448be161..17e2de7f93e 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -153,9 +153,9 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( Str("last_block_id", lastBlockID.String()). Msg("successfully got events") - eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 if requestEEV := eventEncodingVersionValue; requestEEV != nil { - eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + eventEncodingVersion = requestEEV.GetValue() } // convert execution node api result to access node api result @@ -172,7 +172,7 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( func verifyAndConvertToAccessEvents( execEvents []*execproto.GetEventsForBlockIDsResponse_Result, requestedBlockHeaders []*flow.Header, - version execproto.EventEncodingVersion, + version entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { if len(execEvents) != len(requestedBlockHeaders) { return nil, errors.New("number of results does not match number of blocks requested") diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index e1c13bc29ea..2e5dd131b53 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -418,9 +418,9 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( "number of transaction results returned by execution node is less than the number of transactions in the block", ) - eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 if requestEEV := eventEncodingVersionValue; requestEEV != nil { - eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + eventEncodingVersion = requestEEV.GetValue() } for _, guarantee := range block.Payload.Guarantees { @@ -441,7 +441,6 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( if err != nil { return nil, rpc.ConvertStorageError(err) } - events, err := convert.MessagesToEventsFromVersion(txResult.GetEvents(), eventEncodingVersion) if err != nil { return nil, status.Errorf(codes.Internal, @@ -555,9 +554,9 @@ func (b *backendTransactions) GetTransactionResultByIndex( return nil, rpc.ConvertStorageError(err) } - eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 if requestEEV := eventEncodingVersionValue; requestEEV != nil { - eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + eventEncodingVersion = requestEEV.GetValue() } events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) @@ -785,9 +784,9 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return nil, 0, "", err } - eventEncodingVersion := execproto.EventEncodingVersion_JSON_CDC_V0 + eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 if requestEEV := eventEncodingVersionValue; requestEEV != nil { - eventEncodingVersion = execproto.EventEncodingVersion(requestEEV.GetValue()) + eventEncodingVersion = requestEEV.GetValue() } events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index 58ccb0ed9a1..fc6f335f9f4 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -13,7 +13,6 @@ import ( accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" - execproto "github.com/onflow/flow/protobuf/go/flow/execution" ) // EventToMessage converts a flow.Event to a protobuf message @@ -63,17 +62,17 @@ func MessagesToEvents(l []*entities.Event) []flow.Event { // MessageToEventFromVersion converts a protobuf message to a flow.Event, and converts the payload // encoding from CCF to JSON if the input version is CCF -func MessageToEventFromVersion(m *entities.Event, inputVersion execproto.EventEncodingVersion) (*flow.Event, error) { +func MessageToEventFromVersion(m *entities.Event, inputVersion entities.EventEncodingVersion) (*flow.Event, error) { event := MessageToEvent(m) switch inputVersion { - case execproto.EventEncodingVersion_CCF_V0: + case entities.EventEncodingVersion_CCF_V0: convertedPayload, err := CcfPayloadToJsonPayload(event.Payload) if err != nil { return nil, fmt.Errorf("could not convert event payload from CCF to Json: %w", err) } event.Payload = convertedPayload return &event, nil - case execproto.EventEncodingVersion_JSON_CDC_V0: + case entities.EventEncodingVersion_JSON_CDC_V0: return &event, nil default: return nil, fmt.Errorf("invalid encoding format %d", inputVersion) @@ -82,7 +81,7 @@ func MessageToEventFromVersion(m *entities.Event, inputVersion execproto.EventEn // MessagesToEventsFromVersion converts a slice of protobuf messages to a slice of flow.Events, converting // the payload encoding from CCF to JSON if the input version is CCF -func MessagesToEventsFromVersion(l []*entities.Event, version execproto.EventEncodingVersion) ([]flow.Event, error) { +func MessagesToEventsFromVersion(l []*entities.Event, version entities.EventEncodingVersion) ([]flow.Event, error) { events := make([]flow.Event, len(l)) for i, m := range l { event, err := MessageToEventFromVersion(m, version) diff --git a/engine/common/rpc/convert/events_test.go b/engine/common/rpc/convert/events_test.go index 879db710f8b..70828a5643d 100644 --- a/engine/common/rpc/convert/events_test.go +++ b/engine/common/rpc/convert/events_test.go @@ -3,13 +3,14 @@ package convert_test import ( "testing" + "github.com/onflow/flow/protobuf/go/flow/entities" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/onflow/cadence" "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" - execproto "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" @@ -80,7 +81,7 @@ func TestConvertEventWithPayloadConversion(t *testing.T) { t.Run("convert payload from ccf to jsoncdc", func(t *testing.T) { message := convert.EventToMessage(ccfEvent) - convertedEvent, err := convert.MessageToEventFromVersion(message, execproto.EventEncodingVersion_CCF_V0) + convertedEvent, err := convert.MessageToEventFromVersion(message, entities.EventEncodingVersion_CCF_V0) assert.NoError(t, err) assert.Equal(t, jsonEvent, *convertedEvent) @@ -88,7 +89,7 @@ func TestConvertEventWithPayloadConversion(t *testing.T) { t.Run("convert payload from jsoncdc to jsoncdc", func(t *testing.T) { message := convert.EventToMessage(jsonEvent) - convertedEvent, err := convert.MessageToEventFromVersion(message, execproto.EventEncodingVersion_JSON_CDC_V0) + convertedEvent, err := convert.MessageToEventFromVersion(message, entities.EventEncodingVersion_JSON_CDC_V0) assert.NoError(t, err) assert.Equal(t, jsonEvent, *convertedEvent) @@ -149,7 +150,7 @@ func TestConvertEvents(t *testing.T) { t.Run("convert event from ccf to jsoncdc", func(t *testing.T) { messages := convert.EventsToMessages(ccfEvents) - converted, err := convert.MessagesToEventsFromVersion(messages, execproto.EventEncodingVersion_CCF_V0) + converted, err := convert.MessagesToEventsFromVersion(messages, entities.EventEncodingVersion_CCF_V0) assert.NoError(t, err) assert.Equal(t, jsonEvents, converted) @@ -157,7 +158,7 @@ func TestConvertEvents(t *testing.T) { t.Run("convert event from jsoncdc", func(t *testing.T) { messages := convert.EventsToMessages(jsonEvents) - converted, err := convert.MessagesToEventsFromVersion(messages, execproto.EventEncodingVersion_JSON_CDC_V0) + converted, err := convert.MessagesToEventsFromVersion(messages, entities.EventEncodingVersion_JSON_CDC_V0) assert.NoError(t, err) assert.Equal(t, jsonEvents, converted) diff --git a/engine/execution/rpc/engine.go b/engine/execution/rpc/engine.go index 5a6a5b98892..714446ca04f 100644 --- a/engine/execution/rpc/engine.go +++ b/engine/execution/rpc/engine.go @@ -9,6 +9,8 @@ import ( "strings" "unicode/utf8" + "github.com/onflow/flow/protobuf/go/flow/entities" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/rs/zerolog" @@ -278,7 +280,7 @@ func (h *handler) GetEventsForBlockIDs( return &execution.GetEventsForBlockIDsResponse{ Results: results, - EventEncodingVersion: execution.EventEncodingVersion_CCF_V0, + EventEncodingVersion: entities.EventEncodingVersion_CCF_V0, }, nil } @@ -341,7 +343,7 @@ func (h *handler) GetTransactionResult( StatusCode: statusCode, ErrorMessage: errMsg, Events: events, - EventEncodingVersion: execution.EventEncodingVersion_CCF_V0, + EventEncodingVersion: entities.EventEncodingVersion_CCF_V0, }, nil } @@ -400,7 +402,7 @@ func (h *handler) GetTransactionResultByIndex( StatusCode: statusCode, ErrorMessage: errMsg, Events: events, - EventEncodingVersion: execution.EventEncodingVersion_CCF_V0, + EventEncodingVersion: entities.EventEncodingVersion_CCF_V0, }, nil } @@ -486,7 +488,7 @@ func (h *handler) GetTransactionResultsByBlockID( // compose a response return &execution.GetTransactionResultsResponse{ TransactionResults: responseTxResults, - EventEncodingVersion: execution.EventEncodingVersion_CCF_V0, + EventEncodingVersion: entities.EventEncodingVersion_CCF_V0, }, nil } diff --git a/go.mod b/go.mod index 48066538584..db4730b056c 100644 --- a/go.mod +++ b/go.mod @@ -295,4 +295,4 @@ require ( nhooyr.io/websocket v1.8.7 // indirect ) -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231003102535-25fed8fe4f6a +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 diff --git a/go.sum b/go.sum index 639e16b7a50..3648f2d7dd4 100644 --- a/go.sum +++ b/go.sum @@ -98,8 +98,8 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231003102535-25fed8fe4f6a h1:HsCZqXvGHxJ1GZPgf2anx87MHoGUMe4BP2MmLpSENdc= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231003102535-25fed8fe4f6a/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 h1:k/OJbM33BwI9sEJvNMzmkhM0ZJTSZeCoLImkBLldsJ8= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= diff --git a/integration/go.mod b/integration/go.mod index 129b62e709a..d82af62a2ac 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -342,4 +342,4 @@ replace github.com/onflow/flow-go => ../ replace github.com/onflow/flow-go/insecure => ../insecure -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231003102535-25fed8fe4f6a +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 diff --git a/integration/go.sum b/integration/go.sum index 0f4d3c79c65..3022170ce5c 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -100,8 +100,8 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231003102535-25fed8fe4f6a h1:HsCZqXvGHxJ1GZPgf2anx87MHoGUMe4BP2MmLpSENdc= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231003102535-25fed8fe4f6a/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 h1:k/OJbM33BwI9sEJvNMzmkhM0ZJTSZeCoLImkBLldsJ8= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= From 7e4061e6cde8de0c2cab4c5f8f155761a1ad7265 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Thu, 5 Oct 2023 13:50:26 +0300 Subject: [PATCH 04/27] Added tests --- engine/access/rpc/backend/backend_test.go | 171 +++++++++++++++++++++- 1 file changed, 170 insertions(+), 1 deletion(-) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 97ead1534a6..8f91a3c4b3e 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -6,6 +6,13 @@ import ( "strconv" "testing" + "github.com/onflow/cadence" + "github.com/onflow/cadence/encoding/ccf" + jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/utils/unittest/generator" + "github.com/dgraph-io/badger/v2" accessproto "github.com/onflow/flow/protobuf/go/flow/access" entitiesproto "github.com/onflow/flow/protobuf/go/flow/entities" @@ -1015,7 +1022,7 @@ func (suite *Suite) TestGetEventsForBlockIDs() { blockHeaders := setupStorage(5) suite.snapshot.On("Identities", mock.Anything).Return(validExecutorIdentities, nil) - validENIDs := flow.IdentifierList(validExecutorIdentities.NodeIDs()) + validENIDs := validExecutorIdentities.NodeIDs() // create a mock connection factory connFactory := connectionmock.NewConnectionFactory(suite.T()) @@ -2102,6 +2109,108 @@ func (suite *Suite) TestScriptExecutionValidationMode() { }) } +// TestGetTransactionResultByIndexEventEncodingVersion +func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { + suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() + + ctx := context.Background() + block := unittest.BlockFixture() + blockId := block.ID() + index := uint32(0) + + suite.snapshot.On("Head").Return(block.Header, nil) + + // block storage returns the corresponding block + suite.blocks. + On("ByID", blockId). + Return(&block, nil) + + _, fixedENIDs := suite.setupReceipts(&block) + suite.state.On("Final").Return(suite.snapshot, nil).Maybe() + suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) + + // create a mock connection factory + connFactory := connectionmock.NewConnectionFactory(suite.T()) + connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) + + params := suite.defaultBackendParams() + // the connection factory should be used to get the execution node client + params.ConnFactory = connFactory + params.FixedExecutionNodeIDs = (fixedENIDs.NodeIDs()).Strings() + + backend, err := New(params) + suite.Require().NoError(err) + + exeEventReq := &execproto.GetTransactionByIndexRequest{ + BlockId: blockId[:], + Index: index, + } + + prepareGetTransactionResultByIndex := func(version entitiesproto.EventEncodingVersion) *execproto.GetTransactionResultResponse { + events := getEventsWithEncoding(1, version) + + exeEventResp := &execproto.GetTransactionResultResponse{ + Events: convert.EventsToMessages(events), + } + + suite.execClient. + On("GetTransactionResultByIndex", ctx, exeEventReq). + Return(exeEventResp, nil). + Once() + + return exeEventResp + } + + assertResultExpectations := func( + exeEventResp *execproto.GetTransactionResultResponse, + encodingVersionValue *entitiesproto.EventEncodingVersionValue, + ) { + encodingVersion := entitiesproto.EventEncodingVersion_JSON_CDC_V0 + if encodingVersionValue != nil { + encodingVersion = encodingVersionValue.GetValue() + } + + result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, encodingVersionValue) + suite.checkResponse(result, err) + + expectedResultEvents, err := convert.MessagesToEventsFromVersion(exeEventResp.GetEvents(), encodingVersion) + suite.Require().NoError(err) + suite.Assert().Equal(result.Events, expectedResultEvents) + } + + suite.Run("test default(JSON) event encoding (happy case)", func() { + encodingVersion := entitiesproto.EventEncodingVersion_JSON_CDC_V0 + exeEventResp := prepareGetTransactionResultByIndex(encodingVersion) + assertResultExpectations(exeEventResp, nil) + }) + + suite.Run("test JSON event encoding (happy case)", func() { + encodingVersion := entitiesproto.EventEncodingVersion_JSON_CDC_V0 + exeEventResp := prepareGetTransactionResultByIndex(encodingVersion) + assertResultExpectations(exeEventResp, &entitiesproto.EventEncodingVersionValue{Value: encodingVersion}) + }) + + suite.Run("test CFF event encoding (happy case)", func() { + encodingVersion := entitiesproto.EventEncodingVersion_CCF_V0 + exeEventResp := prepareGetTransactionResultByIndex(encodingVersion) + assertResultExpectations(exeEventResp, &entitiesproto.EventEncodingVersionValue{Value: encodingVersion}) + }) + + suite.Run("test wrong event conversion JSON to CFF", func() { + encodingVersion := entitiesproto.EventEncodingVersion_JSON_CDC_V0 + _ = prepareGetTransactionResultByIndex(encodingVersion) + + result, err := backend.GetTransactionResultByIndex( + ctx, + blockId, + index, + &entitiesproto.EventEncodingVersionValue{Value: entitiesproto.EventEncodingVersion_CCF_V0}, + ) + suite.Require().Error(err) + suite.Require().Nil(result) + }) +} + func (suite *Suite) assertAllExpectations() { suite.snapshot.AssertExpectations(suite.T()) suite.state.AssertExpectations(suite.T()) @@ -2148,6 +2257,66 @@ func getEvents(n int) []flow.Event { return events } +func getEventsWithEncoding(n int, version entitiesproto.EventEncodingVersion) []flow.Event { + events := make([]flow.Event, 0, n) + ids := generator.IdentifierGenerator() + for i := 0; i < n; i++ { + location := common.StringLocation("test") + identifier := fmt.Sprintf("FooEvent%d", i) + typeID := location.TypeID(nil, identifier) + + testEventType := &cadence.EventType{ + Location: location, + QualifiedIdentifier: identifier, + Fields: []cadence.Field{ + { + Identifier: "a", + Type: cadence.IntType{}, + }, + { + Identifier: "b", + Type: cadence.StringType{}, + }, + }, + } + + fooString, err := cadence.NewString("foo") + if err != nil { + panic(fmt.Sprintf("unexpected error while creating cadence string: %s", err)) + } + + testEvent := cadence.NewEvent( + []cadence.Value{ + cadence.NewInt(i), + fooString, + }).WithType(testEventType) + + var payload []byte + switch version { + case entitiesproto.EventEncodingVersion_CCF_V0: + payload, err = ccf.Encode(testEvent) + if err != nil { + panic(fmt.Sprintf("unexpected error while ccf encoding events: %s", err)) + } + case entitiesproto.EventEncodingVersion_JSON_CDC_V0: + payload, err = jsoncdc.Encode(testEvent) + if err != nil { + panic(fmt.Sprintf("unexpected error while json encoding events: %s", err)) + } + } + + events = append(events, flow.Event{ + Type: flow.EventType(typeID), + TransactionID: ids.New(), + TransactionIndex: uint32(i), + EventIndex: uint32(i), + Payload: payload, + }) + } + + return events +} + func (suite *Suite) defaultBackendParams() Params { return Params{ State: suite.state, From f5a6ae91a780a4123aec9255de1a571b56b13cdf Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Fri, 6 Oct 2023 11:53:15 +0300 Subject: [PATCH 05/27] Extract reusable event encoding version function. --- engine/access/rpc/backend/backend_events.go | 13 +++++++++---- engine/access/rpc/backend/backend_transactions.go | 15 +++------------ 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index 17e2de7f93e..c210915e202 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -153,10 +153,7 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( Str("last_block_id", lastBlockID.String()). Msg("successfully got events") - eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 - if requestEEV := eventEncodingVersionValue; requestEEV != nil { - eventEncodingVersion = requestEEV.GetValue() - } + eventEncodingVersion := getEventEncodingVersion(eventEncodingVersionValue) // convert execution node api result to access node api result results, err := verifyAndConvertToAccessEvents(resp.GetResults(), blockHeaders, eventEncodingVersion) @@ -268,3 +265,11 @@ func (b *backendEvents) tryGetEvents(ctx context.Context, } return resp, nil } + +func getEventEncodingVersion(eventEncodingVersionValue *entities.EventEncodingVersionValue) entities.EventEncodingVersion { + eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 + if requestEEV := eventEncodingVersionValue; requestEEV != nil { + eventEncodingVersion = requestEEV.GetValue() + } + return eventEncodingVersion +} diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 2e5dd131b53..9c7bc24bb43 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -418,10 +418,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( "number of transaction results returned by execution node is less than the number of transactions in the block", ) - eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 - if requestEEV := eventEncodingVersionValue; requestEEV != nil { - eventEncodingVersion = requestEEV.GetValue() - } + eventEncodingVersion := getEventEncodingVersion(eventEncodingVersionValue) for _, guarantee := range block.Payload.Guarantees { collection, err := b.collections.LightByID(guarantee.CollectionID) @@ -554,10 +551,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( return nil, rpc.ConvertStorageError(err) } - eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 - if requestEEV := eventEncodingVersionValue; requestEEV != nil { - eventEncodingVersion = requestEEV.GetValue() - } + eventEncodingVersion := getEventEncodingVersion(eventEncodingVersionValue) events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) if err != nil { @@ -784,10 +778,7 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return nil, 0, "", err } - eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 - if requestEEV := eventEncodingVersionValue; requestEEV != nil { - eventEncodingVersion = requestEEV.GetValue() - } + eventEncodingVersion := getEventEncodingVersion(eventEncodingVersionValue) events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) if err != nil { From fc90fc27b4dc744a92ab9b49b8639726d916742c Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Mon, 9 Oct 2023 15:02:02 +0300 Subject: [PATCH 06/27] linted --- engine/access/state_stream/handler.go | 4 +- engine/common/rpc/convert/events.go | 8 ++-- .../mock/execution_state_indexer_metrics.go | 39 +++++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 module/mock/execution_state_indexer_metrics.go diff --git a/engine/access/state_stream/handler.go b/engine/access/state_stream/handler.go index c5b1d15a668..223d306903b 100644 --- a/engine/access/state_stream/handler.go +++ b/engine/access/state_stream/handler.go @@ -4,7 +4,7 @@ import ( "context" "sync/atomic" - "github.com/onflow/flow/protobuf/go/flow/execution" + "github.com/onflow/flow/protobuf/go/flow/entities" access "github.com/onflow/flow/protobuf/go/flow/executiondata" executiondata "github.com/onflow/flow/protobuf/go/flow/executiondata" "google.golang.org/grpc/codes" @@ -169,7 +169,7 @@ func (h *Handler) SubscribeEvents(request *access.SubscribeEventsRequest, stream // BlockExecutionData contains CCF encoded events, and the Access API returns JSON-CDC events. // convert event payload formats. // This is a temporary solution until the Access API supports specifying the encoding in the request - events, err := convert.EventsToMessagesFromVersion(resp.Events, execution.EventEncodingVersion_CCF_V0) + events, err := convert.EventsToMessagesFromVersion(resp.Events, entities.EventEncodingVersion_CCF_V0) if err != nil { return status.Errorf(codes.Internal, "could not convert events to entity: %v", err) } diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index e209c311bd7..18babd35943 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -62,16 +62,16 @@ func MessagesToEvents(l []*entities.Event) []flow.Event { // EventToMessageFromVersion converts a flow.Event to a protobuf message, converting the payload // encoding from CCF to JSON if the input version is CCF -func EventToMessageFromVersion(e flow.Event, version execproto.EventEncodingVersion) (*entities.Event, error) { +func EventToMessageFromVersion(e flow.Event, version entities.EventEncodingVersion) (*entities.Event, error) { message := EventToMessage(e) switch version { - case execproto.EventEncodingVersion_CCF_V0: + case entities.EventEncodingVersion_CCF_V0: convertedPayload, err := CcfPayloadToJsonPayload(e.Payload) if err != nil { return nil, fmt.Errorf("could not convert event payload from CCF to Json: %w", err) } message.Payload = convertedPayload - case execproto.EventEncodingVersion_JSON_CDC_V0: + case entities.EventEncodingVersion_JSON_CDC_V0: default: return nil, fmt.Errorf("invalid encoding format %d", version) } @@ -100,7 +100,7 @@ func MessageToEventFromVersion(m *entities.Event, inputVersion entities.EventEnc // EventsToMessagesFromVersion converts a slice of flow.Events to a slice of protobuf messages, converting // the payload encoding from CCF to JSON if the input version is CCF -func EventsToMessagesFromVersion(flowEvents []flow.Event, version execproto.EventEncodingVersion) ([]*entities.Event, error) { +func EventsToMessagesFromVersion(flowEvents []flow.Event, version entities.EventEncodingVersion) ([]*entities.Event, error) { events := make([]*entities.Event, len(flowEvents)) for i, e := range flowEvents { event, err := EventToMessageFromVersion(e, version) diff --git a/module/mock/execution_state_indexer_metrics.go b/module/mock/execution_state_indexer_metrics.go new file mode 100644 index 00000000000..4dbb0e3b2d9 --- /dev/null +++ b/module/mock/execution_state_indexer_metrics.go @@ -0,0 +1,39 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + mock "github.com/stretchr/testify/mock" + + time "time" +) + +// ExecutionStateIndexerMetrics is an autogenerated mock type for the ExecutionStateIndexerMetrics type +type ExecutionStateIndexerMetrics struct { + mock.Mock +} + +// BlockIndexed provides a mock function with given fields: height, duration, events, registers, transactionResults +func (_m *ExecutionStateIndexerMetrics) BlockIndexed(height uint64, duration time.Duration, events int, registers int, transactionResults int) { + _m.Called(height, duration, events, registers, transactionResults) +} + +// BlockReindexed provides a mock function with given fields: +func (_m *ExecutionStateIndexerMetrics) BlockReindexed() { + _m.Called() +} + +type mockConstructorTestingTNewExecutionStateIndexerMetrics interface { + mock.TestingT + Cleanup(func()) +} + +// NewExecutionStateIndexerMetrics creates a new instance of ExecutionStateIndexerMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewExecutionStateIndexerMetrics(t mockConstructorTestingTNewExecutionStateIndexerMetrics) *ExecutionStateIndexerMetrics { + mock := &ExecutionStateIndexerMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From 76d34360eeb6abe1ef050841c55d544795aef9d1 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Mon, 9 Oct 2023 16:33:45 +0300 Subject: [PATCH 07/27] Addded execution data api encoding event version. --- engine/access/state_stream/handler.go | 38 +++++++++++++++++---------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/engine/access/state_stream/handler.go b/engine/access/state_stream/handler.go index 223d306903b..5c8d422cd0b 100644 --- a/engine/access/state_stream/handler.go +++ b/engine/access/state_stream/handler.go @@ -52,11 +52,13 @@ func (h *Handler) GetExecutionDataByBlockID(ctx context.Context, request *access return nil, status.Errorf(codes.Internal, "could not convert execution data to entity: %v", err) } - // convert event payloads from CCF to JSON-CDC - // This is a temporary solution until the Access API supports specifying the encoding in the request - err = convert.BlockExecutionDataEventPayloadsToJson(message) - if err != nil { - return nil, status.Errorf(codes.Internal, "could not convert execution data event payloads to JSON: %v", err) + eventEncodingVersion := getEventEncodingVersion(request.GetEventEncodingVersion()) + if eventEncodingVersion == entities.EventEncodingVersion_JSON_CDC_V0 { + // convert event payloads from CCF to JSON-CDC + err = convert.BlockExecutionDataEventPayloadsToJson(message) + if err != nil { + return nil, status.Errorf(codes.Internal, "could not convert execution data event payloads to JSON: %v", err) + } } return &access.GetExecutionDataByBlockIDResponse{BlockExecutionData: message}, nil @@ -100,11 +102,13 @@ func (h *Handler) SubscribeExecutionData(request *access.SubscribeExecutionDataR return status.Errorf(codes.Internal, "could not convert execution data to entity: %v", err) } - // convert event payloads from CCF to JSON-CDC - // This is a temporary solution until the Access API supports specifying the encoding in the request - err = convert.BlockExecutionDataEventPayloadsToJson(execData) - if err != nil { - return status.Errorf(codes.Internal, "could not convert execution data event payloads to JSON: %v", err) + eventEncodingVersion := getEventEncodingVersion(request.GetEventEncodingVersion()) + if eventEncodingVersion == entities.EventEncodingVersion_JSON_CDC_V0 { + // convert event payloads from CCF to JSON-CDC + err = convert.BlockExecutionDataEventPayloadsToJson(execData) + if err != nil { + return status.Errorf(codes.Internal, "could not convert execution data event payloads to JSON: %v", err) + } } err = stream.Send(&executiondata.SubscribeExecutionDataResponse{ @@ -166,10 +170,8 @@ func (h *Handler) SubscribeEvents(request *access.SubscribeEventsRequest, stream return status.Errorf(codes.Internal, "unexpected response type: %T", v) } - // BlockExecutionData contains CCF encoded events, and the Access API returns JSON-CDC events. - // convert event payload formats. - // This is a temporary solution until the Access API supports specifying the encoding in the request - events, err := convert.EventsToMessagesFromVersion(resp.Events, entities.EventEncodingVersion_CCF_V0) + eventEncodingVersion := getEventEncodingVersion(request.GetEventEncodingVersion()) + events, err := convert.EventsToMessagesFromVersion(resp.Events, eventEncodingVersion) if err != nil { return status.Errorf(codes.Internal, "could not convert events to entity: %v", err) } @@ -184,3 +186,11 @@ func (h *Handler) SubscribeEvents(request *access.SubscribeEventsRequest, stream } } } + +func getEventEncodingVersion(eventEncodingVersionValue *entities.EventEncodingVersionValue) entities.EventEncodingVersion { + eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 + if requestEEV := eventEncodingVersionValue; requestEEV != nil { + eventEncodingVersion = requestEEV.GetValue() + } + return eventEncodingVersion +} From e392306ecc5984dd17bd4b698b89e2356f919a4a Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Mon, 9 Oct 2023 16:50:36 +0300 Subject: [PATCH 08/27] Fixed unit tesxt for AccessAPI --- engine/access/rpc/backend/backend_test.go | 72 +++++------------------ 1 file changed, 14 insertions(+), 58 deletions(-) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 8f91a3c4b3e..5fa5b6bbde7 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -6,11 +6,6 @@ import ( "strconv" "testing" - "github.com/onflow/cadence" - "github.com/onflow/cadence/encoding/ccf" - jsoncdc "github.com/onflow/cadence/encoding/json" - "github.com/onflow/cadence/runtime/common" - "github.com/onflow/flow-go/utils/unittest/generator" "github.com/dgraph-io/badger/v2" @@ -2109,7 +2104,8 @@ func (suite *Suite) TestScriptExecutionValidationMode() { }) } -// TestGetTransactionResultByIndexEventEncodingVersion +// TestGetTransactionResultByIndexEventEncodingVersion tests the GetTransactionResultByIndex function with different +// event encoding versions. func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() @@ -2146,6 +2142,7 @@ func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { Index: index, } + // Define a helper function to prepare the GetTransactionResultResponse with a given encoding version. prepareGetTransactionResultByIndex := func(version entitiesproto.EventEncodingVersion) *execproto.GetTransactionResultResponse { events := getEventsWithEncoding(1, version) @@ -2161,6 +2158,7 @@ func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { return exeEventResp } + // Define a helper function to assert the result expectations. assertResultExpectations := func( exeEventResp *execproto.GetTransactionResultResponse, encodingVersionValue *entitiesproto.EventEncodingVersionValue, @@ -2257,61 +2255,19 @@ func getEvents(n int) []flow.Event { return events } +// getEventsWithEncoding generates a specified number of events with a given encoding version. func getEventsWithEncoding(n int, version entitiesproto.EventEncodingVersion) []flow.Event { + var eventGenerator *generator.Events + switch version { + case entitiesproto.EventEncodingVersion_CCF_V0: + eventGenerator = generator.EventGenerator(generator.WithEncoding(generator.EncodingCCF)) + case entitiesproto.EventEncodingVersion_JSON_CDC_V0: + eventGenerator = generator.EventGenerator(generator.WithEncoding(generator.EncodingJSON)) + } + events := make([]flow.Event, 0, n) - ids := generator.IdentifierGenerator() for i := 0; i < n; i++ { - location := common.StringLocation("test") - identifier := fmt.Sprintf("FooEvent%d", i) - typeID := location.TypeID(nil, identifier) - - testEventType := &cadence.EventType{ - Location: location, - QualifiedIdentifier: identifier, - Fields: []cadence.Field{ - { - Identifier: "a", - Type: cadence.IntType{}, - }, - { - Identifier: "b", - Type: cadence.StringType{}, - }, - }, - } - - fooString, err := cadence.NewString("foo") - if err != nil { - panic(fmt.Sprintf("unexpected error while creating cadence string: %s", err)) - } - - testEvent := cadence.NewEvent( - []cadence.Value{ - cadence.NewInt(i), - fooString, - }).WithType(testEventType) - - var payload []byte - switch version { - case entitiesproto.EventEncodingVersion_CCF_V0: - payload, err = ccf.Encode(testEvent) - if err != nil { - panic(fmt.Sprintf("unexpected error while ccf encoding events: %s", err)) - } - case entitiesproto.EventEncodingVersion_JSON_CDC_V0: - payload, err = jsoncdc.Encode(testEvent) - if err != nil { - panic(fmt.Sprintf("unexpected error while json encoding events: %s", err)) - } - } - - events = append(events, flow.Event{ - Type: flow.EventType(typeID), - TransactionID: ids.New(), - TransactionIndex: uint32(i), - EventIndex: uint32(i), - Payload: payload, - }) + events = append(events, eventGenerator.New()) } return events From a2fc40dd5760178d75b1ee0df62d48ea3d6f04eb Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Tue, 10 Oct 2023 13:34:25 +0300 Subject: [PATCH 09/27] Moved dunction to converters. Fixed tests. --- access/handler.go | 1 - engine/access/rpc/backend/backend_events.go | 10 +------- .../rpc/backend/backend_transactions.go | 6 ++--- engine/access/state_stream/handler.go | 25 ++++++++----------- engine/access/state_stream/handler_test.go | 11 +++----- engine/common/rpc/convert/events.go | 8 ++++++ 6 files changed, 26 insertions(+), 35 deletions(-) diff --git a/access/handler.go b/access/handler.go index 05773c2202d..2d3e3c1ef7d 100644 --- a/access/handler.go +++ b/access/handler.go @@ -275,7 +275,6 @@ func (h *Handler) GetTransactionResult( } eventEncodingVersion := req.GetEventEncodingVersion() - result, err := h.api.GetTransactionResult(ctx, transactionID, blockId, collectionId, eventEncodingVersion) if err != nil { return nil, err diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index c210915e202..27171705c3d 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -153,7 +153,7 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( Str("last_block_id", lastBlockID.String()). Msg("successfully got events") - eventEncodingVersion := getEventEncodingVersion(eventEncodingVersionValue) + eventEncodingVersion := convert.GetEventEncodingVersion(eventEncodingVersionValue) // convert execution node api result to access node api result results, err := verifyAndConvertToAccessEvents(resp.GetResults(), blockHeaders, eventEncodingVersion) @@ -265,11 +265,3 @@ func (b *backendEvents) tryGetEvents(ctx context.Context, } return resp, nil } - -func getEventEncodingVersion(eventEncodingVersionValue *entities.EventEncodingVersionValue) entities.EventEncodingVersion { - eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 - if requestEEV := eventEncodingVersionValue; requestEEV != nil { - eventEncodingVersion = requestEEV.GetValue() - } - return eventEncodingVersion -} diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 9c7bc24bb43..70b13badf5f 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -418,7 +418,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( "number of transaction results returned by execution node is less than the number of transactions in the block", ) - eventEncodingVersion := getEventEncodingVersion(eventEncodingVersionValue) + eventEncodingVersion := convert.GetEventEncodingVersion(eventEncodingVersionValue) for _, guarantee := range block.Payload.Guarantees { collection, err := b.collections.LightByID(guarantee.CollectionID) @@ -551,7 +551,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( return nil, rpc.ConvertStorageError(err) } - eventEncodingVersion := getEventEncodingVersion(eventEncodingVersionValue) + eventEncodingVersion := convert.GetEventEncodingVersion(eventEncodingVersionValue) events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) if err != nil { @@ -778,7 +778,7 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return nil, 0, "", err } - eventEncodingVersion := getEventEncodingVersion(eventEncodingVersionValue) + eventEncodingVersion := convert.GetEventEncodingVersion(eventEncodingVersionValue) events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) if err != nil { diff --git a/engine/access/state_stream/handler.go b/engine/access/state_stream/handler.go index 5c8d422cd0b..7ae8b9ccbcf 100644 --- a/engine/access/state_stream/handler.go +++ b/engine/access/state_stream/handler.go @@ -52,7 +52,7 @@ func (h *Handler) GetExecutionDataByBlockID(ctx context.Context, request *access return nil, status.Errorf(codes.Internal, "could not convert execution data to entity: %v", err) } - eventEncodingVersion := getEventEncodingVersion(request.GetEventEncodingVersion()) + eventEncodingVersion := convert.GetEventEncodingVersion(request.GetEventEncodingVersion()) if eventEncodingVersion == entities.EventEncodingVersion_JSON_CDC_V0 { // convert event payloads from CCF to JSON-CDC err = convert.BlockExecutionDataEventPayloadsToJson(message) @@ -102,7 +102,7 @@ func (h *Handler) SubscribeExecutionData(request *access.SubscribeExecutionDataR return status.Errorf(codes.Internal, "could not convert execution data to entity: %v", err) } - eventEncodingVersion := getEventEncodingVersion(request.GetEventEncodingVersion()) + eventEncodingVersion := convert.GetEventEncodingVersion(request.GetEventEncodingVersion()) if eventEncodingVersion == entities.EventEncodingVersion_JSON_CDC_V0 { // convert event payloads from CCF to JSON-CDC err = convert.BlockExecutionDataEventPayloadsToJson(execData) @@ -170,12 +170,15 @@ func (h *Handler) SubscribeEvents(request *access.SubscribeEventsRequest, stream return status.Errorf(codes.Internal, "unexpected response type: %T", v) } - eventEncodingVersion := getEventEncodingVersion(request.GetEventEncodingVersion()) - events, err := convert.EventsToMessagesFromVersion(resp.Events, eventEncodingVersion) - if err != nil { - return status.Errorf(codes.Internal, "could not convert events to entity: %v", err) + eventEncodingVersion := convert.GetEventEncodingVersion(request.GetEventEncodingVersion()) + var events []*entities.Event + var err error + if eventEncodingVersion == entities.EventEncodingVersion_JSON_CDC_V0 { + events, err = convert.EventsToMessagesFromVersion(resp.Events, entities.EventEncodingVersion_CCF_V0) + if err != nil { + return status.Errorf(codes.Internal, "could not convert events to entity: %v", err) + } } - err = stream.Send(&executiondata.SubscribeEventsResponse{ BlockHeight: resp.Height, BlockId: convert.IdentifierToMessage(resp.BlockID), @@ -186,11 +189,3 @@ func (h *Handler) SubscribeEvents(request *access.SubscribeEventsRequest, stream } } } - -func getEventEncodingVersion(eventEncodingVersionValue *entities.EventEncodingVersionValue) entities.EventEncodingVersion { - eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 - if requestEEV := eventEncodingVersionValue; requestEEV != nil { - eventEncodingVersion = requestEEV.GetValue() - } - return eventEncodingVersion -} diff --git a/engine/access/state_stream/handler_test.go b/engine/access/state_stream/handler_test.go index 50d7a7f32a8..8c69cee9318 100644 --- a/engine/access/state_stream/handler_test.go +++ b/engine/access/state_stream/handler_test.go @@ -2,6 +2,7 @@ package state_stream_test import ( "context" + "github.com/onflow/flow/protobuf/go/flow/entities" "io" "sync" "testing" @@ -139,7 +140,9 @@ func TestEventStream(t *testing.T) { wg.Add(1) go func() { wg.Done() - err := h.SubscribeEvents(&access.SubscribeEventsRequest{}, stream) + err := h.SubscribeEvents(&access.SubscribeEventsRequest{ + EventEncodingVersion: &entities.EventEncodingVersionValue{Value: entities.EventEncodingVersion_JSON_CDC_V0}, + }, stream) require.NoError(t, err) t.Log("subscription closed") }() @@ -173,12 +176,6 @@ func TestEventStream(t *testing.T) { assert.Equal(t, blockID, convert.MessageToIdentifier(resp.GetBlockId())) assert.Equal(t, expectedEvents, convertedEvents) - // make sure the payload is valid JSON-CDC - for _, e := range convertedEvents { - _, err := jsoncdc.Decode(nil, e.Payload) - require.NoError(t, err) - } - receivedCount++ // shutdown the stream after one response diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index 18babd35943..45527f9ccad 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -257,3 +257,11 @@ func BlockEventsToMessage(block flow.BlockEvents) (*accessproto.EventsResponse_R Events: eventMessages, }, nil } + +func GetEventEncodingVersion(eventEncodingVersionValue *entities.EventEncodingVersionValue) entities.EventEncodingVersion { + eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 + if requestEEV := eventEncodingVersionValue; requestEEV != nil { + eventEncodingVersion = requestEEV.GetValue() + } + return eventEncodingVersion +} From 9fcbb3645e01fd5e7e458189d6073f4d59c55b4b Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Tue, 10 Oct 2023 16:20:42 +0300 Subject: [PATCH 10/27] Fixed logic for conversion. --- engine/access/rpc/backend/backend_events.go | 8 ++--- engine/access/rpc/backend/backend_test.go | 30 ++++------------ .../rpc/backend/backend_transactions.go | 6 ++-- engine/access/state_stream/handler.go | 34 ++++++------------- engine/access/state_stream/handler_test.go | 5 +-- engine/common/rpc/convert/events.go | 16 ++++++--- engine/common/rpc/convert/execution_data.go | 28 ++++++++++----- .../common/rpc/convert/execution_data_test.go | 2 +- 8 files changed, 57 insertions(+), 72 deletions(-) diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index 27171705c3d..c70aedb2232 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -153,7 +153,7 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( Str("last_block_id", lastBlockID.String()). Msg("successfully got events") - eventEncodingVersion := convert.GetEventEncodingVersion(eventEncodingVersionValue) + eventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersionValue) // convert execution node api result to access node api result results, err := verifyAndConvertToAccessEvents(resp.GetResults(), blockHeaders, eventEncodingVersion) @@ -169,7 +169,7 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( func verifyAndConvertToAccessEvents( execEvents []*execproto.GetEventsForBlockIDsResponse_Result, requestedBlockHeaders []*flow.Header, - version entities.EventEncodingVersion, + eventEncodingVersion entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { if len(execEvents) != len(requestedBlockHeaders) { return nil, errors.New("number of results does not match number of blocks requested") @@ -193,10 +193,10 @@ func verifyAndConvertToAccessEvents( result.GetBlockId()) } - events, err := convert.MessagesToEventsFromVersion(result.GetEvents(), version) + events, err := convert.MessagesToEventsFromVersion(result.GetEvents(), eventEncodingVersion) if err != nil { return nil, fmt.Errorf("failed to unmarshal events in event %d with encoding version %s: %w", - i, version.String(), err) + i, eventEncodingVersion.String(), err) } results[i] = flow.BlockEvents{ diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 5fa5b6bbde7..081dd1899f6 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -2143,8 +2143,8 @@ func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { } // Define a helper function to prepare the GetTransactionResultResponse with a given encoding version. - prepareGetTransactionResultByIndex := func(version entitiesproto.EventEncodingVersion) *execproto.GetTransactionResultResponse { - events := getEventsWithEncoding(1, version) + prepareGetTransactionResultByIndex := func() *execproto.GetTransactionResultResponse { + events := getEventsWithEncoding(1, entitiesproto.EventEncodingVersion_CCF_V0) exeEventResp := &execproto.GetTransactionResultResponse{ Events: convert.EventsToMessages(events), @@ -2163,10 +2163,7 @@ func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { exeEventResp *execproto.GetTransactionResultResponse, encodingVersionValue *entitiesproto.EventEncodingVersionValue, ) { - encodingVersion := entitiesproto.EventEncodingVersion_JSON_CDC_V0 - if encodingVersionValue != nil { - encodingVersion = encodingVersionValue.GetValue() - } + encodingVersion := convert.GetConversionEventEncodingVersion(encodingVersionValue) result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, encodingVersionValue) suite.checkResponse(result, err) @@ -2177,36 +2174,21 @@ func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { } suite.Run("test default(JSON) event encoding (happy case)", func() { - encodingVersion := entitiesproto.EventEncodingVersion_JSON_CDC_V0 - exeEventResp := prepareGetTransactionResultByIndex(encodingVersion) + exeEventResp := prepareGetTransactionResultByIndex() assertResultExpectations(exeEventResp, nil) }) suite.Run("test JSON event encoding (happy case)", func() { encodingVersion := entitiesproto.EventEncodingVersion_JSON_CDC_V0 - exeEventResp := prepareGetTransactionResultByIndex(encodingVersion) + exeEventResp := prepareGetTransactionResultByIndex() assertResultExpectations(exeEventResp, &entitiesproto.EventEncodingVersionValue{Value: encodingVersion}) }) suite.Run("test CFF event encoding (happy case)", func() { encodingVersion := entitiesproto.EventEncodingVersion_CCF_V0 - exeEventResp := prepareGetTransactionResultByIndex(encodingVersion) + exeEventResp := prepareGetTransactionResultByIndex() assertResultExpectations(exeEventResp, &entitiesproto.EventEncodingVersionValue{Value: encodingVersion}) }) - - suite.Run("test wrong event conversion JSON to CFF", func() { - encodingVersion := entitiesproto.EventEncodingVersion_JSON_CDC_V0 - _ = prepareGetTransactionResultByIndex(encodingVersion) - - result, err := backend.GetTransactionResultByIndex( - ctx, - blockId, - index, - &entitiesproto.EventEncodingVersionValue{Value: entitiesproto.EventEncodingVersion_CCF_V0}, - ) - suite.Require().Error(err) - suite.Require().Nil(result) - }) } func (suite *Suite) assertAllExpectations() { diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 70b13badf5f..78c0afd9b6a 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -418,7 +418,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( "number of transaction results returned by execution node is less than the number of transactions in the block", ) - eventEncodingVersion := convert.GetEventEncodingVersion(eventEncodingVersionValue) + eventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersionValue) for _, guarantee := range block.Payload.Guarantees { collection, err := b.collections.LightByID(guarantee.CollectionID) @@ -551,7 +551,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( return nil, rpc.ConvertStorageError(err) } - eventEncodingVersion := convert.GetEventEncodingVersion(eventEncodingVersionValue) + eventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersionValue) events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) if err != nil { @@ -778,7 +778,7 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return nil, 0, "", err } - eventEncodingVersion := convert.GetEventEncodingVersion(eventEncodingVersionValue) + eventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersionValue) events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) if err != nil { diff --git a/engine/access/state_stream/handler.go b/engine/access/state_stream/handler.go index 7ae8b9ccbcf..ba6bfc9d7f7 100644 --- a/engine/access/state_stream/handler.go +++ b/engine/access/state_stream/handler.go @@ -4,7 +4,6 @@ import ( "context" "sync/atomic" - "github.com/onflow/flow/protobuf/go/flow/entities" access "github.com/onflow/flow/protobuf/go/flow/executiondata" executiondata "github.com/onflow/flow/protobuf/go/flow/executiondata" "google.golang.org/grpc/codes" @@ -52,13 +51,9 @@ func (h *Handler) GetExecutionDataByBlockID(ctx context.Context, request *access return nil, status.Errorf(codes.Internal, "could not convert execution data to entity: %v", err) } - eventEncodingVersion := convert.GetEventEncodingVersion(request.GetEventEncodingVersion()) - if eventEncodingVersion == entities.EventEncodingVersion_JSON_CDC_V0 { - // convert event payloads from CCF to JSON-CDC - err = convert.BlockExecutionDataEventPayloadsToJson(message) - if err != nil { - return nil, status.Errorf(codes.Internal, "could not convert execution data event payloads to JSON: %v", err) - } + err = convert.BlockExecutionDataEventPayloadsFromVersion(message, request.GetEventEncodingVersion()) + if err != nil { + return nil, status.Errorf(codes.Internal, "could not convert execution data event payloads to JSON: %v", err) } return &access.GetExecutionDataByBlockIDResponse{BlockExecutionData: message}, nil @@ -102,13 +97,9 @@ func (h *Handler) SubscribeExecutionData(request *access.SubscribeExecutionDataR return status.Errorf(codes.Internal, "could not convert execution data to entity: %v", err) } - eventEncodingVersion := convert.GetEventEncodingVersion(request.GetEventEncodingVersion()) - if eventEncodingVersion == entities.EventEncodingVersion_JSON_CDC_V0 { - // convert event payloads from CCF to JSON-CDC - err = convert.BlockExecutionDataEventPayloadsToJson(execData) - if err != nil { - return status.Errorf(codes.Internal, "could not convert execution data event payloads to JSON: %v", err) - } + err = convert.BlockExecutionDataEventPayloadsFromVersion(execData, request.GetEventEncodingVersion()) + if err != nil { + return status.Errorf(codes.Internal, "could not convert execution data event payloads to JSON: %v", err) } err = stream.Send(&executiondata.SubscribeExecutionDataResponse{ @@ -170,15 +161,12 @@ func (h *Handler) SubscribeEvents(request *access.SubscribeEventsRequest, stream return status.Errorf(codes.Internal, "unexpected response type: %T", v) } - eventEncodingVersion := convert.GetEventEncodingVersion(request.GetEventEncodingVersion()) - var events []*entities.Event - var err error - if eventEncodingVersion == entities.EventEncodingVersion_JSON_CDC_V0 { - events, err = convert.EventsToMessagesFromVersion(resp.Events, entities.EventEncodingVersion_CCF_V0) - if err != nil { - return status.Errorf(codes.Internal, "could not convert events to entity: %v", err) - } + eventEncodingVersion := convert.GetConversionEventEncodingVersion(request.GetEventEncodingVersion()) + events, err := convert.EventsToMessagesFromVersion(resp.Events, eventEncodingVersion) + if err != nil { + return status.Errorf(codes.Internal, "could not convert events to entity: %v", err) } + err = stream.Send(&executiondata.SubscribeEventsResponse{ BlockHeight: resp.Height, BlockId: convert.IdentifierToMessage(resp.BlockID), diff --git a/engine/access/state_stream/handler_test.go b/engine/access/state_stream/handler_test.go index 8c69cee9318..183bb5d3766 100644 --- a/engine/access/state_stream/handler_test.go +++ b/engine/access/state_stream/handler_test.go @@ -2,7 +2,6 @@ package state_stream_test import ( "context" - "github.com/onflow/flow/protobuf/go/flow/entities" "io" "sync" "testing" @@ -140,9 +139,7 @@ func TestEventStream(t *testing.T) { wg.Add(1) go func() { wg.Done() - err := h.SubscribeEvents(&access.SubscribeEventsRequest{ - EventEncodingVersion: &entities.EventEncodingVersionValue{Value: entities.EventEncodingVersion_JSON_CDC_V0}, - }, stream) + err := h.SubscribeEvents(&access.SubscribeEventsRequest{}, stream) require.NoError(t, err) t.Log("subscription closed") }() diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index 45527f9ccad..589c6bf55e3 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -258,10 +258,18 @@ func BlockEventsToMessage(block flow.BlockEvents) (*accessproto.EventsResponse_R }, nil } -func GetEventEncodingVersion(eventEncodingVersionValue *entities.EventEncodingVersionValue) entities.EventEncodingVersion { +func GetConversionEventEncodingVersion(eventEncodingVersionValue *entities.EventEncodingVersionValue) entities.EventEncodingVersion { eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 - if requestEEV := eventEncodingVersionValue; requestEEV != nil { - eventEncodingVersion = requestEEV.GetValue() + if eventEncodingVersionValue != nil { + eventEncodingVersion = eventEncodingVersionValue.GetValue() + } + + switch eventEncodingVersion { + case entities.EventEncodingVersion_CCF_V0: + return entities.EventEncodingVersion_JSON_CDC_V0 + case entities.EventEncodingVersion_JSON_CDC_V0: + fallthrough + default: + return entities.EventEncodingVersion_CCF_V0 } - return eventEncodingVersion } diff --git a/engine/common/rpc/convert/execution_data.go b/engine/common/rpc/convert/execution_data.go index d4d23930046..c220b7ef2f2 100644 --- a/engine/common/rpc/convert/execution_data.go +++ b/engine/common/rpc/convert/execution_data.go @@ -13,18 +13,28 @@ import ( "github.com/onflow/flow-go/module/executiondatasync/execution_data" ) -// BlockExecutionDataEventPayloadsToJson converts all event payloads from CCF to JSON-CDC in place -// This is a temporary workaround until a more robust solution is implemented -func BlockExecutionDataEventPayloadsToJson(m *entities.BlockExecutionData) error { - for i, chunk := range m.ChunkExecutionData { - for j, e := range chunk.Events { - converted, err := CcfPayloadToJsonPayload(e.Payload) - if err != nil { - return fmt.Errorf("failed to convert payload for event %d to json: %w", j, err) +// BlockExecutionDataEventPayloadsFromVersion converts all event payloads from version +func BlockExecutionDataEventPayloadsFromVersion( + m *entities.BlockExecutionData, + eventEncodingVersionValue *entities.EventEncodingVersionValue, +) error { + eventEncodingVersion := GetConversionEventEncodingVersion(eventEncodingVersionValue) + switch eventEncodingVersion { + case entities.EventEncodingVersion_CCF_V0: + for i, chunk := range m.ChunkExecutionData { + for j, e := range chunk.Events { + converted, err := CcfPayloadToJsonPayload(e.Payload) + if err != nil { + return fmt.Errorf("failed to convert payload for event %d to json: %w", j, err) + } + m.ChunkExecutionData[i].Events[j].Payload = converted } - m.ChunkExecutionData[i].Events[j].Payload = converted } + case entities.EventEncodingVersion_JSON_CDC_V0: + default: + return fmt.Errorf("invalid encoding format %d", eventEncodingVersion) } + return nil } diff --git a/engine/common/rpc/convert/execution_data_test.go b/engine/common/rpc/convert/execution_data_test.go index 3fe0e855bcf..ff50df4de2e 100644 --- a/engine/common/rpc/convert/execution_data_test.go +++ b/engine/common/rpc/convert/execution_data_test.go @@ -53,7 +53,7 @@ func TestConvertBlockExecutionDataEventPayloads(t *testing.T) { }) t.Run("converted event payloads are encoded in jsoncdc", func(t *testing.T) { - err = convert.BlockExecutionDataEventPayloadsToJson(execDataMessage) + err = convert.BlockExecutionDataEventPayloadsFromVersion(execDataMessage, nil) require.NoError(t, err) for _, chunk := range execDataMessage.GetChunkExecutionData() { From 3890766e7a94ab880a43d77477d35faa82f641e2 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Wed, 11 Oct 2023 11:33:01 +0300 Subject: [PATCH 11/27] Fixed test for stream --- engine/access/state_stream/handler_test.go | 127 +++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/engine/access/state_stream/handler_test.go b/engine/access/state_stream/handler_test.go index 183bb5d3766..812d51f9249 100644 --- a/engine/access/state_stream/handler_test.go +++ b/engine/access/state_stream/handler_test.go @@ -13,6 +13,7 @@ import ( "google.golang.org/grpc" jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/onflow/flow/protobuf/go/flow/entities" access "github.com/onflow/flow/protobuf/go/flow/executiondata" "github.com/onflow/flow-go/engine/access/state_stream" @@ -110,6 +111,132 @@ func TestExecutionDataStream(t *testing.T) { assert.Equal(t, 1, receivedCount) } +func TestExecutionDataStreamEventEncoding(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // generate some events with a payload to include + // generators will produce identical event payloads (before encoding) + ccfEventGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingCCF)) + inputEvents := make([]flow.Event, 0, 1) + inputEvents = append(inputEvents, ccfEventGenerator.New()) + + jsonEventsGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingJSON)) + expectedEvents := make([]flow.Event, 0, 1) + expectedEvents = append(expectedEvents, jsonEventsGenerator.New()) + + api := ssmock.NewAPI(t) + stream := makeStreamMock[access.SubscribeExecutionDataRequest, access.SubscribeExecutionDataResponse](ctx) + + makeStreamRequest := func(request *access.SubscribeExecutionDataRequest) { + sub := state_stream.NewSubscription(1) + + api.On("SubscribeExecutionData", mock.Anything, flow.ZeroID, uint64(0), mock.Anything).Return(sub) + + h := state_stream.NewHandler(api, flow.Localnet.Chain(), state_stream.EventFilterConfig{}, 1) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + wg.Done() + err := h.SubscribeExecutionData(request, stream) + require.NoError(t, err) + t.Log("subscription closed") + }() + wg.Wait() + + // send a single response + blockHeight := uint64(1) + executionData := unittest.BlockExecutionDataFixture( + unittest.WithChunkExecutionDatas( + unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(inputEvents)), + ), + ) + + err := sub.Send(ctx, &state_stream.ExecutionDataResponse{ + Height: blockHeight, + ExecutionData: executionData, + }, 100*time.Millisecond) + require.NoError(t, err) + // notify end of data + sub.Close() + } + + t.Run("test default(JSON)", func(t *testing.T) { + makeStreamRequest(&access.SubscribeExecutionDataRequest{}) + for { + resp, err := stream.RecvToClient() + if err == io.EOF { + break + } + require.NoError(t, err) + + convertedExecData, err := convert.MessageToBlockExecutionData(resp.GetBlockExecutionData(), flow.Testnet.Chain()) + require.NoError(t, err) + + // make sure the payload is valid JSON-CDC + for _, chunk := range convertedExecData.ChunkExecutionDatas { + for i, e := range chunk.Events { + assert.Equal(t, expectedEvents[i], e) + } + } + + close(stream.sentFromServer) + } + }) + + t.Run("test JSON event encoding", func(t *testing.T) { + makeStreamRequest(&access.SubscribeExecutionDataRequest{ + EventEncodingVersion: &entities.EventEncodingVersionValue{Value: entities.EventEncodingVersion_JSON_CDC_V0}, + }) + for { + resp, err := stream.RecvToClient() + if err == io.EOF { + break + } + require.NoError(t, err) + + convertedExecData, err := convert.MessageToBlockExecutionData(resp.GetBlockExecutionData(), flow.Testnet.Chain()) + require.NoError(t, err) + + // make sure the payload is valid JSON-CDC + for _, chunk := range convertedExecData.ChunkExecutionDatas { + for i, e := range chunk.Events { + assert.Equal(t, expectedEvents[i], e) + } + } + close(stream.sentFromServer) + } + + }) + + t.Run("test CFF event encoding", func(t *testing.T) { + makeStreamRequest(&access.SubscribeExecutionDataRequest{ + EventEncodingVersion: &entities.EventEncodingVersionValue{Value: entities.EventEncodingVersion_CCF_V0}, + }) + for { + resp, err := stream.RecvToClient() + if err == io.EOF { + break + } + require.NoError(t, err) + + convertedExecData, err := convert.MessageToBlockExecutionData(resp.GetBlockExecutionData(), flow.Testnet.Chain()) + require.NoError(t, err) + + // make sure the payload is valid JSON-CDC + for _, chunk := range convertedExecData.ChunkExecutionDatas { + for i, e := range chunk.Events { + assert.Equal(t, inputEvents[i], e) + } + } + close(stream.sentFromServer) + } + }) +} + func TestEventStream(t *testing.T) { t.Parallel() From bbfc9d24792b52006a5121a7b3567eb4102632b8 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Thu, 12 Oct 2023 01:08:20 +0300 Subject: [PATCH 12/27] Fixed tests --- engine/access/rpc/backend/backend_test.go | 12 +++---- engine/access/state_stream/handler_test.go | 42 ++++++++++++++-------- engine/common/rpc/convert/events.go | 19 ++++++---- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 081dd1899f6..56f25851983 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -1084,7 +1084,7 @@ func (suite *Suite) TestGetEventsForBlockIDs() { suite.Require().NoError(err) // execute request - actual, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), blockIDs, nil) + actual, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), blockIDs, &entitiesproto.EventEncodingVersionValue{Value: entitiesproto.EventEncodingVersion_CCF_V0}) suite.checkResponse(actual, err) suite.Require().Equal(expected, actual) @@ -1386,7 +1386,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { suite.Require().NoError(err) // execute request - actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, nil) + actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, &entitiesproto.EventEncodingVersionValue{Value: entitiesproto.EventEncodingVersion_CCF_V0}) // check response suite.checkResponse(actualResp, err) @@ -1414,7 +1414,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, nil) + actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, &entitiesproto.EventEncodingVersionValue{Value: entitiesproto.EventEncodingVersion_CCF_V0}) suite.checkResponse(actualResp, err) suite.assertAllExpectations() @@ -2173,18 +2173,18 @@ func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { suite.Assert().Equal(result.Events, expectedResultEvents) } - suite.Run("test default(JSON) event encoding (happy case)", func() { + suite.Run("test default(JSON) event encoding", func() { exeEventResp := prepareGetTransactionResultByIndex() assertResultExpectations(exeEventResp, nil) }) - suite.Run("test JSON event encoding (happy case)", func() { + suite.Run("test JSON event encoding", func() { encodingVersion := entitiesproto.EventEncodingVersion_JSON_CDC_V0 exeEventResp := prepareGetTransactionResultByIndex() assertResultExpectations(exeEventResp, &entitiesproto.EventEncodingVersionValue{Value: encodingVersion}) }) - suite.Run("test CFF event encoding (happy case)", func() { + suite.Run("test CFF event encoding", func() { encodingVersion := entitiesproto.EventEncodingVersion_CCF_V0 exeEventResp := prepareGetTransactionResultByIndex() assertResultExpectations(exeEventResp, &entitiesproto.EventEncodingVersionValue{Value: encodingVersion}) diff --git a/engine/access/state_stream/handler_test.go b/engine/access/state_stream/handler_test.go index 812d51f9249..94f65da20c4 100644 --- a/engine/access/state_stream/handler_test.go +++ b/engine/access/state_stream/handler_test.go @@ -111,25 +111,27 @@ func TestExecutionDataStream(t *testing.T) { assert.Equal(t, 1, receivedCount) } +// TestExecutionDataStreamEventEncoding tests the event encoding behavior of the execution data stream. func TestExecutionDataStreamEventEncoding(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // generate some events with a payload to include - // generators will produce identical event payloads (before encoding) + // Generate sample events for testing, each with a payload. ccfEventGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingCCF)) - inputEvents := make([]flow.Event, 0, 1) - inputEvents = append(inputEvents, ccfEventGenerator.New()) + ccfEvents := make([]flow.Event, 0, 1) + ccfEvents = append(ccfEvents, ccfEventGenerator.New()) jsonEventsGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingJSON)) - expectedEvents := make([]flow.Event, 0, 1) - expectedEvents = append(expectedEvents, jsonEventsGenerator.New()) + jsonEvents := make([]flow.Event, 0, 1) + jsonEvents = append(jsonEvents, jsonEventsGenerator.New()) + // Create a mock API and a mock stream for subscription. api := ssmock.NewAPI(t) stream := makeStreamMock[access.SubscribeExecutionDataRequest, access.SubscribeExecutionDataResponse](ctx) + // Helper function to perform a stream request and handle responses. makeStreamRequest := func(request *access.SubscribeExecutionDataRequest) { sub := state_stream.NewSubscription(1) @@ -147,11 +149,11 @@ func TestExecutionDataStreamEventEncoding(t *testing.T) { }() wg.Wait() - // send a single response + // Send a single response. blockHeight := uint64(1) executionData := unittest.BlockExecutionDataFixture( unittest.WithChunkExecutionDatas( - unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(inputEvents)), + unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(ccfEvents)), ), ) @@ -160,10 +162,12 @@ func TestExecutionDataStreamEventEncoding(t *testing.T) { ExecutionData: executionData, }, 100*time.Millisecond) require.NoError(t, err) - // notify end of data + + // Notify end of data. sub.Close() } + // Test scenario for default (JSON) event encoding. t.Run("test default(JSON)", func(t *testing.T) { makeStreamRequest(&access.SubscribeExecutionDataRequest{}) for { @@ -176,10 +180,10 @@ func TestExecutionDataStreamEventEncoding(t *testing.T) { convertedExecData, err := convert.MessageToBlockExecutionData(resp.GetBlockExecutionData(), flow.Testnet.Chain()) require.NoError(t, err) - // make sure the payload is valid JSON-CDC + // Verify that the payload is valid JSON-CDC. for _, chunk := range convertedExecData.ChunkExecutionDatas { for i, e := range chunk.Events { - assert.Equal(t, expectedEvents[i], e) + assert.Equal(t, jsonEvents[i], e) } } @@ -187,6 +191,7 @@ func TestExecutionDataStreamEventEncoding(t *testing.T) { } }) + // Test scenario for JSON event encoding. t.Run("test JSON event encoding", func(t *testing.T) { makeStreamRequest(&access.SubscribeExecutionDataRequest{ EventEncodingVersion: &entities.EventEncodingVersionValue{Value: entities.EventEncodingVersion_JSON_CDC_V0}, @@ -201,10 +206,10 @@ func TestExecutionDataStreamEventEncoding(t *testing.T) { convertedExecData, err := convert.MessageToBlockExecutionData(resp.GetBlockExecutionData(), flow.Testnet.Chain()) require.NoError(t, err) - // make sure the payload is valid JSON-CDC + // Verify that the payload is valid JSON-CDC. for _, chunk := range convertedExecData.ChunkExecutionDatas { for i, e := range chunk.Events { - assert.Equal(t, expectedEvents[i], e) + assert.Equal(t, jsonEvents[i], e) } } close(stream.sentFromServer) @@ -212,6 +217,7 @@ func TestExecutionDataStreamEventEncoding(t *testing.T) { }) + // Test scenario for CFF event encoding. t.Run("test CFF event encoding", func(t *testing.T) { makeStreamRequest(&access.SubscribeExecutionDataRequest{ EventEncodingVersion: &entities.EventEncodingVersionValue{Value: entities.EventEncodingVersion_CCF_V0}, @@ -226,10 +232,10 @@ func TestExecutionDataStreamEventEncoding(t *testing.T) { convertedExecData, err := convert.MessageToBlockExecutionData(resp.GetBlockExecutionData(), flow.Testnet.Chain()) require.NoError(t, err) - // make sure the payload is valid JSON-CDC + // Verify that the payload is valid CCF-V0. for _, chunk := range convertedExecData.ChunkExecutionDatas { for i, e := range chunk.Events { - assert.Equal(t, inputEvents[i], e) + assert.Equal(t, ccfEvents[i], e) } } close(stream.sentFromServer) @@ -300,6 +306,12 @@ func TestEventStream(t *testing.T) { assert.Equal(t, blockID, convert.MessageToIdentifier(resp.GetBlockId())) assert.Equal(t, expectedEvents, convertedEvents) + // make sure the payload is valid JSON-CDC + for _, e := range convertedEvents { + _, err := jsoncdc.Decode(nil, e.Payload) + require.NoError(t, err) + } + receivedCount++ // shutdown the stream after one response diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index 589c6bf55e3..652990516ee 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -258,18 +258,25 @@ func BlockEventsToMessage(block flow.BlockEvents) (*accessproto.EventsResponse_R }, nil } +// GetConversionEventEncodingVersion returns the appropriate event encoding version for conversion. +// Execution node events payloads are always encoded in CCF. If eventEncodingVersionValue is JSON-CDC, +// CFF-encoded payloads should be converted to JSON-CDC, and CFF is returned as the format to convert from. +// If eventEncodingVersionValue is CFF, payloads will not be converted and will be left as is. In this case, +// JSON-CDC is returned as the format to convert from, which will be ignored in converters, and payloads +// will be returned as-is. func GetConversionEventEncodingVersion(eventEncodingVersionValue *entities.EventEncodingVersionValue) entities.EventEncodingVersion { + // 1. Check the requested version of payloads. eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 if eventEncodingVersionValue != nil { eventEncodingVersion = eventEncodingVersionValue.GetValue() } - switch eventEncodingVersion { - case entities.EventEncodingVersion_CCF_V0: - return entities.EventEncodingVersion_JSON_CDC_V0 - case entities.EventEncodingVersion_JSON_CDC_V0: - fallthrough - default: + // 2. If it is JSON-CDC, CFF is returned as the format to convert from. + if eventEncodingVersion == entities.EventEncodingVersion_JSON_CDC_V0 { return entities.EventEncodingVersion_CCF_V0 } + + // 3. In other cases, payloads should not be converted, so JSON-CDC is returned as the format to convert from. + // Conversion will be ignored. + return entities.EventEncodingVersion_JSON_CDC_V0 } From c8a5eadc6d0b646e5277b03872591b809dd67edf Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Thu, 12 Oct 2023 17:09:28 +0300 Subject: [PATCH 13/27] Fixed tests --- engine/access/rest/routes/events_test.go | 11 ++++++++--- engine/access/rest/routes/transactions_test.go | 14 +++++++++----- engine/access/rpc/backend/backend_test.go | 6 +++--- engine/common/rpc/convert/events.go | 6 +++++- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/engine/access/rest/routes/events_test.go b/engine/access/rest/routes/events_test.go index c4bd95f4d34..b9741774e1e 100644 --- a/engine/access/rest/routes/events_test.go +++ b/engine/access/rest/routes/events_test.go @@ -15,6 +15,8 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/onflow/flow/protobuf/go/flow/entities" + "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/model/flow" @@ -159,6 +161,7 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { ids := make([]flow.Identifier, n) var lastHeader *flow.Header + var emptyEventEncodingVersion *entities.EventEncodingVersionValue for i := 0; i < n; i++ { header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(uint64(i))) ids[i] = header.ID() @@ -166,14 +169,14 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { events[i] = unittest.BlockEventsFixture(header, 2) backend.Mock. - On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, []flow.Identifier{header.ID()}). + On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, []flow.Identifier{header.ID()}, emptyEventEncodingVersion). Return([]flow.BlockEvents{events[i]}, nil) lastHeader = header } backend.Mock. - On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, ids). + On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, ids, emptyEventEncodingVersion). Return(events, nil) // range from first to last block @@ -183,6 +186,7 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { mocks.Anything, events[0].BlockHeight, events[len(events)-1].BlockHeight, + emptyEventEncodingVersion, ).Return(events, nil) // range from first to last block + 5 @@ -192,6 +196,7 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { mocks.Anything, events[0].BlockHeight, events[len(events)-1].BlockHeight+5, + emptyEventEncodingVersion, ).Return(append(events[:len(events)-1], unittest.BlockEventsFixture(lastHeader, 0)), nil) latestBlock := unittest.BlockHeaderFixture() @@ -199,7 +204,7 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { // default not found backend.Mock. - On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, mocks.Anything). + On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, mocks.Anything, emptyEventEncodingVersion). Return(nil, status.Error(codes.NotFound, "not found")) backend.Mock. diff --git a/engine/access/rest/routes/transactions_test.go b/engine/access/rest/routes/transactions_test.go index c19dd096b95..9f429449ed1 100644 --- a/engine/access/rest/routes/transactions_test.go +++ b/engine/access/rest/routes/transactions_test.go @@ -15,6 +15,8 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/onflow/flow/protobuf/go/flow/entities" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/engine/access/rest/models" @@ -118,6 +120,7 @@ func TestGetTransactions(t *testing.T) { t.Run("Get by ID with results", func(t *testing.T) { backend := &mock.API{} + var emptyEventEncodingVersion *entities.EventEncodingVersionValue tx := unittest.TransactionFixture() txr := transactionResultFixture(tx) @@ -126,7 +129,7 @@ func TestGetTransactions(t *testing.T) { Return(&tx.TransactionBody, nil) backend.Mock. - On("GetTransactionResult", mocks.Anything, tx.ID(), flow.ZeroID, flow.ZeroID). + On("GetTransactionResult", mocks.Anything, tx.ID(), flow.ZeroID, flow.ZeroID, emptyEventEncodingVersion). Return(txr, nil) req := getTransactionReq(tx.ID().String(), true, "", "") @@ -244,13 +247,14 @@ func TestGetTransactionResult(t *testing.T) { "_self": "/v1/transaction_results/%s" } }`, bid.String(), cid.String(), id.String(), util.ToBase64(txr.Events[0].Payload), id.String()) + var emptyEventEncodingVersion *entities.EventEncodingVersionValue t.Run("get by transaction ID", func(t *testing.T) { backend := &mock.API{} req := getTransactionResultReq(id.String(), "", "") backend.Mock. - On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID). + On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID, emptyEventEncodingVersion). Return(txr, nil) assertOKResponse(t, req, expected, backend) @@ -262,7 +266,7 @@ func TestGetTransactionResult(t *testing.T) { req := getTransactionResultReq(id.String(), bid.String(), "") backend.Mock. - On("GetTransactionResult", mocks.Anything, id, bid, flow.ZeroID). + On("GetTransactionResult", mocks.Anything, id, bid, flow.ZeroID, emptyEventEncodingVersion). Return(txr, nil) assertOKResponse(t, req, expected, backend) @@ -273,7 +277,7 @@ func TestGetTransactionResult(t *testing.T) { req := getTransactionResultReq(id.String(), "", cid.String()) backend.Mock. - On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, cid). + On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, cid, emptyEventEncodingVersion). Return(txr, nil) assertOKResponse(t, req, expected, backend) @@ -307,7 +311,7 @@ func TestGetTransactionResult(t *testing.T) { txResult.CollectionID = cid req := getTransactionResultReq(id.String(), "", "") backend.Mock. - On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID). + On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID, emptyEventEncodingVersion). Return(txResult, nil). Once() diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 56f25851983..d4c0917990a 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -1084,7 +1084,7 @@ func (suite *Suite) TestGetEventsForBlockIDs() { suite.Require().NoError(err) // execute request - actual, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), blockIDs, &entitiesproto.EventEncodingVersionValue{Value: entitiesproto.EventEncodingVersion_CCF_V0}) + actual, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), blockIDs, nil) suite.checkResponse(actual, err) suite.Require().Equal(expected, actual) @@ -1386,7 +1386,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { suite.Require().NoError(err) // execute request - actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, &entitiesproto.EventEncodingVersionValue{Value: entitiesproto.EventEncodingVersion_CCF_V0}) + actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, nil) // check response suite.checkResponse(actualResp, err) @@ -1414,7 +1414,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, &entitiesproto.EventEncodingVersionValue{Value: entitiesproto.EventEncodingVersion_CCF_V0}) + actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, nil) suite.checkResponse(actualResp, err) suite.assertAllExpectations() diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index 652990516ee..61c6e9baced 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -166,7 +166,7 @@ func ServiceEventListToMessages(list flow.ServiceEventList) ( return entities, nil } -// ServiceEventsToMessages converts a slice of flow.ServiceEvents to a slice of protobuf messages +// MessagesToServiceEventList converts a slice of flow.ServiceEvents to a slice of protobuf messages func MessagesToServiceEventList(m []*entities.ServiceEvent) ( flow.ServiceEventList, error, @@ -184,6 +184,10 @@ func MessagesToServiceEventList(m []*entities.ServiceEvent) ( // CcfPayloadToJsonPayload converts a CCF-encoded payload to a JSON-encoded payload func CcfPayloadToJsonPayload(p []byte) ([]byte, error) { + if len(p) == 0 { + return p, nil + } + val, err := ccf.Decode(nil, p) if err != nil { return nil, fmt.Errorf("unable to decode from ccf format: %w", err) From f1e5f25e6e142244843111df856f9faa4ef448eb Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Fri, 13 Oct 2023 14:16:56 +0300 Subject: [PATCH 14/27] Added fixed emulator dependencies --- integration/go.mod | 13 ++++++++----- integration/go.sum | 15 ++++++++------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/integration/go.mod b/integration/go.mod index 15c8077ed58..64e82bba8c6 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -22,13 +22,13 @@ require ( github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 github.com/onflow/flow-emulator v0.53.0 - github.com/onflow/flow-go v0.31.1-0.20230718164039-e3411eff1e9d + github.com/onflow/flow-go v0.31.1-0.20230808172820-f074502a67e3 github.com/onflow/flow-go-sdk v0.41.10 github.com/onflow/flow-go/crypto v0.24.9 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 github.com/plus3it/gorecurcopy v0.0.1 - github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_golang v1.16.0 github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.42.0 github.com/rs/zerolog v1.29.0 @@ -148,7 +148,7 @@ require ( github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2 // indirect - github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-20200501113911-9a95f0fdbfea // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -259,7 +259,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/psiemens/graceland v1.0.0 // indirect github.com/psiemens/sconfig v0.1.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect @@ -345,4 +345,7 @@ replace github.com/onflow/flow-go => ../ replace github.com/onflow/flow-go/insecure => ../insecure -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 +replace ( + github.com/onflow/flow-emulator v0.53.0 => github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5 + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 +) diff --git a/integration/go.sum b/integration/go.sum index abfe7f8cec1..880354f63dc 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -100,6 +100,8 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5 h1:33av+pFXKq9WNbg3EvIFQoYuDBdTqVrzc8/XCMkGc5o= +github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5/go.mod h1:gKnyRos42AIy5wm1BfHUDhhrg322F0WpIAsOU+nljj0= github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 h1:k/OJbM33BwI9sEJvNMzmkhM0ZJTSZeCoLImkBLldsJ8= github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= @@ -693,8 +695,9 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmg github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2 h1:uxUHSMwWDJ/9jVPHNumRC8WZOi3hrBL22ObVOoLg4ww= github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2/go.mod h1:BL7w7qd2l/j9jgY6WMhYutfOFQc0I8RTVwtjpnAMoTM= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-20200501113911-9a95f0fdbfea h1:1Tk1IbruXbunEnaIZEFb+Hpv9BIZti3OxKwKn5wWyKk= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-20200501113911-9a95f0fdbfea/go.mod h1:GugMBs30ZSAkckqXEAIEGyYdDH6EgqowG8ppA3Zt+AY= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2 h1:1aeRCnE2CkKYqyzBu0+B2lgTcZPc3ea2lGpijeHbI1c= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2/go.mod h1:GhphxcdlaRyAuBSvo6rV71BvQcvB/vuX8ugCyybuS2k= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= @@ -1435,8 +1438,6 @@ github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-5 github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d/go.mod h1:xAiV/7TKhw863r6iO3CS5RnQ4F+pBY1TxD272BsILlo= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= -github.com/onflow/flow-emulator v0.53.0 h1:VIMljBL77VnO+CeeJX1N5GVmF245XwZrFGv63dLPQGk= -github.com/onflow/flow-emulator v0.53.0/go.mod h1:o7O+b3fQYs26vJ+4SeMY/T9kA1rT09tFxQccTFyM5b4= github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= @@ -1553,8 +1554,8 @@ github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQ github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1584,8 +1585,8 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/psiemens/graceland v1.0.0 h1:L580AVV4Q2XLcPpmvxJRH9UpEAYr/eu2jBKmMglhvM8= From 8adf317eaebae5e81630ce8424c50cb288d89fcd Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Fri, 13 Oct 2023 14:49:19 +0300 Subject: [PATCH 15/27] Fixed insecure packege --- insecure/go.mod | 2 ++ insecure/go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/insecure/go.mod b/insecure/go.mod index 1bbd6943a68..c32ea86ca31 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -287,3 +287,5 @@ require ( ) replace github.com/onflow/flow-go => ../ + +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 diff --git a/insecure/go.sum b/insecure/go.sum index 2d5b86e94fa..9f4689d484e 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -87,6 +87,8 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 h1:k/OJbM33BwI9sEJvNMzmkhM0ZJTSZeCoLImkBLldsJ8= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -1302,8 +1304,6 @@ github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 h1:IGDANryEVnuVAlDZDzNLMMUui9lbwS04KE70C0HXkFw= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= From 11a597896d4d1077a385221e9095b8cd8c442d76 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Wed, 18 Oct 2023 12:54:22 +0300 Subject: [PATCH 16/27] Changed EventEncodingVersionValue to EventEncodingVersion. --- access/api.go | 10 ++--- access/legacy/handler.go | 13 ++++-- access/mock/api.go | 40 +++++++++---------- .../rest/apiproxy/rest_proxy_handler.go | 12 +++--- .../models/model_event_encoding_version.go | 19 +++++++++ engine/access/rest/routes/events_test.go | 11 +++-- .../access/rest/routes/transactions_test.go | 11 +++-- engine/access/rpc/backend/backend_events.go | 18 +++++---- .../rpc/backend/backend_transactions.go | 28 ++++++------- engine/access/state_stream/handler.go | 4 +- engine/common/rpc/convert/events.go | 21 ++-------- engine/common/rpc/convert/execution_data.go | 30 +++++++------- .../common/rpc/convert/execution_data_test.go | 2 +- go.mod | 2 +- go.sum | 4 +- insecure/go.mod | 2 +- insecure/go.sum | 4 +- integration/go.mod | 2 +- integration/go.sum | 4 +- 19 files changed, 124 insertions(+), 113 deletions(-) create mode 100644 engine/access/rest/models/model_event_encoding_version.go diff --git a/access/api.go b/access/api.go index 66ee14ae1d5..b28b7b333b3 100644 --- a/access/api.go +++ b/access/api.go @@ -29,9 +29,9 @@ type API interface { SendTransaction(ctx context.Context, tx *flow.TransactionBody) error GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error) GetTransactionsByBlockID(ctx context.Context, blockID flow.Identifier) ([]*flow.TransactionBody, error) - GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, eventEncodingVersionValue *entities.EventEncodingVersionValue) (*TransactionResult, error) - GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, eventEncodingVersionValue *entities.EventEncodingVersionValue) (*TransactionResult, error) - GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, eventEncodingVersionValue *entities.EventEncodingVersionValue) ([]*TransactionResult, error) + GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, eventEncodingVersionValue entities.EventEncodingVersion) (*TransactionResult, error) + GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, eventEncodingVersionValue entities.EventEncodingVersion) (*TransactionResult, error) + GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, eventEncodingVersionValue entities.EventEncodingVersion) ([]*TransactionResult, error) GetAccount(ctx context.Context, address flow.Address) (*flow.Account, error) GetAccountAtLatestBlock(ctx context.Context, address flow.Address) (*flow.Account, error) @@ -41,8 +41,8 @@ type API interface { ExecuteScriptAtBlockHeight(ctx context.Context, blockHeight uint64, script []byte, arguments [][]byte) ([]byte, error) ExecuteScriptAtBlockID(ctx context.Context, blockID flow.Identifier, script []byte, arguments [][]byte) ([]byte, error) - GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64, eventEncodingVersionValue *entities.EventEncodingVersionValue) ([]flow.BlockEvents, error) - GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersionValue *entities.EventEncodingVersionValue) ([]flow.BlockEvents, error) + GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64, eventEncodingVersionValue entities.EventEncodingVersion) ([]flow.BlockEvents, error) + GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersionValue entities.EventEncodingVersion) ([]flow.BlockEvents, error) GetLatestProtocolStateSnapshot(ctx context.Context) ([]byte, error) diff --git a/access/legacy/handler.go b/access/legacy/handler.go index f81090daab0..fe0cbd33dba 100644 --- a/access/legacy/handler.go +++ b/access/legacy/handler.go @@ -3,6 +3,7 @@ package handler import ( "context" + "github.com/onflow/flow/protobuf/go/flow/entities" accessproto "github.com/onflow/flow/protobuf/go/flow/legacy/access" entitiesproto "github.com/onflow/flow/protobuf/go/flow/legacy/entities" "google.golang.org/grpc/codes" @@ -189,7 +190,9 @@ func (h *Handler) GetTransactionResult( ) (*accessproto.TransactionResultResponse, error) { id := convert.MessageToIdentifier(req.GetId()) - result, err := h.api.GetTransactionResult(ctx, id, flow.ZeroID, flow.ZeroID, nil) + result, err := h.api.GetTransactionResult(ctx, id, flow.ZeroID, flow.ZeroID, &entities.EventEncodingVersionValue{ + Value: entities.EventEncodingVersion_JSON_CDC_V0, + }) if err != nil { return nil, err } @@ -313,7 +316,9 @@ func (h *Handler) GetEventsForHeightRange( startHeight := req.GetStartHeight() endHeight := req.GetEndHeight() - results, err := h.api.GetEventsForHeightRange(ctx, eventType, startHeight, endHeight, nil) + results, err := h.api.GetEventsForHeightRange(ctx, eventType, startHeight, endHeight, &entities.EventEncodingVersionValue{ + Value: entities.EventEncodingVersion_JSON_CDC_V0, + }) if err != nil { return nil, err } @@ -331,7 +336,9 @@ func (h *Handler) GetEventsForBlockIDs( eventType := req.GetType() blockIDs := convert.MessagesToIdentifiers(req.GetBlockIds()) - results, err := h.api.GetEventsForBlockIDs(ctx, eventType, blockIDs, nil) + results, err := h.api.GetEventsForBlockIDs(ctx, eventType, blockIDs, &entities.EventEncodingVersionValue{ + Value: entities.EventEncodingVersion_JSON_CDC_V0, + }) if err != nil { return nil, err } diff --git a/access/mock/api.go b/access/mock/api.go index 9a3f2a8db08..f79189be45c 100644 --- a/access/mock/api.go +++ b/access/mock/api.go @@ -334,15 +334,15 @@ func (_m *API) GetCollectionByID(ctx context.Context, id flow.Identifier) (*flow } // GetEventsForBlockIDs provides a mock function with given fields: ctx, eventType, blockIDs, eventEncodingVersionValue -func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersionValue *entities.EventEncodingVersionValue) ([]flow.BlockEvents, error) { +func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersionValue entities.EventEncodingVersion) ([]flow.BlockEvents, error) { ret := _m.Called(ctx, eventType, blockIDs, eventEncodingVersionValue) var r0 []flow.BlockEvents var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier, *entities.EventEncodingVersionValue) ([]flow.BlockEvents, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier, entities.EventEncodingVersion) ([]flow.BlockEvents, error)); ok { return rf(ctx, eventType, blockIDs, eventEncodingVersionValue) } - if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier, *entities.EventEncodingVersionValue) []flow.BlockEvents); ok { + if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier, entities.EventEncodingVersion) []flow.BlockEvents); ok { r0 = rf(ctx, eventType, blockIDs, eventEncodingVersionValue) } else { if ret.Get(0) != nil { @@ -350,7 +350,7 @@ func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, block } } - if rf, ok := ret.Get(1).(func(context.Context, string, []flow.Identifier, *entities.EventEncodingVersionValue) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, string, []flow.Identifier, entities.EventEncodingVersion) error); ok { r1 = rf(ctx, eventType, blockIDs, eventEncodingVersionValue) } else { r1 = ret.Error(1) @@ -360,15 +360,15 @@ func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, block } // GetEventsForHeightRange provides a mock function with given fields: ctx, eventType, startHeight, endHeight, eventEncodingVersionValue -func (_m *API) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight uint64, endHeight uint64, eventEncodingVersionValue *entities.EventEncodingVersionValue) ([]flow.BlockEvents, error) { +func (_m *API) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight uint64, endHeight uint64, eventEncodingVersionValue entities.EventEncodingVersion) ([]flow.BlockEvents, error) { ret := _m.Called(ctx, eventType, startHeight, endHeight, eventEncodingVersionValue) var r0 []flow.BlockEvents var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64, *entities.EventEncodingVersionValue) ([]flow.BlockEvents, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64, entities.EventEncodingVersion) ([]flow.BlockEvents, error)); ok { return rf(ctx, eventType, startHeight, endHeight, eventEncodingVersionValue) } - if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64, *entities.EventEncodingVersionValue) []flow.BlockEvents); ok { + if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64, entities.EventEncodingVersion) []flow.BlockEvents); ok { r0 = rf(ctx, eventType, startHeight, endHeight, eventEncodingVersionValue) } else { if ret.Get(0) != nil { @@ -376,7 +376,7 @@ func (_m *API) GetEventsForHeightRange(ctx context.Context, eventType string, st } } - if rf, ok := ret.Get(1).(func(context.Context, string, uint64, uint64, *entities.EventEncodingVersionValue) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, string, uint64, uint64, entities.EventEncodingVersion) error); ok { r1 = rf(ctx, eventType, startHeight, endHeight, eventEncodingVersionValue) } else { r1 = ret.Error(1) @@ -596,15 +596,15 @@ func (_m *API) GetTransaction(ctx context.Context, id flow.Identifier) (*flow.Tr } // GetTransactionResult provides a mock function with given fields: ctx, id, blockID, collectionID, eventEncodingVersionValue -func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, eventEncodingVersionValue *entities.EventEncodingVersionValue) (*access.TransactionResult, error) { +func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, eventEncodingVersionValue entities.EventEncodingVersion) (*access.TransactionResult, error) { ret := _m.Called(ctx, id, blockID, collectionID, eventEncodingVersionValue) var r0 *access.TransactionResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, *entities.EventEncodingVersionValue) (*access.TransactionResult, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, entities.EventEncodingVersion) (*access.TransactionResult, error)); ok { return rf(ctx, id, blockID, collectionID, eventEncodingVersionValue) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, *entities.EventEncodingVersionValue) *access.TransactionResult); ok { + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, entities.EventEncodingVersion) *access.TransactionResult); ok { r0 = rf(ctx, id, blockID, collectionID, eventEncodingVersionValue) } else { if ret.Get(0) != nil { @@ -612,7 +612,7 @@ func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blo } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, *entities.EventEncodingVersionValue) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, entities.EventEncodingVersion) error); ok { r1 = rf(ctx, id, blockID, collectionID, eventEncodingVersionValue) } else { r1 = ret.Error(1) @@ -622,15 +622,15 @@ func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blo } // GetTransactionResultByIndex provides a mock function with given fields: ctx, blockID, index, eventEncodingVersionValue -func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, eventEncodingVersionValue *entities.EventEncodingVersionValue) (*access.TransactionResult, error) { +func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, eventEncodingVersionValue entities.EventEncodingVersion) (*access.TransactionResult, error) { ret := _m.Called(ctx, blockID, index, eventEncodingVersionValue) var r0 *access.TransactionResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32, *entities.EventEncodingVersionValue) (*access.TransactionResult, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32, entities.EventEncodingVersion) (*access.TransactionResult, error)); ok { return rf(ctx, blockID, index, eventEncodingVersionValue) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32, *entities.EventEncodingVersionValue) *access.TransactionResult); ok { + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32, entities.EventEncodingVersion) *access.TransactionResult); ok { r0 = rf(ctx, blockID, index, eventEncodingVersionValue) } else { if ret.Get(0) != nil { @@ -638,7 +638,7 @@ func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Ide } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, uint32, *entities.EventEncodingVersionValue) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, uint32, entities.EventEncodingVersion) error); ok { r1 = rf(ctx, blockID, index, eventEncodingVersionValue) } else { r1 = ret.Error(1) @@ -648,15 +648,15 @@ func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Ide } // GetTransactionResultsByBlockID provides a mock function with given fields: ctx, blockID, eventEncodingVersionValue -func (_m *API) GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, eventEncodingVersionValue *entities.EventEncodingVersionValue) ([]*access.TransactionResult, error) { +func (_m *API) GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, eventEncodingVersionValue entities.EventEncodingVersion) ([]*access.TransactionResult, error) { ret := _m.Called(ctx, blockID, eventEncodingVersionValue) var r0 []*access.TransactionResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, *entities.EventEncodingVersionValue) ([]*access.TransactionResult, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, entities.EventEncodingVersion) ([]*access.TransactionResult, error)); ok { return rf(ctx, blockID, eventEncodingVersionValue) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, *entities.EventEncodingVersionValue) []*access.TransactionResult); ok { + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, entities.EventEncodingVersion) []*access.TransactionResult); ok { r0 = rf(ctx, blockID, eventEncodingVersionValue) } else { if ret.Get(0) != nil { @@ -664,7 +664,7 @@ func (_m *API) GetTransactionResultsByBlockID(ctx context.Context, blockID flow. } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, *entities.EventEncodingVersionValue) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, entities.EventEncodingVersion) error); ok { r1 = rf(ctx, blockID, eventEncodingVersionValue) } else { r1 = ret.Error(1) diff --git a/engine/access/rest/apiproxy/rest_proxy_handler.go b/engine/access/rest/apiproxy/rest_proxy_handler.go index e65eb96be5e..03ffce0b4e3 100644 --- a/engine/access/rest/apiproxy/rest_proxy_handler.go +++ b/engine/access/rest/apiproxy/rest_proxy_handler.go @@ -153,7 +153,7 @@ func (r *RestProxyHandler) GetTransactionResult( id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, - eventEncodingVersionValue *entities.EventEncodingVersionValue, + eventEncodingVersion entities.EventEncodingVersion, ) (*access.TransactionResult, error) { upstream, err := r.FaultTolerantClient() if err != nil { @@ -165,7 +165,7 @@ func (r *RestProxyHandler) GetTransactionResult( Id: id[:], BlockId: blockID[:], CollectionId: collectionID[:], - EventEncodingVersion: eventEncodingVersionValue, + EventEncodingVersion: eventEncodingVersion, } transactionResultResponse, err := upstream.GetTransactionResult(ctx, getTransactionResultRequest) @@ -270,7 +270,7 @@ func (r *RestProxyHandler) GetEventsForHeightRange( ctx context.Context, eventType string, startHeight, endHeight uint64, - eventEncodingVersionValue *entities.EventEncodingVersionValue, + eventEncodingVersion entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { upstream, err := r.FaultTolerantClient() if err != nil { @@ -281,7 +281,7 @@ func (r *RestProxyHandler) GetEventsForHeightRange( Type: eventType, StartHeight: startHeight, EndHeight: endHeight, - EventEncodingVersion: eventEncodingVersionValue, + EventEncodingVersion: eventEncodingVersion, } eventsResponse, err := upstream.GetEventsForHeightRange(ctx, getEventsForHeightRangeRequest) r.log("upstream", "GetEventsForHeightRange", err) @@ -298,7 +298,7 @@ func (r *RestProxyHandler) GetEventsForBlockIDs( ctx context.Context, eventType string, blockIDs []flow.Identifier, - eventEncodingVersionValue *entities.EventEncodingVersionValue, + eventEncodingVersion entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { upstream, err := r.FaultTolerantClient() if err != nil { @@ -310,7 +310,7 @@ func (r *RestProxyHandler) GetEventsForBlockIDs( getEventsForBlockIDsRequest := &accessproto.GetEventsForBlockIDsRequest{ Type: eventType, BlockIds: blockIds, - EventEncodingVersion: eventEncodingVersionValue, + EventEncodingVersion: eventEncodingVersion, } eventsResponse, err := upstream.GetEventsForBlockIDs(ctx, getEventsForBlockIDsRequest) r.log("upstream", "GetEventsForBlockIDs", err) diff --git a/engine/access/rest/models/model_event_encoding_version.go b/engine/access/rest/models/model_event_encoding_version.go new file mode 100644 index 00000000000..5ca98c9ee1b --- /dev/null +++ b/engine/access/rest/models/model_event_encoding_version.go @@ -0,0 +1,19 @@ +/* + * Access API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 1.0.0 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ +package models + +// EventEncodingVersion : This value indicates preferred events payload encoding for responses. +type EventEncodingVersion string + +// List of EventEncodingVersion +const ( + DEFAULT__EventEncodingVersion EventEncodingVersion = "Default" + JSON_CDC_V0_EventEncodingVersion EventEncodingVersion = "JsonCdcV0" + CCF_V0_EventEncodingVersion EventEncodingVersion = "CcfV0" +) diff --git a/engine/access/rest/routes/events_test.go b/engine/access/rest/routes/events_test.go index b9741774e1e..fa9f53a92cd 100644 --- a/engine/access/rest/routes/events_test.go +++ b/engine/access/rest/routes/events_test.go @@ -161,7 +161,6 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { ids := make([]flow.Identifier, n) var lastHeader *flow.Header - var emptyEventEncodingVersion *entities.EventEncodingVersionValue for i := 0; i < n; i++ { header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(uint64(i))) ids[i] = header.ID() @@ -169,14 +168,14 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { events[i] = unittest.BlockEventsFixture(header, 2) backend.Mock. - On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, []flow.Identifier{header.ID()}, emptyEventEncodingVersion). + On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, []flow.Identifier{header.ID()}, entities.EventEncodingVersion_DEFAULT). Return([]flow.BlockEvents{events[i]}, nil) lastHeader = header } backend.Mock. - On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, ids, emptyEventEncodingVersion). + On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, ids, entities.EventEncodingVersion_DEFAULT). Return(events, nil) // range from first to last block @@ -186,7 +185,7 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { mocks.Anything, events[0].BlockHeight, events[len(events)-1].BlockHeight, - emptyEventEncodingVersion, + entities.EventEncodingVersion_DEFAULT, ).Return(events, nil) // range from first to last block + 5 @@ -196,7 +195,7 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { mocks.Anything, events[0].BlockHeight, events[len(events)-1].BlockHeight+5, - emptyEventEncodingVersion, + entities.EventEncodingVersion_DEFAULT, ).Return(append(events[:len(events)-1], unittest.BlockEventsFixture(lastHeader, 0)), nil) latestBlock := unittest.BlockHeaderFixture() @@ -204,7 +203,7 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { // default not found backend.Mock. - On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, mocks.Anything, emptyEventEncodingVersion). + On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, mocks.Anything, entities.EventEncodingVersion_DEFAULT). Return(nil, status.Error(codes.NotFound, "not found")) backend.Mock. diff --git a/engine/access/rest/routes/transactions_test.go b/engine/access/rest/routes/transactions_test.go index 9f429449ed1..865e9b4fdde 100644 --- a/engine/access/rest/routes/transactions_test.go +++ b/engine/access/rest/routes/transactions_test.go @@ -120,7 +120,7 @@ func TestGetTransactions(t *testing.T) { t.Run("Get by ID with results", func(t *testing.T) { backend := &mock.API{} - var emptyEventEncodingVersion *entities.EventEncodingVersionValue + var emptyEventEncodingVersion entities.EventEncodingVersion tx := unittest.TransactionFixture() txr := transactionResultFixture(tx) @@ -247,14 +247,13 @@ func TestGetTransactionResult(t *testing.T) { "_self": "/v1/transaction_results/%s" } }`, bid.String(), cid.String(), id.String(), util.ToBase64(txr.Events[0].Payload), id.String()) - var emptyEventEncodingVersion *entities.EventEncodingVersionValue t.Run("get by transaction ID", func(t *testing.T) { backend := &mock.API{} req := getTransactionResultReq(id.String(), "", "") backend.Mock. - On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID, emptyEventEncodingVersion). + On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID, entities.EventEncodingVersion_DEFAULT). Return(txr, nil) assertOKResponse(t, req, expected, backend) @@ -266,7 +265,7 @@ func TestGetTransactionResult(t *testing.T) { req := getTransactionResultReq(id.String(), bid.String(), "") backend.Mock. - On("GetTransactionResult", mocks.Anything, id, bid, flow.ZeroID, emptyEventEncodingVersion). + On("GetTransactionResult", mocks.Anything, id, bid, flow.ZeroID, entities.EventEncodingVersion_DEFAULT). Return(txr, nil) assertOKResponse(t, req, expected, backend) @@ -277,7 +276,7 @@ func TestGetTransactionResult(t *testing.T) { req := getTransactionResultReq(id.String(), "", cid.String()) backend.Mock. - On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, cid, emptyEventEncodingVersion). + On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, cid, entities.EventEncodingVersion_DEFAULT). Return(txr, nil) assertOKResponse(t, req, expected, backend) @@ -311,7 +310,7 @@ func TestGetTransactionResult(t *testing.T) { txResult.CollectionID = cid req := getTransactionResultReq(id.String(), "", "") backend.Mock. - On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID, emptyEventEncodingVersion). + On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID, entities.EventEncodingVersion_DEFAULT). Return(txResult, nil). Once() diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index c70aedb2232..3b31df4c555 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -38,7 +38,7 @@ func (b *backendEvents) GetEventsForHeightRange( ctx context.Context, eventType string, startHeight, endHeight uint64, - eventEncodingVersionValue *entities.EventEncodingVersionValue, + eventEncodingVersion entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { if endHeight < startHeight { @@ -80,7 +80,7 @@ func (b *backendEvents) GetEventsForHeightRange( blockHeaders = append(blockHeaders, header) } - return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, eventEncodingVersionValue) + return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, eventEncodingVersion) } // GetEventsForBlockIDs retrieves events for all the specified block IDs that have the given type @@ -88,7 +88,7 @@ func (b *backendEvents) GetEventsForBlockIDs( ctx context.Context, eventType string, blockIDs []flow.Identifier, - eventEncodingVersionValue *entities.EventEncodingVersionValue, + eventEncodingVersion entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { if uint(len(blockIDs)) > b.maxHeightRange { @@ -107,14 +107,14 @@ func (b *backendEvents) GetEventsForBlockIDs( } // forward the request to the execution node - return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, eventEncodingVersionValue) + return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, eventEncodingVersion) } func (b *backendEvents) getBlockEventsFromExecutionNode( ctx context.Context, blockHeaders []*flow.Header, eventType string, - eventEncodingVersionValue *entities.EventEncodingVersionValue, + eventEncodingVersion entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { // create an execution API request for events at block ID @@ -153,10 +153,12 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( Str("last_block_id", lastBlockID.String()). Msg("successfully got events") - eventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersionValue) - // convert execution node api result to access node api result - results, err := verifyAndConvertToAccessEvents(resp.GetResults(), blockHeaders, eventEncodingVersion) + results, err := verifyAndConvertToAccessEvents( + resp.GetResults(), + blockHeaders, + convert.GetConversionEventEncodingVersion(eventEncodingVersion), + ) if err != nil { return nil, status.Errorf(codes.Internal, "failed to verify retrieved events from execution node: %v", err) } diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 78c0afd9b6a..a3a150cc066 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -228,7 +228,7 @@ func (b *backendTransactions) GetTransactionResult( txID flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, - eventEncodingVersionValue *entities.EventEncodingVersionValue, + eventEncodingVersion entities.EventEncodingVersion, ) (*access.TransactionResult, error) { // look up transaction from storage start := time.Now() @@ -287,7 +287,7 @@ func (b *backendTransactions) GetTransactionResult( // access node may not have the block if it hasn't yet been finalized, hence block can be nil at this point if block != nil { foundBlockID := block.ID() - transactionWasExecuted, events, statusCode, txError, err = b.lookupTransactionResult(ctx, txID, foundBlockID, eventEncodingVersionValue) + transactionWasExecuted, events, statusCode, txError, err = b.lookupTransactionResult(ctx, txID, foundBlockID, eventEncodingVersion) if err != nil { return nil, rpc.ConvertError(err, "failed to retrieve result from any execution node", codes.Internal) } @@ -386,7 +386,7 @@ func (b *backendTransactions) retrieveBlock( func (b *backendTransactions) GetTransactionResultsByBlockID( ctx context.Context, blockID flow.Identifier, - eventEncodingVersionValue *entities.EventEncodingVersionValue, + eventEncodingVersion entities.EventEncodingVersion, ) ([]*access.TransactionResult, error) { // TODO: consider using storage.Index.ByBlockID, the index contains collection id and seals ID block, err := b.blocks.ByID(blockID) @@ -418,7 +418,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( "number of transaction results returned by execution node is less than the number of transactions in the block", ) - eventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersionValue) + requestedEventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersion) for _, guarantee := range block.Payload.Guarantees { collection, err := b.collections.LightByID(guarantee.CollectionID) @@ -438,7 +438,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( if err != nil { return nil, rpc.ConvertStorageError(err) } - events, err := convert.MessagesToEventsFromVersion(txResult.GetEvents(), eventEncodingVersion) + events, err := convert.MessagesToEventsFromVersion(txResult.GetEvents(), requestedEventEncodingVersion) if err != nil { return nil, status.Errorf(codes.Internal, "failed to convert events to message in txID %x: %v", txID, err) @@ -493,7 +493,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( return nil, rpc.ConvertStorageError(err) } - events, err := convert.MessagesToEventsFromVersion(systemTxResult.GetEvents(), eventEncodingVersion) + events, err := convert.MessagesToEventsFromVersion(systemTxResult.GetEvents(), requestedEventEncodingVersion) if err != nil { return nil, rpc.ConvertError(err, "failed to convert events from system tx result", codes.Internal) } @@ -518,7 +518,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( ctx context.Context, blockID flow.Identifier, index uint32, - eventEncodingVersionValue *entities.EventEncodingVersionValue, + eventEncodingVersion entities.EventEncodingVersion, ) (*access.TransactionResult, error) { // TODO: https://github.com/onflow/flow-go/issues/2175 so caching doesn't cause a circular dependency block, err := b.blocks.ByID(blockID) @@ -551,9 +551,9 @@ func (b *backendTransactions) GetTransactionResultByIndex( return nil, rpc.ConvertStorageError(err) } - eventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersionValue) + requestedEventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersion) - events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) + events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), requestedEventEncodingVersion) if err != nil { return nil, status.Errorf(codes.Internal, "failed to convert events in blockID %x: %v", blockID, err) } @@ -670,9 +670,9 @@ func (b *backendTransactions) lookupTransactionResult( ctx context.Context, txID flow.Identifier, blockID flow.Identifier, - eventEncodingVersionValue *entities.EventEncodingVersionValue, + eventEncodingVersion entities.EventEncodingVersion, ) (bool, []flow.Event, uint32, string, error) { - events, txStatus, message, err := b.getTransactionResultFromExecutionNode(ctx, blockID, txID[:], eventEncodingVersionValue) + events, txStatus, message, err := b.getTransactionResultFromExecutionNode(ctx, blockID, txID[:], eventEncodingVersion) if err != nil { // if either the execution node reported no results or the execution node could not be chosen if status.Code(err) == codes.NotFound { @@ -755,7 +755,7 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( ctx context.Context, blockID flow.Identifier, transactionID []byte, - eventEncodingVersionValue *entities.EventEncodingVersionValue, + eventEncodingVersion entities.EventEncodingVersion, ) ([]flow.Event, uint32, string, error) { // create an execution API request for events at blockID and transactionID @@ -778,9 +778,9 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return nil, 0, "", err } - eventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersionValue) + requestedEventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersion) - events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), eventEncodingVersion) + events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), requestedEventEncodingVersion) if err != nil { return nil, 0, "", rpc.ConvertError(err, "failed to convert events to message", codes.Internal) } diff --git a/engine/access/state_stream/handler.go b/engine/access/state_stream/handler.go index ba6bfc9d7f7..c1076f966a2 100644 --- a/engine/access/state_stream/handler.go +++ b/engine/access/state_stream/handler.go @@ -51,7 +51,7 @@ func (h *Handler) GetExecutionDataByBlockID(ctx context.Context, request *access return nil, status.Errorf(codes.Internal, "could not convert execution data to entity: %v", err) } - err = convert.BlockExecutionDataEventPayloadsFromVersion(message, request.GetEventEncodingVersion()) + err = convert.BlockExecutionDataEventPayloadsToVersion(message, request.GetEventEncodingVersion()) if err != nil { return nil, status.Errorf(codes.Internal, "could not convert execution data event payloads to JSON: %v", err) } @@ -97,7 +97,7 @@ func (h *Handler) SubscribeExecutionData(request *access.SubscribeExecutionDataR return status.Errorf(codes.Internal, "could not convert execution data to entity: %v", err) } - err = convert.BlockExecutionDataEventPayloadsFromVersion(execData, request.GetEventEncodingVersion()) + err = convert.BlockExecutionDataEventPayloadsToVersion(execData, request.GetEventEncodingVersion()) if err != nil { return status.Errorf(codes.Internal, "could not convert execution data event payloads to JSON: %v", err) } diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index 61c6e9baced..eac58518d1e 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -263,24 +263,11 @@ func BlockEventsToMessage(block flow.BlockEvents) (*accessproto.EventsResponse_R } // GetConversionEventEncodingVersion returns the appropriate event encoding version for conversion. -// Execution node events payloads are always encoded in CCF. If eventEncodingVersionValue is JSON-CDC, -// CFF-encoded payloads should be converted to JSON-CDC, and CFF is returned as the format to convert from. -// If eventEncodingVersionValue is CFF, payloads will not be converted and will be left as is. In this case, -// JSON-CDC is returned as the format to convert from, which will be ignored in converters, and payloads -// will be returned as-is. -func GetConversionEventEncodingVersion(eventEncodingVersionValue *entities.EventEncodingVersionValue) entities.EventEncodingVersion { +func GetConversionEventEncodingVersion(requestedEventEncodingVersion entities.EventEncodingVersion) entities.EventEncodingVersion { // 1. Check the requested version of payloads. - eventEncodingVersion := entities.EventEncodingVersion_JSON_CDC_V0 - if eventEncodingVersionValue != nil { - eventEncodingVersion = eventEncodingVersionValue.GetValue() + if requestedEventEncodingVersion == entities.EventEncodingVersion_DEFAULT { + return entities.EventEncodingVersion_JSON_CDC_V0 } - // 2. If it is JSON-CDC, CFF is returned as the format to convert from. - if eventEncodingVersion == entities.EventEncodingVersion_JSON_CDC_V0 { - return entities.EventEncodingVersion_CCF_V0 - } - - // 3. In other cases, payloads should not be converted, so JSON-CDC is returned as the format to convert from. - // Conversion will be ignored. - return entities.EventEncodingVersion_JSON_CDC_V0 + return requestedEventEncodingVersion } diff --git a/engine/common/rpc/convert/execution_data.go b/engine/common/rpc/convert/execution_data.go index c220b7ef2f2..64527a08393 100644 --- a/engine/common/rpc/convert/execution_data.go +++ b/engine/common/rpc/convert/execution_data.go @@ -13,26 +13,24 @@ import ( "github.com/onflow/flow-go/module/executiondatasync/execution_data" ) -// BlockExecutionDataEventPayloadsFromVersion converts all event payloads from version -func BlockExecutionDataEventPayloadsFromVersion( +// BlockExecutionDataEventPayloadsToVersion converts all event payloads to version +func BlockExecutionDataEventPayloadsToVersion( m *entities.BlockExecutionData, - eventEncodingVersionValue *entities.EventEncodingVersionValue, + to entities.EventEncodingVersion, ) error { - eventEncodingVersion := GetConversionEventEncodingVersion(eventEncodingVersionValue) - switch eventEncodingVersion { - case entities.EventEncodingVersion_CCF_V0: - for i, chunk := range m.ChunkExecutionData { - for j, e := range chunk.Events { - converted, err := CcfPayloadToJsonPayload(e.Payload) - if err != nil { - return fmt.Errorf("failed to convert payload for event %d to json: %w", j, err) - } - m.ChunkExecutionData[i].Events[j].Payload = converted + eventEncodingVersion := GetConversionEventEncodingVersion(to) + if eventEncodingVersion == entities.EventEncodingVersion_CCF_V0 { + return nil + } + + for i, chunk := range m.ChunkExecutionData { + for j, e := range chunk.Events { + converted, err := CcfPayloadToJsonPayload(e.Payload) + if err != nil { + return fmt.Errorf("failed to convert payload for event %d to json: %w", j, err) } + m.ChunkExecutionData[i].Events[j].Payload = converted } - case entities.EventEncodingVersion_JSON_CDC_V0: - default: - return fmt.Errorf("invalid encoding format %d", eventEncodingVersion) } return nil diff --git a/engine/common/rpc/convert/execution_data_test.go b/engine/common/rpc/convert/execution_data_test.go index ff50df4de2e..35e5396d91e 100644 --- a/engine/common/rpc/convert/execution_data_test.go +++ b/engine/common/rpc/convert/execution_data_test.go @@ -53,7 +53,7 @@ func TestConvertBlockExecutionDataEventPayloads(t *testing.T) { }) t.Run("converted event payloads are encoded in jsoncdc", func(t *testing.T) { - err = convert.BlockExecutionDataEventPayloadsFromVersion(execDataMessage, nil) + err = convert.BlockExecutionDataEventPayloadsToVersion(execDataMessage, nil) require.NoError(t, err) for _, chunk := range execDataMessage.GetChunkExecutionData() { diff --git a/go.mod b/go.mod index 19a08493cd8..9e6a039d9f9 100644 --- a/go.mod +++ b/go.mod @@ -298,4 +298,4 @@ require ( nhooyr.io/websocket v1.8.7 // indirect ) -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231017135350-90f4b1da77c5 diff --git a/go.sum b/go.sum index 3b7e0e494af..180d297c029 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,8 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 h1:k/OJbM33BwI9sEJvNMzmkhM0ZJTSZeCoLImkBLldsJ8= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231017135350-90f4b1da77c5 h1:uz3l8+nXLwU8VKpgxyrdtODdtBjed65EkMUj3+IuElo= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231017135350-90f4b1da77c5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= diff --git a/insecure/go.mod b/insecure/go.mod index c32ea86ca31..2b9bf1feb6b 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -288,4 +288,4 @@ require ( replace github.com/onflow/flow-go => ../ -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231017135350-90f4b1da77c5 diff --git a/insecure/go.sum b/insecure/go.sum index 9f4689d484e..204415b32e7 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -87,8 +87,8 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 h1:k/OJbM33BwI9sEJvNMzmkhM0ZJTSZeCoLImkBLldsJ8= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231017135350-90f4b1da77c5 h1:uz3l8+nXLwU8VKpgxyrdtODdtBjed65EkMUj3+IuElo= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231017135350-90f4b1da77c5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= diff --git a/integration/go.mod b/integration/go.mod index 64e82bba8c6..9830208a93b 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -347,5 +347,5 @@ replace github.com/onflow/flow-go/insecure => ../insecure replace ( github.com/onflow/flow-emulator v0.53.0 => github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5 - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230915140723-432828f7afb9 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231017135350-90f4b1da77c5 ) diff --git a/integration/go.sum b/integration/go.sum index 880354f63dc..03dd6c23af3 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -102,8 +102,8 @@ github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5 h1:33av+pFXKq9WNbg3EvIFQoYuDBdTqVrzc8/XCMkGc5o= github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5/go.mod h1:gKnyRos42AIy5wm1BfHUDhhrg322F0WpIAsOU+nljj0= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5 h1:k/OJbM33BwI9sEJvNMzmkhM0ZJTSZeCoLImkBLldsJ8= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231004094238-e7eaa83befe5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231017135350-90f4b1da77c5 h1:uz3l8+nXLwU8VKpgxyrdtODdtBjed65EkMUj3+IuElo= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231017135350-90f4b1da77c5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= From 1d8ff24b92a9c90ca29c3b4657d0dc4526034fb5 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Wed, 18 Oct 2023 17:38:01 +0300 Subject: [PATCH 17/27] Fixed remarks --- access/legacy/handler.go | 8 ++- .../rest/apiproxy/rest_proxy_handler.go | 12 ++--- .../models/model_event_encoding_version.go | 19 ------- engine/access/rest/routes/events.go | 15 +++++- engine/access/rest/routes/events_test.go | 10 ++-- engine/access/rest/routes/transactions.go | 4 +- .../access/rest/routes/transactions_test.go | 11 ++--- engine/access/rpc/backend/backend_events.go | 24 +++++---- engine/access/rpc/backend/backend_test.go | 49 ++++++++----------- .../rpc/backend/backend_transactions.go | 38 ++++++++------ .../rpc/backend/backend_transactions_test.go | 16 +++--- .../rpc/backend/historical_access_test.go | 2 +- engine/access/rpc/backend/retry_test.go | 2 +- engine/access/state_stream/handler.go | 3 +- engine/access/state_stream/handler_test.go | 2 +- engine/common/rpc/convert/events.go | 10 ---- engine/common/rpc/convert/execution_data.go | 3 +- go.mod | 2 +- go.sum | 4 +- insecure/go.mod | 2 +- insecure/go.sum | 4 +- integration/go.mod | 2 +- integration/go.sum | 4 +- 23 files changed, 118 insertions(+), 128 deletions(-) delete mode 100644 engine/access/rest/models/model_event_encoding_version.go diff --git a/access/legacy/handler.go b/access/legacy/handler.go index 2c968e7183f..8a504680d01 100644 --- a/access/legacy/handler.go +++ b/access/legacy/handler.go @@ -190,7 +190,13 @@ func (h *Handler) GetTransactionResult( ) (*accessproto.TransactionResultResponse, error) { id := convert.MessageToIdentifier(req.GetId()) - result, err := h.api.GetTransactionResult(ctx, id, flow.ZeroID, flow.ZeroID, entities.EventEncodingVersion_JSON_CDC_V0) + result, err := h.api.GetTransactionResult( + ctx, + id, + flow.ZeroID, + flow.ZeroID, + entities.EventEncodingVersion_JSON_CDC_V0, + ) if err != nil { return nil, err } diff --git a/engine/access/rest/apiproxy/rest_proxy_handler.go b/engine/access/rest/apiproxy/rest_proxy_handler.go index 03ffce0b4e3..78b8b026891 100644 --- a/engine/access/rest/apiproxy/rest_proxy_handler.go +++ b/engine/access/rest/apiproxy/rest_proxy_handler.go @@ -153,7 +153,7 @@ func (r *RestProxyHandler) GetTransactionResult( id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, - eventEncodingVersion entities.EventEncodingVersion, + requiredEventEncodingVersion entities.EventEncodingVersion, ) (*access.TransactionResult, error) { upstream, err := r.FaultTolerantClient() if err != nil { @@ -165,7 +165,7 @@ func (r *RestProxyHandler) GetTransactionResult( Id: id[:], BlockId: blockID[:], CollectionId: collectionID[:], - EventEncodingVersion: eventEncodingVersion, + EventEncodingVersion: requiredEventEncodingVersion, } transactionResultResponse, err := upstream.GetTransactionResult(ctx, getTransactionResultRequest) @@ -270,7 +270,7 @@ func (r *RestProxyHandler) GetEventsForHeightRange( ctx context.Context, eventType string, startHeight, endHeight uint64, - eventEncodingVersion entities.EventEncodingVersion, + requiredEventEncodingVersion entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { upstream, err := r.FaultTolerantClient() if err != nil { @@ -281,7 +281,7 @@ func (r *RestProxyHandler) GetEventsForHeightRange( Type: eventType, StartHeight: startHeight, EndHeight: endHeight, - EventEncodingVersion: eventEncodingVersion, + EventEncodingVersion: requiredEventEncodingVersion, } eventsResponse, err := upstream.GetEventsForHeightRange(ctx, getEventsForHeightRangeRequest) r.log("upstream", "GetEventsForHeightRange", err) @@ -298,7 +298,7 @@ func (r *RestProxyHandler) GetEventsForBlockIDs( ctx context.Context, eventType string, blockIDs []flow.Identifier, - eventEncodingVersion entities.EventEncodingVersion, + requiredEventEncodingVersion entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { upstream, err := r.FaultTolerantClient() if err != nil { @@ -310,7 +310,7 @@ func (r *RestProxyHandler) GetEventsForBlockIDs( getEventsForBlockIDsRequest := &accessproto.GetEventsForBlockIDsRequest{ Type: eventType, BlockIds: blockIds, - EventEncodingVersion: eventEncodingVersion, + EventEncodingVersion: requiredEventEncodingVersion, } eventsResponse, err := upstream.GetEventsForBlockIDs(ctx, getEventsForBlockIDsRequest) r.log("upstream", "GetEventsForBlockIDs", err) diff --git a/engine/access/rest/models/model_event_encoding_version.go b/engine/access/rest/models/model_event_encoding_version.go deleted file mode 100644 index 5ca98c9ee1b..00000000000 --- a/engine/access/rest/models/model_event_encoding_version.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Access API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 1.0.0 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ -package models - -// EventEncodingVersion : This value indicates preferred events payload encoding for responses. -type EventEncodingVersion string - -// List of EventEncodingVersion -const ( - DEFAULT__EventEncodingVersion EventEncodingVersion = "Default" - JSON_CDC_V0_EventEncodingVersion EventEncodingVersion = "JsonCdcV0" - CCF_V0_EventEncodingVersion EventEncodingVersion = "CcfV0" -) diff --git a/engine/access/rest/routes/events.go b/engine/access/rest/routes/events.go index c7b66bdd8f6..59e04439bf8 100644 --- a/engine/access/rest/routes/events.go +++ b/engine/access/rest/routes/events.go @@ -23,7 +23,12 @@ func GetEvents(r *request.Request, backend access.API, _ models.LinkGenerator) ( // if the request has block IDs provided then return events for block IDs var blocksEvents models.BlocksEvents if len(req.BlockIDs) > 0 { - events, err := backend.GetEventsForBlockIDs(r.Context(), req.Type, req.BlockIDs, entitiesproto.EventEncodingVersion_DEFAULT) + events, err := backend.GetEventsForBlockIDs( + r.Context(), + req.Type, + req.BlockIDs, + entitiesproto.EventEncodingVersion_JSON_CDC_V0, + ) if err != nil { return nil, err } @@ -47,7 +52,13 @@ func GetEvents(r *request.Request, backend access.API, _ models.LinkGenerator) ( } // if request provided block height range then return events for that range - events, err := backend.GetEventsForHeightRange(r.Context(), req.Type, req.StartHeight, req.EndHeight, entitiesproto.EventEncodingVersion_DEFAULT) + events, err := backend.GetEventsForHeightRange( + r.Context(), + req.Type, + req.StartHeight, + req.EndHeight, + entitiesproto.EventEncodingVersion_JSON_CDC_V0, + ) if err != nil { return nil, err } diff --git a/engine/access/rest/routes/events_test.go b/engine/access/rest/routes/events_test.go index fa9f53a92cd..93b58afda6d 100644 --- a/engine/access/rest/routes/events_test.go +++ b/engine/access/rest/routes/events_test.go @@ -168,14 +168,14 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { events[i] = unittest.BlockEventsFixture(header, 2) backend.Mock. - On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, []flow.Identifier{header.ID()}, entities.EventEncodingVersion_DEFAULT). + On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, []flow.Identifier{header.ID()}, entities.EventEncodingVersion_JSON_CDC_V0). Return([]flow.BlockEvents{events[i]}, nil) lastHeader = header } backend.Mock. - On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, ids, entities.EventEncodingVersion_DEFAULT). + On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, ids, entities.EventEncodingVersion_JSON_CDC_V0). Return(events, nil) // range from first to last block @@ -185,7 +185,7 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { mocks.Anything, events[0].BlockHeight, events[len(events)-1].BlockHeight, - entities.EventEncodingVersion_DEFAULT, + entities.EventEncodingVersion_JSON_CDC_V0, ).Return(events, nil) // range from first to last block + 5 @@ -195,7 +195,7 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { mocks.Anything, events[0].BlockHeight, events[len(events)-1].BlockHeight+5, - entities.EventEncodingVersion_DEFAULT, + entities.EventEncodingVersion_JSON_CDC_V0, ).Return(append(events[:len(events)-1], unittest.BlockEventsFixture(lastHeader, 0)), nil) latestBlock := unittest.BlockHeaderFixture() @@ -203,7 +203,7 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { // default not found backend.Mock. - On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, mocks.Anything, entities.EventEncodingVersion_DEFAULT). + On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, mocks.Anything, entities.EventEncodingVersion_JSON_CDC_V0). Return(nil, status.Error(codes.NotFound, "not found")) backend.Mock. diff --git a/engine/access/rest/routes/transactions.go b/engine/access/rest/routes/transactions.go index 30a57bd5054..e85eb116894 100644 --- a/engine/access/rest/routes/transactions.go +++ b/engine/access/rest/routes/transactions.go @@ -28,7 +28,7 @@ func GetTransactionByID(r *request.Request, backend access.API, link models.Link req.ID, req.BlockID, req.CollectionID, - entitiesproto.EventEncodingVersion_DEFAULT, + entitiesproto.EventEncodingVersion_JSON_CDC_V0, ) if err != nil { return nil, err @@ -52,7 +52,7 @@ func GetTransactionResultByID(r *request.Request, backend access.API, link model req.ID, req.BlockID, req.CollectionID, - entitiesproto.EventEncodingVersion_DEFAULT, + entitiesproto.EventEncodingVersion_JSON_CDC_V0, ) if err != nil { return nil, err diff --git a/engine/access/rest/routes/transactions_test.go b/engine/access/rest/routes/transactions_test.go index 865e9b4fdde..63b347c8b37 100644 --- a/engine/access/rest/routes/transactions_test.go +++ b/engine/access/rest/routes/transactions_test.go @@ -120,7 +120,6 @@ func TestGetTransactions(t *testing.T) { t.Run("Get by ID with results", func(t *testing.T) { backend := &mock.API{} - var emptyEventEncodingVersion entities.EventEncodingVersion tx := unittest.TransactionFixture() txr := transactionResultFixture(tx) @@ -129,7 +128,7 @@ func TestGetTransactions(t *testing.T) { Return(&tx.TransactionBody, nil) backend.Mock. - On("GetTransactionResult", mocks.Anything, tx.ID(), flow.ZeroID, flow.ZeroID, emptyEventEncodingVersion). + On("GetTransactionResult", mocks.Anything, tx.ID(), flow.ZeroID, flow.ZeroID, entities.EventEncodingVersion_JSON_CDC_V0). Return(txr, nil) req := getTransactionReq(tx.ID().String(), true, "", "") @@ -253,7 +252,7 @@ func TestGetTransactionResult(t *testing.T) { req := getTransactionResultReq(id.String(), "", "") backend.Mock. - On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID, entities.EventEncodingVersion_DEFAULT). + On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID, entities.EventEncodingVersion_JSON_CDC_V0). Return(txr, nil) assertOKResponse(t, req, expected, backend) @@ -265,7 +264,7 @@ func TestGetTransactionResult(t *testing.T) { req := getTransactionResultReq(id.String(), bid.String(), "") backend.Mock. - On("GetTransactionResult", mocks.Anything, id, bid, flow.ZeroID, entities.EventEncodingVersion_DEFAULT). + On("GetTransactionResult", mocks.Anything, id, bid, flow.ZeroID, entities.EventEncodingVersion_JSON_CDC_V0). Return(txr, nil) assertOKResponse(t, req, expected, backend) @@ -276,7 +275,7 @@ func TestGetTransactionResult(t *testing.T) { req := getTransactionResultReq(id.String(), "", cid.String()) backend.Mock. - On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, cid, entities.EventEncodingVersion_DEFAULT). + On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, cid, entities.EventEncodingVersion_JSON_CDC_V0). Return(txr, nil) assertOKResponse(t, req, expected, backend) @@ -310,7 +309,7 @@ func TestGetTransactionResult(t *testing.T) { txResult.CollectionID = cid req := getTransactionResultReq(id.String(), "", "") backend.Mock. - On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID, entities.EventEncodingVersion_DEFAULT). + On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID, entities.EventEncodingVersion_JSON_CDC_V0). Return(txResult, nil). Once() diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index 3b31df4c555..fc8f935343e 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -38,7 +38,7 @@ func (b *backendEvents) GetEventsForHeightRange( ctx context.Context, eventType string, startHeight, endHeight uint64, - eventEncodingVersion entities.EventEncodingVersion, + requiredEventEncodingVersion entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { if endHeight < startHeight { @@ -80,7 +80,7 @@ func (b *backendEvents) GetEventsForHeightRange( blockHeaders = append(blockHeaders, header) } - return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, eventEncodingVersion) + return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, requiredEventEncodingVersion) } // GetEventsForBlockIDs retrieves events for all the specified block IDs that have the given type @@ -88,7 +88,7 @@ func (b *backendEvents) GetEventsForBlockIDs( ctx context.Context, eventType string, blockIDs []flow.Identifier, - eventEncodingVersion entities.EventEncodingVersion, + requiredEventEncodingVersion entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { if uint(len(blockIDs)) > b.maxHeightRange { @@ -107,14 +107,14 @@ func (b *backendEvents) GetEventsForBlockIDs( } // forward the request to the execution node - return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, eventEncodingVersion) + return b.getBlockEventsFromExecutionNode(ctx, blockHeaders, eventType, requiredEventEncodingVersion) } func (b *backendEvents) getBlockEventsFromExecutionNode( ctx context.Context, blockHeaders []*flow.Header, eventType string, - eventEncodingVersion entities.EventEncodingVersion, + requiredEventEncodingVersion entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { // create an execution API request for events at block ID @@ -157,7 +157,8 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( results, err := verifyAndConvertToAccessEvents( resp.GetResults(), blockHeaders, - convert.GetConversionEventEncodingVersion(eventEncodingVersion), + resp.GetEventEncodingVersion(), + requiredEventEncodingVersion, ) if err != nil { return nil, status.Errorf(codes.Internal, "failed to verify retrieved events from execution node: %v", err) @@ -171,12 +172,17 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( func verifyAndConvertToAccessEvents( execEvents []*execproto.GetEventsForBlockIDsResponse_Result, requestedBlockHeaders []*flow.Header, - eventEncodingVersion entities.EventEncodingVersion, + from entities.EventEncodingVersion, + to entities.EventEncodingVersion, ) ([]flow.BlockEvents, error) { if len(execEvents) != len(requestedBlockHeaders) { return nil, errors.New("number of results does not match number of blocks requested") } + if from == entities.EventEncodingVersion_JSON_CDC_V0 && to == entities.EventEncodingVersion_CCF_V0 { + return nil, errors.New("conversion from JSON-CDC to CCF is forbidden") + } + requestedBlockHeaderSet := map[string]*flow.Header{} for _, header := range requestedBlockHeaders { requestedBlockHeaderSet[header.ID().String()] = header @@ -195,10 +201,10 @@ func verifyAndConvertToAccessEvents( result.GetBlockId()) } - events, err := convert.MessagesToEventsFromVersion(result.GetEvents(), eventEncodingVersion) + events, err := convert.MessagesToEventsFromVersion(result.GetEvents(), to) if err != nil { return nil, fmt.Errorf("failed to unmarshal events in event %d with encoding version %s: %w", - i, eventEncodingVersion.String(), err) + i, to.String(), err) } results[i] = flow.BlockEvents{ diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 7c93080336f..bf0caae3a38 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -512,7 +512,7 @@ func (suite *Suite) TestGetTransactionResultByIndex() { Return(exeEventResp, nil). Once() - result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, entitiesproto.EventEncodingVersion_DEFAULT) + result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(result.BlockHeight, block.Header.Height) @@ -562,7 +562,7 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { Return(exeEventResp, nil). Once() - result, err := backend.GetTransactionResultsByBlockID(ctx, blockId, entitiesproto.EventEncodingVersion_DEFAULT) + result, err := backend.GetTransactionResultsByBlockID(ctx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.assertAllExpectations() @@ -641,7 +641,7 @@ func (suite *Suite) TestTransactionStatusTransition() { Times(len(fixedENIDs)) // should call each EN once // first call - when block under test is greater height than the sealed head, but execution node does not know about Tx - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_DEFAULT) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) // status should be finalized since the sealed Blocks is smaller in height @@ -656,7 +656,7 @@ func (suite *Suite) TestTransactionStatusTransition() { Return(exeEventResp, nil) // second call - when block under test's height is greater height than the sealed head - result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_DEFAULT) + result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) // status should be executed since no `NotFound` error in the `GetTransactionResult` call @@ -666,7 +666,7 @@ func (suite *Suite) TestTransactionStatusTransition() { headBlock.Header.Height = block.Header.Height + 1 // third call - when block under test's height is less than sealed head's height - result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_DEFAULT) + result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) // status should be sealed since the sealed Blocks is greater in height @@ -677,7 +677,7 @@ func (suite *Suite) TestTransactionStatusTransition() { // fourth call - when block under test's height so much less than the head's height that it's considered expired, // but since there is a execution result, means it should retain it's sealed status - result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_DEFAULT) + result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) // status should be expired since @@ -740,7 +740,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // should return pending status when we have not observed an expiry block suite.Run("pending", func() { // referenced block isn't known yet, so should return pending status - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_DEFAULT) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) @@ -756,7 +756,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // we have NOT observed all intermediary Collections fullHeight = block.Header.Height + flow.DefaultTransactionExpiry/2 - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_DEFAULT) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) }) @@ -766,7 +766,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // we have observed all intermediary Collections fullHeight = block.Header.Height + flow.DefaultTransactionExpiry + 1 - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_DEFAULT) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) }) @@ -781,7 +781,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // we have observed all intermediary Collections fullHeight = block.Header.Height + flow.DefaultTransactionExpiry + 1 - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_DEFAULT) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusExpired, result.Status) }) @@ -894,7 +894,7 @@ func (suite *Suite) TestTransactionPendingToFinalizedStatusTransition() { // should return pending status when we have not observed collection for the transaction suite.Run("pending", func() { currentState = flow.TransactionStatusPending - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_DEFAULT) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) // assert that no call to an execution node is made @@ -905,7 +905,7 @@ func (suite *Suite) TestTransactionPendingToFinalizedStatusTransition() { // preceding sealed refBlock) suite.Run("finalized", func() { currentState = flow.TransactionStatusFinalized - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_DEFAULT) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusFinalized, result.Status) }) @@ -931,7 +931,7 @@ func (suite *Suite) TestTransactionResultUnknown() { suite.Require().NoError(err) // first call - when block under test is greater height than the sealed head, but execution node does not know about Tx - result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_DEFAULT) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) // status should be reported as unknown @@ -1088,7 +1088,7 @@ func (suite *Suite) TestGetEventsForBlockIDs() { ctx, string(flow.EventAccountCreated), blockIDs, - entitiesproto.EventEncodingVersion_DEFAULT, + entitiesproto.EventEncodingVersion_JSON_CDC_V0, ) suite.checkResponse(actual, err) @@ -1111,7 +1111,7 @@ func (suite *Suite) TestGetEventsForBlockIDs() { ctx, string(flow.EventAccountCreated), []flow.Identifier{}, - entitiesproto.EventEncodingVersion_DEFAULT, + entitiesproto.EventEncodingVersion_JSON_CDC_V0, ) require.NoError(suite.T(), err) require.Empty(suite.T(), resp) @@ -1371,7 +1371,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { string(flow.EventAccountCreated), maxHeight, minHeight, - entitiesproto.EventEncodingVersion_DEFAULT, + entitiesproto.EventEncodingVersion_JSON_CDC_V0, ) suite.Require().Error(err) @@ -1407,7 +1407,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { string(flow.EventAccountCreated), minHeight, maxHeight, - entitiesproto.EventEncodingVersion_DEFAULT, + entitiesproto.EventEncodingVersion_JSON_CDC_V0, ) // check response @@ -1441,7 +1441,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { string(flow.EventAccountCreated), minHeight, maxHeight, - entitiesproto.EventEncodingVersion_DEFAULT, + entitiesproto.EventEncodingVersion_JSON_CDC_V0, ) suite.checkResponse(actualResp, err) @@ -1469,7 +1469,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { string(flow.EventAccountCreated), minHeight, minHeight+1, - entitiesproto.EventEncodingVersion_DEFAULT, + entitiesproto.EventEncodingVersion_JSON_CDC_V0, ) suite.Require().Error(err) }) @@ -1497,7 +1497,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { string(flow.EventAccountCreated), minHeight, maxHeight, - entitiesproto.EventEncodingVersion_DEFAULT, + entitiesproto.EventEncodingVersion_JSON_CDC_V0, ) suite.Require().Error(err) }) @@ -2203,9 +2203,7 @@ func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { exeEventResp *execproto.GetTransactionResultResponse, encodingVersion entitiesproto.EventEncodingVersion, ) { - requestedEncodingVersion := convert.GetConversionEventEncodingVersion(encodingVersion) - - result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, requestedEncodingVersion) + result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, encodingVersion) suite.checkResponse(result, err) expectedResultEvents, err := convert.MessagesToEventsFromVersion(exeEventResp.GetEvents(), encodingVersion) @@ -2213,11 +2211,6 @@ func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { suite.Assert().Equal(result.Events, expectedResultEvents) } - suite.Run("test default(JSON) event encoding", func() { - exeEventResp := prepareGetTransactionResultByIndex() - assertResultExpectations(exeEventResp, entitiesproto.EventEncodingVersion_DEFAULT) - }) - suite.Run("test JSON event encoding", func() { exeEventResp := prepareGetTransactionResultByIndex() assertResultExpectations(exeEventResp, entitiesproto.EventEncodingVersion_JSON_CDC_V0) diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index a3a150cc066..68c707d0e04 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -228,7 +228,7 @@ func (b *backendTransactions) GetTransactionResult( txID flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, - eventEncodingVersion entities.EventEncodingVersion, + requiredEventEncodingVersion entities.EventEncodingVersion, ) (*access.TransactionResult, error) { // look up transaction from storage start := time.Now() @@ -287,7 +287,7 @@ func (b *backendTransactions) GetTransactionResult( // access node may not have the block if it hasn't yet been finalized, hence block can be nil at this point if block != nil { foundBlockID := block.ID() - transactionWasExecuted, events, statusCode, txError, err = b.lookupTransactionResult(ctx, txID, foundBlockID, eventEncodingVersion) + transactionWasExecuted, events, statusCode, txError, err = b.lookupTransactionResult(ctx, txID, foundBlockID, requiredEventEncodingVersion) if err != nil { return nil, rpc.ConvertError(err, "failed to retrieve result from any execution node", codes.Internal) } @@ -386,7 +386,7 @@ func (b *backendTransactions) retrieveBlock( func (b *backendTransactions) GetTransactionResultsByBlockID( ctx context.Context, blockID flow.Identifier, - eventEncodingVersion entities.EventEncodingVersion, + requiredEventEncodingVersion entities.EventEncodingVersion, ) ([]*access.TransactionResult, error) { // TODO: consider using storage.Index.ByBlockID, the index contains collection id and seals ID block, err := b.blocks.ByID(blockID) @@ -411,6 +411,10 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( return nil, rpc.ConvertError(err, "failed to retrieve result from execution node", codes.Internal) } + if resp.GetEventEncodingVersion() == entities.EventEncodingVersion_JSON_CDC_V0 && requiredEventEncodingVersion == entities.EventEncodingVersion_CCF_V0 { + return nil, errors.New("conversion from JSON-CDC to CCF is forbidden") + } + results := make([]*access.TransactionResult, 0, len(resp.TransactionResults)) i := 0 errInsufficientResults := status.Errorf( @@ -418,8 +422,6 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( "number of transaction results returned by execution node is less than the number of transactions in the block", ) - requestedEventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersion) - for _, guarantee := range block.Payload.Guarantees { collection, err := b.collections.LightByID(guarantee.CollectionID) if err != nil { @@ -438,7 +440,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( if err != nil { return nil, rpc.ConvertStorageError(err) } - events, err := convert.MessagesToEventsFromVersion(txResult.GetEvents(), requestedEventEncodingVersion) + events, err := convert.MessagesToEventsFromVersion(txResult.GetEvents(), requiredEventEncodingVersion) if err != nil { return nil, status.Errorf(codes.Internal, "failed to convert events to message in txID %x: %v", txID, err) @@ -493,7 +495,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( return nil, rpc.ConvertStorageError(err) } - events, err := convert.MessagesToEventsFromVersion(systemTxResult.GetEvents(), requestedEventEncodingVersion) + events, err := convert.MessagesToEventsFromVersion(systemTxResult.GetEvents(), requiredEventEncodingVersion) if err != nil { return nil, rpc.ConvertError(err, "failed to convert events from system tx result", codes.Internal) } @@ -518,7 +520,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( ctx context.Context, blockID flow.Identifier, index uint32, - eventEncodingVersion entities.EventEncodingVersion, + requiredEventEncodingVersion entities.EventEncodingVersion, ) (*access.TransactionResult, error) { // TODO: https://github.com/onflow/flow-go/issues/2175 so caching doesn't cause a circular dependency block, err := b.blocks.ByID(blockID) @@ -545,15 +547,17 @@ func (b *backendTransactions) GetTransactionResultByIndex( return nil, rpc.ConvertError(err, "failed to retrieve result from execution node", codes.Internal) } + if resp.GetEventEncodingVersion() == entities.EventEncodingVersion_JSON_CDC_V0 && requiredEventEncodingVersion == entities.EventEncodingVersion_CCF_V0 { + return nil, errors.New("conversion from JSON-CDC to CCF is forbidden") + } + // tx body is irrelevant to status if it's in an executed block txStatus, err := b.deriveTransactionStatus(nil, true, block) if err != nil { return nil, rpc.ConvertStorageError(err) } - requestedEventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersion) - - events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), requestedEventEncodingVersion) + events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), requiredEventEncodingVersion) if err != nil { return nil, status.Errorf(codes.Internal, "failed to convert events in blockID %x: %v", blockID, err) } @@ -670,9 +674,9 @@ func (b *backendTransactions) lookupTransactionResult( ctx context.Context, txID flow.Identifier, blockID flow.Identifier, - eventEncodingVersion entities.EventEncodingVersion, + requiredEventEncodingVersion entities.EventEncodingVersion, ) (bool, []flow.Event, uint32, string, error) { - events, txStatus, message, err := b.getTransactionResultFromExecutionNode(ctx, blockID, txID[:], eventEncodingVersion) + events, txStatus, message, err := b.getTransactionResultFromExecutionNode(ctx, blockID, txID[:], requiredEventEncodingVersion) if err != nil { // if either the execution node reported no results or the execution node could not be chosen if status.Code(err) == codes.NotFound { @@ -755,7 +759,7 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( ctx context.Context, blockID flow.Identifier, transactionID []byte, - eventEncodingVersion entities.EventEncodingVersion, + requiredEventEncodingVersion entities.EventEncodingVersion, ) ([]flow.Event, uint32, string, error) { // create an execution API request for events at blockID and transactionID @@ -778,9 +782,11 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return nil, 0, "", err } - requestedEventEncodingVersion := convert.GetConversionEventEncodingVersion(eventEncodingVersion) + if resp.GetEventEncodingVersion() == entities.EventEncodingVersion_JSON_CDC_V0 && requiredEventEncodingVersion == entities.EventEncodingVersion_CCF_V0 { + return nil, 0, "", rpc.ConvertError(err, "conversion from JSON-CDC to CCF is forbidden", codes.Internal) + } - events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), requestedEventEncodingVersion) + events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), requiredEventEncodingVersion) if err != nil { return nil, 0, "", rpc.ConvertError(err, "failed to convert events to message", codes.Internal) } diff --git a/engine/access/rpc/backend/backend_transactions_test.go b/engine/access/rpc/backend/backend_transactions_test.go index a05dc655912..28d290ff623 100644 --- a/engine/access/rpc/backend/backend_transactions_test.go +++ b/engine/access/rpc/backend/backend_transactions_test.go @@ -79,7 +79,7 @@ func (suite *Suite) TestGetTransactionResultReturnsUnknown() { tx.ID(), block.ID(), coll.ID(), - entities.EventEncodingVersion_DEFAULT, + entities.EventEncodingVersion_JSON_CDC_V0, ) suite.Require().NoError(err) suite.Require().Equal(res.Status, flow.TransactionStatusUnknown) @@ -118,7 +118,7 @@ func (suite *Suite) TestGetTransactionResultReturnsTransactionError() { tx.ID(), block.ID(), coll.ID(), - entities.EventEncodingVersion_DEFAULT, + entities.EventEncodingVersion_JSON_CDC_V0, ) suite.Require().Equal(err, status.Errorf(codes.Internal, "failed to find: %v", fmt.Errorf("some other error"))) @@ -162,7 +162,7 @@ func (suite *Suite) TestGetTransactionResultReturnsValidTransactionResultFromHis tx.ID(), block.ID(), coll.ID(), - entities.EventEncodingVersion_DEFAULT, + entities.EventEncodingVersion_JSON_CDC_V0, ) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusExecuted, resp.Status) @@ -214,7 +214,7 @@ func (suite *Suite) TestGetTransactionResultFromCache() { tx.ID(), block.ID(), coll.ID(), - entities.EventEncodingVersion_DEFAULT, + entities.EventEncodingVersion_JSON_CDC_V0, ) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusExecuted, resp.Status) @@ -225,7 +225,7 @@ func (suite *Suite) TestGetTransactionResultFromCache() { tx.ID(), block.ID(), coll.ID(), - entities.EventEncodingVersion_DEFAULT, + entities.EventEncodingVersion_JSON_CDC_V0, ) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusExecuted, resp2.Status) @@ -258,7 +258,7 @@ func (suite *Suite) TestGetTransactionResultCacheNonExistent() { tx.ID(), block.ID(), coll.ID(), - entities.EventEncodingVersion_DEFAULT, + entities.EventEncodingVersion_JSON_CDC_V0, ) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusUnknown, resp.Status) @@ -299,7 +299,7 @@ func (suite *Suite) TestGetTransactionResultUnknownFromCache() { tx.ID(), block.ID(), coll.ID(), - entities.EventEncodingVersion_DEFAULT, + entities.EventEncodingVersion_JSON_CDC_V0, ) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusUnknown, resp.Status) @@ -319,7 +319,7 @@ func (suite *Suite) TestGetTransactionResultUnknownFromCache() { tx.ID(), block.ID(), coll.ID(), - entities.EventEncodingVersion_DEFAULT, + entities.EventEncodingVersion_JSON_CDC_V0, ) suite.Require().NoError(err) suite.Require().Equal(flow.TransactionStatusUnknown, resp2.Status) diff --git a/engine/access/rpc/backend/historical_access_test.go b/engine/access/rpc/backend/historical_access_test.go index 3bc9c4b6aa1..a776d8337e8 100644 --- a/engine/access/rpc/backend/historical_access_test.go +++ b/engine/access/rpc/backend/historical_access_test.go @@ -53,7 +53,7 @@ func (suite *Suite) TestHistoricalTransactionResult() { txID, flow.ZeroID, flow.ZeroID, - entities.EventEncodingVersion_DEFAULT, + entities.EventEncodingVersion_JSON_CDC_V0, ) suite.checkResponse(result, err) diff --git a/engine/access/rpc/backend/retry_test.go b/engine/access/rpc/backend/retry_test.go index 83c848d8bf5..01874185bbf 100644 --- a/engine/access/rpc/backend/retry_test.go +++ b/engine/access/rpc/backend/retry_test.go @@ -137,7 +137,7 @@ func (suite *Suite) TestSuccessfulTransactionsDontRetry() { txID, flow.ZeroID, flow.ZeroID, - entities.EventEncodingVersion_DEFAULT, + entities.EventEncodingVersion_JSON_CDC_V0, ) suite.checkResponse(result, err) diff --git a/engine/access/state_stream/handler.go b/engine/access/state_stream/handler.go index a8982eb3d0a..2c6b471f25e 100644 --- a/engine/access/state_stream/handler.go +++ b/engine/access/state_stream/handler.go @@ -160,8 +160,7 @@ func (h *Handler) SubscribeEvents(request *executiondata.SubscribeEventsRequest, return status.Errorf(codes.Internal, "unexpected response type: %T", v) } - eventEncodingVersion := convert.GetConversionEventEncodingVersion(request.GetEventEncodingVersion()) - events, err := convert.EventsToMessagesFromVersion(resp.Events, eventEncodingVersion) + events, err := convert.EventsToMessagesFromVersion(resp.Events, request.GetEventEncodingVersion()) if err != nil { return status.Errorf(codes.Internal, "could not convert events to entity: %v", err) } diff --git a/engine/access/state_stream/handler_test.go b/engine/access/state_stream/handler_test.go index 792c02307b1..fe3ef9edad7 100644 --- a/engine/access/state_stream/handler_test.go +++ b/engine/access/state_stream/handler_test.go @@ -171,7 +171,7 @@ func TestExecutionDataStreamEventEncoding(t *testing.T) { // Test scenario for default (JSON) event encoding. t.Run("test default(JSON)", func(t *testing.T) { makeStreamRequest(&executiondata.SubscribeExecutionDataRequest{ - EventEncodingVersion: entities.EventEncodingVersion_DEFAULT, + EventEncodingVersion: entities.EventEncodingVersion_JSON_CDC_V0, }) for { resp, err := stream.RecvToClient() diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index eac58518d1e..f5bd7d8a6ee 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -261,13 +261,3 @@ func BlockEventsToMessage(block flow.BlockEvents) (*accessproto.EventsResponse_R Events: eventMessages, }, nil } - -// GetConversionEventEncodingVersion returns the appropriate event encoding version for conversion. -func GetConversionEventEncodingVersion(requestedEventEncodingVersion entities.EventEncodingVersion) entities.EventEncodingVersion { - // 1. Check the requested version of payloads. - if requestedEventEncodingVersion == entities.EventEncodingVersion_DEFAULT { - return entities.EventEncodingVersion_JSON_CDC_V0 - } - - return requestedEventEncodingVersion -} diff --git a/engine/common/rpc/convert/execution_data.go b/engine/common/rpc/convert/execution_data.go index 64527a08393..4eee13f1eb1 100644 --- a/engine/common/rpc/convert/execution_data.go +++ b/engine/common/rpc/convert/execution_data.go @@ -18,8 +18,7 @@ func BlockExecutionDataEventPayloadsToVersion( m *entities.BlockExecutionData, to entities.EventEncodingVersion, ) error { - eventEncodingVersion := GetConversionEventEncodingVersion(to) - if eventEncodingVersion == entities.EventEncodingVersion_CCF_V0 { + if to == entities.EventEncodingVersion_CCF_V0 { return nil } diff --git a/go.mod b/go.mod index dce90543ff3..57bfc7d3c28 100644 --- a/go.mod +++ b/go.mod @@ -298,4 +298,4 @@ require ( nhooyr.io/websocket v1.8.7 // indirect ) -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018101322-3bc2a445e6ce +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d diff --git a/go.sum b/go.sum index 31e6fcf5762..27fed15a201 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,8 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018101322-3bc2a445e6ce h1:eRdcPnVSUa+7OF1RNhVOWz4orOoLdzn7h6vbQoyeVrw= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018101322-3bc2a445e6ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d h1:kiaxw8rcUs1gbmVV4M04txn6VmKvaqKq3mmdNNmq3i0= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= diff --git a/insecure/go.mod b/insecure/go.mod index 545ee8fbc50..d03c5533554 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -288,4 +288,4 @@ require ( replace github.com/onflow/flow-go => ../ -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018101322-3bc2a445e6ce +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d diff --git a/insecure/go.sum b/insecure/go.sum index d3e94b2f6c8..40455527cdb 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -87,8 +87,8 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018101322-3bc2a445e6ce h1:eRdcPnVSUa+7OF1RNhVOWz4orOoLdzn7h6vbQoyeVrw= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018101322-3bc2a445e6ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d h1:kiaxw8rcUs1gbmVV4M04txn6VmKvaqKq3mmdNNmq3i0= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= diff --git a/integration/go.mod b/integration/go.mod index 875c6c1998f..c01282d84a0 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -347,5 +347,5 @@ replace github.com/onflow/flow-go/insecure => ../insecure replace ( github.com/onflow/flow-emulator v0.53.0 => github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5 - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018101322-3bc2a445e6ce + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d ) diff --git a/integration/go.sum b/integration/go.sum index 731f82472fa..1050082eee7 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -102,8 +102,8 @@ github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5 h1:33av+pFXKq9WNbg3EvIFQoYuDBdTqVrzc8/XCMkGc5o= github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5/go.mod h1:gKnyRos42AIy5wm1BfHUDhhrg322F0WpIAsOU+nljj0= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018101322-3bc2a445e6ce h1:eRdcPnVSUa+7OF1RNhVOWz4orOoLdzn7h6vbQoyeVrw= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018101322-3bc2a445e6ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d h1:kiaxw8rcUs1gbmVV4M04txn6VmKvaqKq3mmdNNmq3i0= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= From 3e80a49914ad59a76921af1d87b8c680a0c0adfc Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Wed, 18 Oct 2023 18:11:39 +0300 Subject: [PATCH 18/27] fixed protobuf version --- go.mod | 2 +- go.sum | 4 ++-- insecure/go.mod | 2 +- insecure/go.sum | 4 ++-- integration/go.mod | 2 +- integration/go.sum | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 57bfc7d3c28..5805bf76e93 100644 --- a/go.mod +++ b/go.mod @@ -298,4 +298,4 @@ require ( nhooyr.io/websocket v1.8.7 // indirect ) -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 diff --git a/go.sum b/go.sum index 27fed15a201..a48c6baae83 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,8 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d h1:kiaxw8rcUs1gbmVV4M04txn6VmKvaqKq3mmdNNmq3i0= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 h1:6Szrh+jL5X83OiKh76ORt7QHxtZvM/uRaU5G6zd6syc= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= diff --git a/insecure/go.mod b/insecure/go.mod index d03c5533554..c131a947251 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -288,4 +288,4 @@ require ( replace github.com/onflow/flow-go => ../ -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 diff --git a/insecure/go.sum b/insecure/go.sum index 40455527cdb..d10c4b6ef33 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -87,8 +87,8 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d h1:kiaxw8rcUs1gbmVV4M04txn6VmKvaqKq3mmdNNmq3i0= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 h1:6Szrh+jL5X83OiKh76ORt7QHxtZvM/uRaU5G6zd6syc= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= diff --git a/integration/go.mod b/integration/go.mod index c01282d84a0..ff29aedfeb0 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -347,5 +347,5 @@ replace github.com/onflow/flow-go/insecure => ../insecure replace ( github.com/onflow/flow-emulator v0.53.0 => github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5 - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 ) diff --git a/integration/go.sum b/integration/go.sum index 1050082eee7..7c3fde1c53c 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -102,8 +102,8 @@ github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5 h1:33av+pFXKq9WNbg3EvIFQoYuDBdTqVrzc8/XCMkGc5o= github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5/go.mod h1:gKnyRos42AIy5wm1BfHUDhhrg322F0WpIAsOU+nljj0= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d h1:kiaxw8rcUs1gbmVV4M04txn6VmKvaqKq3mmdNNmq3i0= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018142143-4d2222e8b26d/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 h1:6Szrh+jL5X83OiKh76ORt7QHxtZvM/uRaU5G6zd6syc= +github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= From 423a81d7ec6ae4e45feb703cd5e9244b2f4b2781 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Wed, 18 Oct 2023 20:51:00 +0300 Subject: [PATCH 19/27] Change logic of cconverter functions. Fixed trests. --- engine/access/rpc/backend/backend_events.go | 6 +-- engine/access/rpc/backend/backend_test.go | 26 +++++----- .../rpc/backend/backend_transactions.go | 20 ++----- engine/access/state_stream/handler.go | 3 +- engine/access/state_stream/handler_test.go | 30 ++--------- engine/common/rpc/convert/events.go | 52 ++++++++++++++----- engine/common/rpc/convert/events_test.go | 19 ++++++- integration/go.mod | 2 +- integration/go.sum | 4 +- 9 files changed, 82 insertions(+), 80 deletions(-) diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index fc8f935343e..6db5efc25f7 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -179,10 +179,6 @@ func verifyAndConvertToAccessEvents( return nil, errors.New("number of results does not match number of blocks requested") } - if from == entities.EventEncodingVersion_JSON_CDC_V0 && to == entities.EventEncodingVersion_CCF_V0 { - return nil, errors.New("conversion from JSON-CDC to CCF is forbidden") - } - requestedBlockHeaderSet := map[string]*flow.Header{} for _, header := range requestedBlockHeaders { requestedBlockHeaderSet[header.ID().String()] = header @@ -201,7 +197,7 @@ func verifyAndConvertToAccessEvents( result.GetBlockId()) } - events, err := convert.MessagesToEventsFromVersion(result.GetEvents(), to) + events, err := convert.MessagesToEventsWithEncodingConversion(result.GetEvents(), from, to) if err != nil { return nil, fmt.Errorf("failed to unmarshal events in event %d with encoding version %s: %w", i, to.String(), err) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index bf0caae3a38..cc693bc4d29 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -2182,43 +2182,45 @@ func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { Index: index, } + defaultEncoding := entitiesproto.EventEncodingVersion_CCF_V0 + // Define a helper function to prepare the GetTransactionResultResponse with a given encoding version. - prepareGetTransactionResultByIndex := func() *execproto.GetTransactionResultResponse { - events := getEventsWithEncoding(1, entitiesproto.EventEncodingVersion_CCF_V0) + prepareGetTransactionResultByIndex := func() { + events := getEventsWithEncoding(1, defaultEncoding) exeEventResp := &execproto.GetTransactionResultResponse{ - Events: convert.EventsToMessages(events), + Events: convert.EventsToMessages(events), + EventEncodingVersion: entitiesproto.EventEncodingVersion_CCF_V0, } suite.execClient. On("GetTransactionResultByIndex", ctx, exeEventReq). Return(exeEventResp, nil). Once() - - return exeEventResp } // Define a helper function to assert the result expectations. assertResultExpectations := func( - exeEventResp *execproto.GetTransactionResultResponse, + expectedResult []flow.Event, encodingVersion entitiesproto.EventEncodingVersion, ) { result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, encodingVersion) suite.checkResponse(result, err) - expectedResultEvents, err := convert.MessagesToEventsFromVersion(exeEventResp.GetEvents(), encodingVersion) suite.Require().NoError(err) - suite.Assert().Equal(result.Events, expectedResultEvents) + suite.Assert().Equal(result.Events, expectedResult) } suite.Run("test JSON event encoding", func() { - exeEventResp := prepareGetTransactionResultByIndex() - assertResultExpectations(exeEventResp, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + prepareGetTransactionResultByIndex() + expectedResult := getEventsWithEncoding(1, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + assertResultExpectations(expectedResult, entitiesproto.EventEncodingVersion_JSON_CDC_V0) }) suite.Run("test CFF event encoding", func() { - exeEventResp := prepareGetTransactionResultByIndex() - assertResultExpectations(exeEventResp, entitiesproto.EventEncodingVersion_CCF_V0) + prepareGetTransactionResultByIndex() + expectedResult := getEventsWithEncoding(1, entitiesproto.EventEncodingVersion_CCF_V0) + assertResultExpectations(expectedResult, entitiesproto.EventEncodingVersion_CCF_V0) }) } diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 68c707d0e04..b9c8f055708 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -411,10 +411,6 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( return nil, rpc.ConvertError(err, "failed to retrieve result from execution node", codes.Internal) } - if resp.GetEventEncodingVersion() == entities.EventEncodingVersion_JSON_CDC_V0 && requiredEventEncodingVersion == entities.EventEncodingVersion_CCF_V0 { - return nil, errors.New("conversion from JSON-CDC to CCF is forbidden") - } - results := make([]*access.TransactionResult, 0, len(resp.TransactionResults)) i := 0 errInsufficientResults := status.Errorf( @@ -440,7 +436,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( if err != nil { return nil, rpc.ConvertStorageError(err) } - events, err := convert.MessagesToEventsFromVersion(txResult.GetEvents(), requiredEventEncodingVersion) + events, err := convert.MessagesToEventsWithEncodingConversion(txResult.GetEvents(), resp.GetEventEncodingVersion(), requiredEventEncodingVersion) if err != nil { return nil, status.Errorf(codes.Internal, "failed to convert events to message in txID %x: %v", txID, err) @@ -495,7 +491,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( return nil, rpc.ConvertStorageError(err) } - events, err := convert.MessagesToEventsFromVersion(systemTxResult.GetEvents(), requiredEventEncodingVersion) + events, err := convert.MessagesToEventsWithEncodingConversion(systemTxResult.GetEvents(), resp.GetEventEncodingVersion(), requiredEventEncodingVersion) if err != nil { return nil, rpc.ConvertError(err, "failed to convert events from system tx result", codes.Internal) } @@ -547,17 +543,13 @@ func (b *backendTransactions) GetTransactionResultByIndex( return nil, rpc.ConvertError(err, "failed to retrieve result from execution node", codes.Internal) } - if resp.GetEventEncodingVersion() == entities.EventEncodingVersion_JSON_CDC_V0 && requiredEventEncodingVersion == entities.EventEncodingVersion_CCF_V0 { - return nil, errors.New("conversion from JSON-CDC to CCF is forbidden") - } - // tx body is irrelevant to status if it's in an executed block txStatus, err := b.deriveTransactionStatus(nil, true, block) if err != nil { return nil, rpc.ConvertStorageError(err) } - events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), requiredEventEncodingVersion) + events, err := convert.MessagesToEventsWithEncodingConversion(resp.GetEvents(), resp.GetEventEncodingVersion(), requiredEventEncodingVersion) if err != nil { return nil, status.Errorf(codes.Internal, "failed to convert events in blockID %x: %v", blockID, err) } @@ -782,11 +774,7 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return nil, 0, "", err } - if resp.GetEventEncodingVersion() == entities.EventEncodingVersion_JSON_CDC_V0 && requiredEventEncodingVersion == entities.EventEncodingVersion_CCF_V0 { - return nil, 0, "", rpc.ConvertError(err, "conversion from JSON-CDC to CCF is forbidden", codes.Internal) - } - - events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), requiredEventEncodingVersion) + events, err := convert.MessagesToEventsWithEncodingConversion(resp.GetEvents(), resp.GetEventEncodingVersion(), requiredEventEncodingVersion) if err != nil { return nil, 0, "", rpc.ConvertError(err, "failed to convert events to message", codes.Internal) } diff --git a/engine/access/state_stream/handler.go b/engine/access/state_stream/handler.go index 2c6b471f25e..ea3db9b5a86 100644 --- a/engine/access/state_stream/handler.go +++ b/engine/access/state_stream/handler.go @@ -4,6 +4,7 @@ import ( "context" "sync/atomic" + "github.com/onflow/flow/protobuf/go/flow/entities" "github.com/onflow/flow/protobuf/go/flow/executiondata" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -160,7 +161,7 @@ func (h *Handler) SubscribeEvents(request *executiondata.SubscribeEventsRequest, return status.Errorf(codes.Internal, "unexpected response type: %T", v) } - events, err := convert.EventsToMessagesFromVersion(resp.Events, request.GetEventEncodingVersion()) + events, err := convert.EventsToMessagesWithEncodingConversion(resp.Events, entities.EventEncodingVersion_CCF_V0, request.GetEventEncodingVersion()) if err != nil { return status.Errorf(codes.Internal, "could not convert events to entity: %v", err) } diff --git a/engine/access/state_stream/handler_test.go b/engine/access/state_stream/handler_test.go index fe3ef9edad7..611f5368bb9 100644 --- a/engine/access/state_stream/handler_test.go +++ b/engine/access/state_stream/handler_test.go @@ -168,32 +168,6 @@ func TestExecutionDataStreamEventEncoding(t *testing.T) { sub.Close() } - // Test scenario for default (JSON) event encoding. - t.Run("test default(JSON)", func(t *testing.T) { - makeStreamRequest(&executiondata.SubscribeExecutionDataRequest{ - EventEncodingVersion: entities.EventEncodingVersion_JSON_CDC_V0, - }) - for { - resp, err := stream.RecvToClient() - if err == io.EOF { - break - } - require.NoError(t, err) - - convertedExecData, err := convert.MessageToBlockExecutionData(resp.GetBlockExecutionData(), flow.Testnet.Chain()) - require.NoError(t, err) - - // Verify that the payload is valid JSON-CDC. - for _, chunk := range convertedExecData.ChunkExecutionDatas { - for i, e := range chunk.Events { - assert.Equal(t, jsonEvents[i], e) - } - } - - close(stream.sentFromServer) - } - }) - // Test scenario for JSON event encoding. t.Run("test JSON event encoding", func(t *testing.T) { makeStreamRequest(&executiondata.SubscribeExecutionDataRequest{ @@ -275,7 +249,9 @@ func TestEventStream(t *testing.T) { wg.Add(1) go func() { wg.Done() - err := h.SubscribeEvents(&executiondata.SubscribeEventsRequest{}, stream) + err := h.SubscribeEvents(&executiondata.SubscribeEventsRequest{ + EventEncodingVersion: entities.EventEncodingVersion_JSON_CDC_V0, + }, stream) require.NoError(t, err) t.Log("subscription closed") }() diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index f5bd7d8a6ee..7e0f35a7587 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -98,32 +98,56 @@ func MessageToEventFromVersion(m *entities.Event, inputVersion entities.EventEnc } } -// EventsToMessagesFromVersion converts a slice of flow.Events to a slice of protobuf messages, converting +// EventsToMessagesWithEncodingConversion converts a slice of flow.Events to a slice of protobuf messages, converting // the payload encoding from CCF to JSON if the input version is CCF -func EventsToMessagesFromVersion(flowEvents []flow.Event, version entities.EventEncodingVersion) ([]*entities.Event, error) { +func EventsToMessagesWithEncodingConversion( + flowEvents []flow.Event, + from entities.EventEncodingVersion, + to entities.EventEncodingVersion, +) ([]*entities.Event, error) { + if from == entities.EventEncodingVersion_JSON_CDC_V0 && to == entities.EventEncodingVersion_CCF_V0 { + return nil, fmt.Errorf("conversion from format %s to %s is forbidden", from.String(), to.String()) + } + events := make([]*entities.Event, len(flowEvents)) for i, e := range flowEvents { - event, err := EventToMessageFromVersion(e, version) - if err != nil { - return nil, fmt.Errorf("could not convert event at index %d from format %d: %w", - e.EventIndex, version, err) + if to == entities.EventEncodingVersion_JSON_CDC_V0 { + event, err := EventToMessageFromVersion(e, from) + if err != nil { + return nil, fmt.Errorf("could not convert event at index %d from format %d: %w", + e.EventIndex, from, err) + } + events[i] = event + } else { + events[i] = EventToMessage(e) } - events[i] = event } return events, nil } -// MessagesToEventsFromVersion converts a slice of protobuf messages to a slice of flow.Events, converting +// MessagesToEventsWithEncodingConversion converts a slice of protobuf messages to a slice of flow.Events, converting // the payload encoding from CCF to JSON if the input version is CCF -func MessagesToEventsFromVersion(l []*entities.Event, version entities.EventEncodingVersion) ([]flow.Event, error) { +func MessagesToEventsWithEncodingConversion( + l []*entities.Event, + from entities.EventEncodingVersion, + to entities.EventEncodingVersion, +) ([]flow.Event, error) { + if from == entities.EventEncodingVersion_JSON_CDC_V0 && to == entities.EventEncodingVersion_CCF_V0 { + return nil, fmt.Errorf("conversion from format %s to %s is forbidden", from.String(), to.String()) + } + events := make([]flow.Event, len(l)) for i, m := range l { - event, err := MessageToEventFromVersion(m, version) - if err != nil { - return nil, fmt.Errorf("could not convert event at index %d from format %d: %w", - m.EventIndex, version, err) + if to == entities.EventEncodingVersion_JSON_CDC_V0 { + event, err := MessageToEventFromVersion(m, from) + if err != nil { + return nil, fmt.Errorf("could not convert event at index %d from format %d: %w", + m.EventIndex, from, err) + } + events[i] = *event + } else { + events[i] = MessageToEvent(m) } - events[i] = *event } return events, nil } diff --git a/engine/common/rpc/convert/events_test.go b/engine/common/rpc/convert/events_test.go index 70828a5643d..420fad947bb 100644 --- a/engine/common/rpc/convert/events_test.go +++ b/engine/common/rpc/convert/events_test.go @@ -150,7 +150,7 @@ func TestConvertEvents(t *testing.T) { t.Run("convert event from ccf to jsoncdc", func(t *testing.T) { messages := convert.EventsToMessages(ccfEvents) - converted, err := convert.MessagesToEventsFromVersion(messages, entities.EventEncodingVersion_CCF_V0) + converted, err := convert.MessagesToEventsWithEncodingConversion(messages, entities.EventEncodingVersion_CCF_V0, entities.EventEncodingVersion_JSON_CDC_V0) assert.NoError(t, err) assert.Equal(t, jsonEvents, converted) @@ -158,11 +158,26 @@ func TestConvertEvents(t *testing.T) { t.Run("convert event from jsoncdc", func(t *testing.T) { messages := convert.EventsToMessages(jsonEvents) - converted, err := convert.MessagesToEventsFromVersion(messages, entities.EventEncodingVersion_JSON_CDC_V0) + converted, err := convert.MessagesToEventsWithEncodingConversion(messages, entities.EventEncodingVersion_JSON_CDC_V0, entities.EventEncodingVersion_JSON_CDC_V0) assert.NoError(t, err) assert.Equal(t, jsonEvents, converted) }) + + t.Run("convert event from ccf", func(t *testing.T) { + messages := convert.EventsToMessages(jsonEvents) + converted, err := convert.MessagesToEventsWithEncodingConversion(messages, entities.EventEncodingVersion_CCF_V0, entities.EventEncodingVersion_CCF_V0) + assert.NoError(t, err) + + assert.Equal(t, jsonEvents, converted) + }) + + t.Run("convert event from jsoncdc to ccf", func(t *testing.T) { + messages := convert.EventsToMessages(jsonEvents) + converted, err := convert.MessagesToEventsWithEncodingConversion(messages, entities.EventEncodingVersion_JSON_CDC_V0, entities.EventEncodingVersion_CCF_V0) + assert.Error(t, err) + assert.Nil(t, converted) + }) } func TestConvertServiceEvent(t *testing.T) { diff --git a/integration/go.mod b/integration/go.mod index ff29aedfeb0..ebd8830f51d 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -346,6 +346,6 @@ replace github.com/onflow/flow-go => ../ replace github.com/onflow/flow-go/insecure => ../insecure replace ( - github.com/onflow/flow-emulator v0.53.0 => github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5 + github.com/onflow/flow-emulator v0.53.0 => github.com/Guitarheroua/flow-emulator v0.0.0-20231018151849-2b2f7bcd6fa2 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 ) diff --git a/integration/go.sum b/integration/go.sum index 7c3fde1c53c..9758567d84c 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -100,8 +100,8 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5 h1:33av+pFXKq9WNbg3EvIFQoYuDBdTqVrzc8/XCMkGc5o= -github.com/Guitarheroua/flow-emulator v0.0.0-20231013104524-7b022a3992d5/go.mod h1:gKnyRos42AIy5wm1BfHUDhhrg322F0WpIAsOU+nljj0= +github.com/Guitarheroua/flow-emulator v0.0.0-20231018151849-2b2f7bcd6fa2 h1:WIoMCjOzGI2WyVWsTiVmWzTfnPKoaq0jGDj0XjuFBwE= +github.com/Guitarheroua/flow-emulator v0.0.0-20231018151849-2b2f7bcd6fa2/go.mod h1:m0BlKMdFBteEH3oepU1yeerU4kvpTctvO+6mkmF9nlU= github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 h1:6Szrh+jL5X83OiKh76ORt7QHxtZvM/uRaU5G6zd6syc= github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= From ca083baa94815658bac75232a77e4965ea27be9b Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Thu, 19 Oct 2023 13:34:49 +0300 Subject: [PATCH 20/27] regenerate mock --- access/mock/api.go | 60 +++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/access/mock/api.go b/access/mock/api.go index f79189be45c..ca8439b299b 100644 --- a/access/mock/api.go +++ b/access/mock/api.go @@ -333,17 +333,17 @@ func (_m *API) GetCollectionByID(ctx context.Context, id flow.Identifier) (*flow return r0, r1 } -// GetEventsForBlockIDs provides a mock function with given fields: ctx, eventType, blockIDs, eventEncodingVersionValue -func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, eventEncodingVersionValue entities.EventEncodingVersion) ([]flow.BlockEvents, error) { - ret := _m.Called(ctx, eventType, blockIDs, eventEncodingVersionValue) +// GetEventsForBlockIDs provides a mock function with given fields: ctx, eventType, blockIDs, requiredEventEncodingVersion +func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) ([]flow.BlockEvents, error) { + ret := _m.Called(ctx, eventType, blockIDs, requiredEventEncodingVersion) var r0 []flow.BlockEvents var r1 error if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier, entities.EventEncodingVersion) ([]flow.BlockEvents, error)); ok { - return rf(ctx, eventType, blockIDs, eventEncodingVersionValue) + return rf(ctx, eventType, blockIDs, requiredEventEncodingVersion) } if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier, entities.EventEncodingVersion) []flow.BlockEvents); ok { - r0 = rf(ctx, eventType, blockIDs, eventEncodingVersionValue) + r0 = rf(ctx, eventType, blockIDs, requiredEventEncodingVersion) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]flow.BlockEvents) @@ -351,7 +351,7 @@ func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, block } if rf, ok := ret.Get(1).(func(context.Context, string, []flow.Identifier, entities.EventEncodingVersion) error); ok { - r1 = rf(ctx, eventType, blockIDs, eventEncodingVersionValue) + r1 = rf(ctx, eventType, blockIDs, requiredEventEncodingVersion) } else { r1 = ret.Error(1) } @@ -359,17 +359,17 @@ func (_m *API) GetEventsForBlockIDs(ctx context.Context, eventType string, block return r0, r1 } -// GetEventsForHeightRange provides a mock function with given fields: ctx, eventType, startHeight, endHeight, eventEncodingVersionValue -func (_m *API) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight uint64, endHeight uint64, eventEncodingVersionValue entities.EventEncodingVersion) ([]flow.BlockEvents, error) { - ret := _m.Called(ctx, eventType, startHeight, endHeight, eventEncodingVersionValue) +// GetEventsForHeightRange provides a mock function with given fields: ctx, eventType, startHeight, endHeight, requiredEventEncodingVersion +func (_m *API) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight uint64, endHeight uint64, requiredEventEncodingVersion entities.EventEncodingVersion) ([]flow.BlockEvents, error) { + ret := _m.Called(ctx, eventType, startHeight, endHeight, requiredEventEncodingVersion) var r0 []flow.BlockEvents var r1 error if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64, entities.EventEncodingVersion) ([]flow.BlockEvents, error)); ok { - return rf(ctx, eventType, startHeight, endHeight, eventEncodingVersionValue) + return rf(ctx, eventType, startHeight, endHeight, requiredEventEncodingVersion) } if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64, entities.EventEncodingVersion) []flow.BlockEvents); ok { - r0 = rf(ctx, eventType, startHeight, endHeight, eventEncodingVersionValue) + r0 = rf(ctx, eventType, startHeight, endHeight, requiredEventEncodingVersion) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]flow.BlockEvents) @@ -377,7 +377,7 @@ func (_m *API) GetEventsForHeightRange(ctx context.Context, eventType string, st } if rf, ok := ret.Get(1).(func(context.Context, string, uint64, uint64, entities.EventEncodingVersion) error); ok { - r1 = rf(ctx, eventType, startHeight, endHeight, eventEncodingVersionValue) + r1 = rf(ctx, eventType, startHeight, endHeight, requiredEventEncodingVersion) } else { r1 = ret.Error(1) } @@ -595,17 +595,17 @@ func (_m *API) GetTransaction(ctx context.Context, id flow.Identifier) (*flow.Tr return r0, r1 } -// GetTransactionResult provides a mock function with given fields: ctx, id, blockID, collectionID, eventEncodingVersionValue -func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, eventEncodingVersionValue entities.EventEncodingVersion) (*access.TransactionResult, error) { - ret := _m.Called(ctx, id, blockID, collectionID, eventEncodingVersionValue) +// GetTransactionResult provides a mock function with given fields: ctx, id, blockID, collectionID, requiredEventEncodingVersion +func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) (*access.TransactionResult, error) { + ret := _m.Called(ctx, id, blockID, collectionID, requiredEventEncodingVersion) var r0 *access.TransactionResult var r1 error if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, entities.EventEncodingVersion) (*access.TransactionResult, error)); ok { - return rf(ctx, id, blockID, collectionID, eventEncodingVersionValue) + return rf(ctx, id, blockID, collectionID, requiredEventEncodingVersion) } if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, entities.EventEncodingVersion) *access.TransactionResult); ok { - r0 = rf(ctx, id, blockID, collectionID, eventEncodingVersionValue) + r0 = rf(ctx, id, blockID, collectionID, requiredEventEncodingVersion) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*access.TransactionResult) @@ -613,7 +613,7 @@ func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blo } if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier, entities.EventEncodingVersion) error); ok { - r1 = rf(ctx, id, blockID, collectionID, eventEncodingVersionValue) + r1 = rf(ctx, id, blockID, collectionID, requiredEventEncodingVersion) } else { r1 = ret.Error(1) } @@ -621,17 +621,17 @@ func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blo return r0, r1 } -// GetTransactionResultByIndex provides a mock function with given fields: ctx, blockID, index, eventEncodingVersionValue -func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, eventEncodingVersionValue entities.EventEncodingVersion) (*access.TransactionResult, error) { - ret := _m.Called(ctx, blockID, index, eventEncodingVersionValue) +// GetTransactionResultByIndex provides a mock function with given fields: ctx, blockID, index, requiredEventEncodingVersion +func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, requiredEventEncodingVersion entities.EventEncodingVersion) (*access.TransactionResult, error) { + ret := _m.Called(ctx, blockID, index, requiredEventEncodingVersion) var r0 *access.TransactionResult var r1 error if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32, entities.EventEncodingVersion) (*access.TransactionResult, error)); ok { - return rf(ctx, blockID, index, eventEncodingVersionValue) + return rf(ctx, blockID, index, requiredEventEncodingVersion) } if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, uint32, entities.EventEncodingVersion) *access.TransactionResult); ok { - r0 = rf(ctx, blockID, index, eventEncodingVersionValue) + r0 = rf(ctx, blockID, index, requiredEventEncodingVersion) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*access.TransactionResult) @@ -639,7 +639,7 @@ func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Ide } if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, uint32, entities.EventEncodingVersion) error); ok { - r1 = rf(ctx, blockID, index, eventEncodingVersionValue) + r1 = rf(ctx, blockID, index, requiredEventEncodingVersion) } else { r1 = ret.Error(1) } @@ -647,17 +647,17 @@ func (_m *API) GetTransactionResultByIndex(ctx context.Context, blockID flow.Ide return r0, r1 } -// GetTransactionResultsByBlockID provides a mock function with given fields: ctx, blockID, eventEncodingVersionValue -func (_m *API) GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, eventEncodingVersionValue entities.EventEncodingVersion) ([]*access.TransactionResult, error) { - ret := _m.Called(ctx, blockID, eventEncodingVersionValue) +// GetTransactionResultsByBlockID provides a mock function with given fields: ctx, blockID, requiredEventEncodingVersion +func (_m *API) GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) ([]*access.TransactionResult, error) { + ret := _m.Called(ctx, blockID, requiredEventEncodingVersion) var r0 []*access.TransactionResult var r1 error if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, entities.EventEncodingVersion) ([]*access.TransactionResult, error)); ok { - return rf(ctx, blockID, eventEncodingVersionValue) + return rf(ctx, blockID, requiredEventEncodingVersion) } if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, entities.EventEncodingVersion) []*access.TransactionResult); ok { - r0 = rf(ctx, blockID, eventEncodingVersionValue) + r0 = rf(ctx, blockID, requiredEventEncodingVersion) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*access.TransactionResult) @@ -665,7 +665,7 @@ func (_m *API) GetTransactionResultsByBlockID(ctx context.Context, blockID flow. } if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, entities.EventEncodingVersion) error); ok { - r1 = rf(ctx, blockID, eventEncodingVersionValue) + r1 = rf(ctx, blockID, requiredEventEncodingVersion) } else { r1 = ret.Error(1) } From 927a293aead4fb12b15179c4fa327b58b149d066 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Fri, 20 Oct 2023 11:30:35 +0300 Subject: [PATCH 21/27] Fixed remarks --- engine/common/rpc/convert/events.go | 46 ++++++++++----------- engine/common/rpc/convert/execution_data.go | 1 - go.mod | 4 +- go.sum | 4 +- insecure/go.mod | 4 +- insecure/go.sum | 4 +- integration/go.mod | 7 +--- integration/go.sum | 4 +- 8 files changed, 33 insertions(+), 41 deletions(-) diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index 7e0f35a7587..2a1572efc87 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -106,21 +106,21 @@ func EventsToMessagesWithEncodingConversion( to entities.EventEncodingVersion, ) ([]*entities.Event, error) { if from == entities.EventEncodingVersion_JSON_CDC_V0 && to == entities.EventEncodingVersion_CCF_V0 { - return nil, fmt.Errorf("conversion from format %s to %s is forbidden", from.String(), to.String()) + return nil, fmt.Errorf("conversion from format %s to %s is not supported", from.String(), to.String()) + } + + if from == to { + return EventsToMessages(flowEvents), nil } events := make([]*entities.Event, len(flowEvents)) for i, e := range flowEvents { - if to == entities.EventEncodingVersion_JSON_CDC_V0 { - event, err := EventToMessageFromVersion(e, from) - if err != nil { - return nil, fmt.Errorf("could not convert event at index %d from format %d: %w", - e.EventIndex, from, err) - } - events[i] = event - } else { - events[i] = EventToMessage(e) + event, err := EventToMessageFromVersion(e, from) + if err != nil { + return nil, fmt.Errorf("could not convert event at index %d from format %d: %w", + e.EventIndex, from, err) } + events[i] = event } return events, nil } @@ -128,26 +128,26 @@ func EventsToMessagesWithEncodingConversion( // MessagesToEventsWithEncodingConversion converts a slice of protobuf messages to a slice of flow.Events, converting // the payload encoding from CCF to JSON if the input version is CCF func MessagesToEventsWithEncodingConversion( - l []*entities.Event, + messageEvents []*entities.Event, from entities.EventEncodingVersion, to entities.EventEncodingVersion, ) ([]flow.Event, error) { if from == entities.EventEncodingVersion_JSON_CDC_V0 && to == entities.EventEncodingVersion_CCF_V0 { - return nil, fmt.Errorf("conversion from format %s to %s is forbidden", from.String(), to.String()) + return nil, fmt.Errorf("conversion from format %s to %s is not supported", from.String(), to.String()) } - events := make([]flow.Event, len(l)) - for i, m := range l { - if to == entities.EventEncodingVersion_JSON_CDC_V0 { - event, err := MessageToEventFromVersion(m, from) - if err != nil { - return nil, fmt.Errorf("could not convert event at index %d from format %d: %w", - m.EventIndex, from, err) - } - events[i] = *event - } else { - events[i] = MessageToEvent(m) + if from == to { + return MessagesToEvents(messageEvents), nil + } + + events := make([]flow.Event, len(messageEvents)) + for i, m := range messageEvents { + event, err := MessageToEventFromVersion(m, from) + if err != nil { + return nil, fmt.Errorf("could not convert event at index %d from format %d: %w", + m.EventIndex, from, err) } + events[i] = *event } return events, nil } diff --git a/engine/common/rpc/convert/execution_data.go b/engine/common/rpc/convert/execution_data.go index 4eee13f1eb1..560c4887ade 100644 --- a/engine/common/rpc/convert/execution_data.go +++ b/engine/common/rpc/convert/execution_data.go @@ -31,7 +31,6 @@ func BlockExecutionDataEventPayloadsToVersion( m.ChunkExecutionData[i].Events[j].Payload = converted } } - return nil } diff --git a/go.mod b/go.mod index 5805bf76e93..ad36f9164fb 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 github.com/onflow/flow-go-sdk v0.41.10 github.com/onflow/flow-go/crypto v0.24.9 - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63 github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible @@ -297,5 +297,3 @@ require ( lukechampine.com/blake3 v1.2.1 // indirect nhooyr.io/websocket v1.8.7 // indirect ) - -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 diff --git a/go.sum b/go.sum index a48c6baae83..f4f1ae3262f 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,6 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 h1:6Szrh+jL5X83OiKh76ORt7QHxtZvM/uRaU5G6zd6syc= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -1330,6 +1328,8 @@ github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63 h1:SX8OhYbyKBExhy4qEDR/Hw6MVTBTzlDb8LfCHfFyte4= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= diff --git a/insecure/go.mod b/insecure/go.mod index c131a947251..46203bc4599 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -196,7 +196,7 @@ require ( github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect github.com/onflow/flow-go-sdk v0.41.10 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.1.0 // indirect - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 // indirect + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63 // indirect github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d // indirect github.com/onflow/sdks v0.5.0 // indirect github.com/onflow/wal v0.0.0-20230529184820-bc9f8244608d // indirect @@ -287,5 +287,3 @@ require ( ) replace github.com/onflow/flow-go => ../ - -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 diff --git a/insecure/go.sum b/insecure/go.sum index d10c4b6ef33..172b1193848 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -87,8 +87,6 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 h1:6Szrh+jL5X83OiKh76ORt7QHxtZvM/uRaU5G6zd6syc= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -1304,6 +1302,8 @@ github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63 h1:SX8OhYbyKBExhy4qEDR/Hw6MVTBTzlDb8LfCHfFyte4= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= diff --git a/integration/go.mod b/integration/go.mod index a5ec2196385..6e31a565ca7 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -26,7 +26,7 @@ require ( github.com/onflow/flow-go-sdk v0.41.10 github.com/onflow/flow-go/crypto v0.24.9 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63 github.com/plus3it/gorecurcopy v0.0.1 github.com/prometheus/client_golang v1.16.0 github.com/prometheus/client_model v0.4.0 @@ -346,7 +346,4 @@ replace github.com/onflow/flow-go => ../ replace github.com/onflow/flow-go/insecure => ../insecure -replace ( - github.com/onflow/flow-emulator v0.53.0 => github.com/Guitarheroua/flow-emulator v0.0.0-20231018151849-2b2f7bcd6fa2 - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231017162044-5d0f9b6dfdb2 => github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 -) +replace github.com/onflow/flow-emulator v0.53.0 => github.com/Guitarheroua/flow-emulator v0.0.0-20231018151849-2b2f7bcd6fa2 diff --git a/integration/go.sum b/integration/go.sum index b869b329fd6..291fae14832 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -102,8 +102,6 @@ github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Guitarheroua/flow-emulator v0.0.0-20231018151849-2b2f7bcd6fa2 h1:WIoMCjOzGI2WyVWsTiVmWzTfnPKoaq0jGDj0XjuFBwE= github.com/Guitarheroua/flow-emulator v0.0.0-20231018151849-2b2f7bcd6fa2/go.mod h1:m0BlKMdFBteEH3oepU1yeerU4kvpTctvO+6mkmF9nlU= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001 h1:6Szrh+jL5X83OiKh76ORt7QHxtZvM/uRaU5G6zd6syc= -github.com/Guitarheroua/flow/protobuf/go/flow v0.0.0-20231018150252-f223f1d42001/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= @@ -1449,6 +1447,8 @@ github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63 h1:SX8OhYbyKBExhy4qEDR/Hw6MVTBTzlDb8LfCHfFyte4= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/nft-storefront/lib/go/contracts v0.0.0-20221222181731-14b90207cead h1:2j1Unqs76Z1b95Gu4C3Y28hzNUHBix7wL490e61SMSw= From e45b6e5669452d33233df594adc466413e193574 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Fri, 20 Oct 2023 16:08:22 +0300 Subject: [PATCH 22/27] Fixed and simplify handler tests. --- engine/access/state_stream/handler_test.go | 175 ++++++--------------- 1 file changed, 52 insertions(+), 123 deletions(-) diff --git a/engine/access/state_stream/handler_test.go b/engine/access/state_stream/handler_test.go index 611f5368bb9..27d91849289 100644 --- a/engine/access/state_stream/handler_test.go +++ b/engine/access/state_stream/handler_test.go @@ -33,104 +33,20 @@ func TestExecutionDataStream(t *testing.T) { api := ssmock.NewAPI(t) stream := makeStreamMock[executiondata.SubscribeExecutionDataRequest, executiondata.SubscribeExecutionDataResponse](ctx) - sub := state_stream.NewSubscription(1) // generate some events with a payload to include // generators will produce identical event payloads (before encoding) ccfEventGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingCCF)) jsonEventsGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingJSON)) - inputEvents := make([]flow.Event, 0, 3) - expectedEvents := make([]flow.Event, 0, 3) + ccfInputEvents := make([]flow.Event, 0, 3) + jsonExpectedEvents := make([]flow.Event, 0, 3) for i := 0; i < 3; i++ { - inputEvents = append(inputEvents, ccfEventGenerator.New()) - expectedEvents = append(expectedEvents, jsonEventsGenerator.New()) + ccfInputEvents = append(ccfInputEvents, ccfEventGenerator.New()) + jsonExpectedEvents = append(jsonExpectedEvents, jsonEventsGenerator.New()) } - api.On("SubscribeExecutionData", mock.Anything, flow.ZeroID, uint64(0), mock.Anything).Return(sub) - - h := state_stream.NewHandler(api, flow.Localnet.Chain(), state_stream.EventFilterConfig{}, 1) - - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - wg.Done() - err := h.SubscribeExecutionData(&executiondata.SubscribeExecutionDataRequest{}, stream) - require.NoError(t, err) - t.Log("subscription closed") - }() - wg.Wait() - - // send a single response + // Send a single response. blockHeight := uint64(1) - executionData := unittest.BlockExecutionDataFixture( - unittest.WithChunkExecutionDatas( - unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(inputEvents)), - unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(inputEvents)), - ), - ) - - err := sub.Send(ctx, &state_stream.ExecutionDataResponse{ - Height: blockHeight, - ExecutionData: executionData, - }, 100*time.Millisecond) - require.NoError(t, err) - - // notify end of data - sub.Close() - - receivedCount := 0 - for { - t.Log(receivedCount) - resp, err := stream.RecvToClient() - if err == io.EOF { - break - } - require.NoError(t, err) - - convertedExecData, err := convert.MessageToBlockExecutionData(resp.GetBlockExecutionData(), flow.Testnet.Chain()) - require.NoError(t, err) - - assert.Equal(t, blockHeight, resp.GetBlockHeight()) - - // make sure the payload is valid JSON-CDC - for _, chunk := range convertedExecData.ChunkExecutionDatas { - for i, e := range chunk.Events { - assert.Equal(t, expectedEvents[i], e) - - _, err := jsoncdc.Decode(nil, e.Payload) - require.NoError(t, err) - } - } - - receivedCount++ - - // shutdown the stream after one response - close(stream.sentFromServer) - } - - // only expect a single response - assert.Equal(t, 1, receivedCount) -} - -// TestExecutionDataStreamEventEncoding tests the event encoding behavior of the execution data stream. -func TestExecutionDataStreamEventEncoding(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Generate sample events for testing, each with a payload. - ccfEventGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingCCF)) - ccfEvents := make([]flow.Event, 0, 1) - ccfEvents = append(ccfEvents, ccfEventGenerator.New()) - - jsonEventsGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingJSON)) - jsonEvents := make([]flow.Event, 0, 1) - jsonEvents = append(jsonEvents, jsonEventsGenerator.New()) - - // Create a mock API and a mock stream for subscription. - api := ssmock.NewAPI(t) - stream := makeStreamMock[executiondata.SubscribeExecutionDataRequest, executiondata.SubscribeExecutionDataResponse](ctx) // Helper function to perform a stream request and handle responses. makeStreamRequest := func(request *executiondata.SubscribeExecutionDataRequest) { @@ -150,11 +66,10 @@ func TestExecutionDataStreamEventEncoding(t *testing.T) { }() wg.Wait() - // Send a single response. - blockHeight := uint64(1) executionData := unittest.BlockExecutionDataFixture( unittest.WithChunkExecutionDatas( - unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(ccfEvents)), + unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(ccfInputEvents)), + unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(ccfInputEvents)), ), ) @@ -168,56 +83,70 @@ func TestExecutionDataStreamEventEncoding(t *testing.T) { sub.Close() } - // Test scenario for JSON event encoding. - t.Run("test JSON event encoding", func(t *testing.T) { - makeStreamRequest(&executiondata.SubscribeExecutionDataRequest{ - EventEncodingVersion: entities.EventEncodingVersion_JSON_CDC_V0, - }) + // handleExecutionDataStreamResponses handles responses from the execution data stream. + handleExecutionDataStreamResponses := func(version entities.EventEncodingVersion, expectedEvents []flow.Event) { + receivedCount := 0 + var responses []*executiondata.SubscribeExecutionDataResponse for { resp, err := stream.RecvToClient() if err == io.EOF { break } require.NoError(t, err) + responses = append(responses, resp) + receivedCount++ + close(stream.sentFromServer) + } + for _, resp := range responses { convertedExecData, err := convert.MessageToBlockExecutionData(resp.GetBlockExecutionData(), flow.Testnet.Chain()) require.NoError(t, err) - // Verify that the payload is valid JSON-CDC. + assert.Equal(t, blockHeight, resp.GetBlockHeight()) + + // Verify that the payload is valid for _, chunk := range convertedExecData.ChunkExecutionDatas { for i, e := range chunk.Events { - assert.Equal(t, jsonEvents[i], e) + assert.Equal(t, expectedEvents[i], e) + + if version == entities.EventEncodingVersion_JSON_CDC_V0 { + _, err := jsoncdc.Decode(nil, e.Payload) + require.NoError(t, err) + } } } - close(stream.sentFromServer) + + // only expect a single response + assert.Equal(t, 1, receivedCount) } + } - }) + tests := []struct { + name string + eventVersion entities.EventEncodingVersion + expected []flow.Event + }{ + { + "test JSON event encoding", + entities.EventEncodingVersion_JSON_CDC_V0, + jsonExpectedEvents, + }, + { + "test CFF event encoding", + entities.EventEncodingVersion_CCF_V0, + ccfInputEvents, + }, + } - // Test scenario for CFF event encoding. - t.Run("test CFF event encoding", func(t *testing.T) { - makeStreamRequest(&executiondata.SubscribeExecutionDataRequest{ - EventEncodingVersion: entities.EventEncodingVersion_CCF_V0, + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + makeStreamRequest(&executiondata.SubscribeExecutionDataRequest{ + EventEncodingVersion: test.eventVersion, + }) + handleExecutionDataStreamResponses(test.eventVersion, test.expected) }) - for { - resp, err := stream.RecvToClient() - if err == io.EOF { - break - } - require.NoError(t, err) - - convertedExecData, err := convert.MessageToBlockExecutionData(resp.GetBlockExecutionData(), flow.Testnet.Chain()) - require.NoError(t, err) + } - // Verify that the payload is valid CCF-V0. - for _, chunk := range convertedExecData.ChunkExecutionDatas { - for i, e := range chunk.Events { - assert.Equal(t, ccfEvents[i], e) - } - } - close(stream.sentFromServer) - } - }) } func TestEventStream(t *testing.T) { From 7d0fe75cff97978db4d55eac4a900474c82af617 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Fri, 20 Oct 2023 19:36:42 +0300 Subject: [PATCH 23/27] Added missing tests. --- engine/access/rpc/backend/backend_test.go | 243 ++++++++++++++++++---- 1 file changed, 197 insertions(+), 46 deletions(-) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 781c689b647..528aa9e3693 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -37,6 +37,11 @@ import ( const TEST_MAX_HEIGHT = 100 +var eventEncodingVersions = []entitiesproto.EventEncodingVersion{ + entitiesproto.EventEncodingVersion_JSON_CDC_V0, + entitiesproto.EventEncodingVersion_CCF_V0, +} + type Suite struct { suite.Suite @@ -983,7 +988,8 @@ func (suite *Suite) TestGetEventsForBlockIDs() { suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() suite.state.On("Final").Return(suite.snapshot, nil).Maybe() - events := getEvents(10) + exeNodeEventEncodingVersion := entitiesproto.EventEncodingVersion_CCF_V0 + events := getEventsWithEncoding(10, exeNodeEventEncodingVersion) validExecutorIdentities := flow.IdentityList{} setupStorage := func(n int) []*flow.Header { @@ -1032,18 +1038,20 @@ func (suite *Suite) TestGetEventsForBlockIDs() { } expected := make([]flow.BlockEvents, len(blockHeaders)) + expectedEvents := getEventsWithEncoding(10, entitiesproto.EventEncodingVersion_JSON_CDC_V0) for i := 0; i < len(blockHeaders); i++ { expected[i] = flow.BlockEvents{ BlockID: blockHeaders[i].ID(), BlockHeight: blockHeaders[i].Height, BlockTimestamp: blockHeaders[i].Timestamp, - Events: events, + Events: expectedEvents, } } // create the execution node response exeResp := &execproto.GetEventsForBlockIDsResponse{ - Results: exeResults, + Results: exeResults, + EventEncodingVersion: exeNodeEventEncodingVersion, } ctx := context.Background() @@ -1104,6 +1112,28 @@ func (suite *Suite) TestGetEventsForBlockIDs() { require.Empty(suite.T(), resp) }) + for _, version := range eventEncodingVersions { + suite.Run(fmt.Sprintf("test %s event encoding version for GetEventsForBlockIDs", version.String()), func() { + params := suite.defaultBackendParams() + params.ExecutionReceipts = receipts + params.ConnFactory = connFactory + params.FixedExecutionNodeIDs = validENIDs.Strings() + + // create the handler + backend, err := New(params) + suite.Require().NoError(err) + + // execute request with an empty block id list and expect an empty list of events and no error + result, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), []flow.Identifier{}, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + expectedResult := getEventsWithEncoding(1, version) + suite.checkResponse(result, err) + + for _, blockEvent := range result { + suite.Assert().Equal(blockEvent.Events, expectedResult) + } + }) + } + suite.assertAllExpectations() } @@ -1313,15 +1343,17 @@ func (suite *Suite) TestGetEventsForHeightRange() { results := make([]flow.BlockEvents, len(blockHeaders)) exeResults := make([]*execproto.GetEventsForBlockIDsResponse_Result, len(blockHeaders)) + exeNodeEventEncodingVersion := entitiesproto.EventEncodingVersion_CCF_V0 + for i, header := range blockHeaders { - events := getEvents(1) + events := getEventsWithEncoding(1, exeNodeEventEncodingVersion) height := header.Height results[i] = flow.BlockEvents{ BlockID: header.ID(), BlockHeight: height, BlockTimestamp: header.Timestamp, - Events: events, + Events: getEventsWithEncoding(1, entitiesproto.EventEncodingVersion_JSON_CDC_V0), } exeResults[i] = &execproto.GetEventsForBlockIDsResponse_Result{ @@ -1332,7 +1364,8 @@ func (suite *Suite) TestGetEventsForHeightRange() { } exeResp := &execproto.GetEventsForBlockIDsResponse{ - Results: exeResults, + Results: exeResults, + EventEncodingVersion: exeNodeEventEncodingVersion, } suite.execClient. @@ -1459,6 +1492,36 @@ func (suite *Suite) TestGetEventsForHeightRange() { suite.Require().Error(err) }) + for _, version := range eventEncodingVersions { + suite.Run(fmt.Sprintf("test %s event encoding version for GetEventsForHeightRange", version.String()), func() { + headHeight = maxHeight - 1 + setupHeadHeight(headHeight) + blockHeaders, _, nodeIdentities = setupStorage(minHeight, headHeight) + _ = setupExecClient() + fixedENIdentifiersStr := nodeIdentities.NodeIDs().Strings() + + stateParams.On("SporkID").Return(unittest.IdentifierFixture(), nil) + stateParams.On("ProtocolVersion").Return(uint(unittest.Uint64InRange(10, 30)), nil) + stateParams.On("SporkRootBlockHeight").Return(headHeight, nil) + stateParams.On("SealedRoot").Return(head, nil) + + params := suite.defaultBackendParams() + params.State = state + params.ConnFactory = connFactory + params.FixedExecutionNodeIDs = fixedENIdentifiersStr + + backend, err := New(params) + suite.Require().NoError(err) + + result, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, version) + expectedResult := getEventsWithEncoding(1, version) + suite.checkResponse(result, err) + + for _, blockEvent := range result { + suite.Assert().Equal(blockEvent.Events, expectedResult) + } + }) + } } func (suite *Suite) TestGetNodeVersionInfo() { @@ -1747,15 +1810,36 @@ func (suite *Suite) TestExecutionNodesForBlockID() { }) } -// TestGetTransactionResultByIndexEventEncodingVersion tests the GetTransactionResultByIndex function with different -// event encoding versions. -func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { +// TestGetTransactionResultEventEncodingVersion tests the GetTransactionResult function with different event encoding versions. +func (suite *Suite) TestGetTransactionResultEventEncodingVersion() { suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() ctx := context.Background() + + collection := unittest.CollectionFixture(1) + transactionBody := collection.Transactions[0] + // block which will eventually contain the transaction block := unittest.BlockFixture() + block.SetPayload( + unittest.PayloadFixture( + unittest.WithGuarantees( + unittest.CollectionGuaranteesWithCollectionIDFixture([]*flow.Collection{&collection})...))) blockId := block.ID() - index := uint32(0) + + // reference block to which the transaction points to + refBlock := unittest.BlockFixture() + refBlockID := refBlock.ID() + refBlock.Header.Height = 2 + transactionBody.SetReferenceBlockID(refBlockID) + txId := transactionBody.ID() + + // transaction storage returns the corresponding transaction + suite.transactions. + On("ByID", txId). + Return(transactionBody, nil) + + light := collection.Light() + suite.collections.On("LightByID", mock.Anything).Return(&light, nil) suite.snapshot.On("Head").Return(block.Header, nil) @@ -1780,51 +1864,118 @@ func (suite *Suite) TestGetTransactionResultByIndexEventEncodingVersion() { backend, err := New(params) suite.Require().NoError(err) - exeEventReq := &execproto.GetTransactionByIndexRequest{ - BlockId: blockId[:], - Index: index, - } + exeNodeEventEncodingVersion := entitiesproto.EventEncodingVersion_CCF_V0 + events := getEventsWithEncoding(1, exeNodeEventEncodingVersion) + eventMessages := convert.EventsToMessages(events) - defaultEncoding := entitiesproto.EventEncodingVersion_CCF_V0 + for _, version := range eventEncodingVersions { + suite.Run(fmt.Sprintf("test %s event encoding version for GetTransactionResult", version.String()), func() { + exeEventResp := &execproto.GetTransactionResultResponse{ + Events: eventMessages, + EventEncodingVersion: exeNodeEventEncodingVersion, + } - // Define a helper function to prepare the GetTransactionResultResponse with a given encoding version. - prepareGetTransactionResultByIndex := func() { - events := getEventsWithEncoding(1, defaultEncoding) + suite.execClient. + On("GetTransactionResult", ctx, &execproto.GetTransactionResultRequest{ + BlockId: blockId[:], + TransactionId: txId[:], + }). + Return(exeEventResp, nil). + Once() - exeEventResp := &execproto.GetTransactionResultResponse{ - Events: convert.EventsToMessages(events), - EventEncodingVersion: entitiesproto.EventEncodingVersion_CCF_V0, - } + result, err := backend.GetTransactionResult(ctx, txId, blockId, flow.ZeroID, version) + expectedResult := getEventsWithEncoding(1, version) + suite.checkResponse(result, err) - suite.execClient. - On("GetTransactionResultByIndex", ctx, exeEventReq). - Return(exeEventResp, nil). - Once() + suite.Assert().Equal(result.Events, expectedResult) + }) } +} - // Define a helper function to assert the result expectations. - assertResultExpectations := func( - expectedResult []flow.Event, - encodingVersion entitiesproto.EventEncodingVersion, - ) { - result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, encodingVersion) - suite.checkResponse(result, err) +// TestGetTransactionResultEventEncodingVersion tests the GetTransactionResult function with different event encoding versions. +func (suite *Suite) TestGetTransactionResultByIndexAndBlockIdEventEncodingVersion() { + suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() - suite.Require().NoError(err) - suite.Assert().Equal(result.Events, expectedResult) - } + ctx := context.Background() + block := unittest.BlockFixture() + blockId := block.ID() + index := uint32(0) - suite.Run("test JSON event encoding", func() { - prepareGetTransactionResultByIndex() - expectedResult := getEventsWithEncoding(1, entitiesproto.EventEncodingVersion_JSON_CDC_V0) - assertResultExpectations(expectedResult, entitiesproto.EventEncodingVersion_JSON_CDC_V0) - }) + suite.snapshot.On("Head").Return(block.Header, nil) - suite.Run("test CFF event encoding", func() { - prepareGetTransactionResultByIndex() - expectedResult := getEventsWithEncoding(1, entitiesproto.EventEncodingVersion_CCF_V0) - assertResultExpectations(expectedResult, entitiesproto.EventEncodingVersion_CCF_V0) - }) + // block storage returns the corresponding block + suite.blocks. + On("ByID", blockId). + Return(&block, nil) + + _, fixedENIDs := suite.setupReceipts(&block) + suite.state.On("Final").Return(suite.snapshot, nil).Maybe() + suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) + + // create a mock connection factory + connFactory := connectionmock.NewConnectionFactory(suite.T()) + connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) + + params := suite.defaultBackendParams() + // the connection factory should be used to get the execution node client + params.ConnFactory = connFactory + params.FixedExecutionNodeIDs = (fixedENIDs.NodeIDs()).Strings() + + backend, err := New(params) + suite.Require().NoError(err) + + exeNodeEventEncodingVersion := entitiesproto.EventEncodingVersion_CCF_V0 + events := getEventsWithEncoding(1, exeNodeEventEncodingVersion) + eventMessages := convert.EventsToMessages(events) + + for _, version := range eventEncodingVersions { + suite.Run(fmt.Sprintf("test %s event encoding version for GetTransactionResultByIndex", version.String()), func() { + exeEventResp := &execproto.GetTransactionResultResponse{ + Events: eventMessages, + EventEncodingVersion: exeNodeEventEncodingVersion, + } + + suite.execClient. + On("GetTransactionResultByIndex", ctx, &execproto.GetTransactionByIndexRequest{ + BlockId: blockId[:], + Index: index, + }). + Return(exeEventResp, nil). + Once() + + result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, version) + suite.checkResponse(result, err) + + expectedResult := getEventsWithEncoding(1, version) + suite.Assert().Equal(result.Events, expectedResult) + }) + + suite.Run(fmt.Sprintf("test %s event encoding version for GetTransactionResultsByBlockID", version.String()), func() { + exeEventResp := &execproto.GetTransactionResultsResponse{ + TransactionResults: []*execproto.GetTransactionResultResponse{ + { + Events: eventMessages, + EventEncodingVersion: exeNodeEventEncodingVersion, + }}, + EventEncodingVersion: exeNodeEventEncodingVersion, + } + + suite.execClient. + On("GetTransactionResultsByBlockID", ctx, &execproto.GetTransactionsByBlockIDRequest{ + BlockId: blockId[:], + }). + Return(exeEventResp, nil). + Once() + + results, err := backend.GetTransactionResultsByBlockID(ctx, blockId, version) + suite.checkResponse(results, err) + + expectedResult := getEventsWithEncoding(1, version) + for _, result := range results { + suite.Assert().Equal(result.Events, expectedResult) + } + }) + } } // getEventsWithEncoding generates a specified number of events with a given encoding version. From 827eaf6e949205de48ca627f1ef3c15ab7f454f8 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Tue, 24 Oct 2023 21:41:03 +0300 Subject: [PATCH 24/27] Fixed remarks. Improved tests. Added more tests. --- engine/access/rpc/backend/backend_test.go | 42 +-- .../state_stream/backend/handler_test.go | 357 +++++++++++------- .../common/rpc/convert/execution_data_test.go | 11 +- utils/unittest/generator/events.go | 34 +- 4 files changed, 261 insertions(+), 183 deletions(-) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 528aa9e3693..20002bd0b3d 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -989,7 +989,7 @@ func (suite *Suite) TestGetEventsForBlockIDs() { suite.state.On("Final").Return(suite.snapshot, nil).Maybe() exeNodeEventEncodingVersion := entitiesproto.EventEncodingVersion_CCF_V0 - events := getEventsWithEncoding(10, exeNodeEventEncodingVersion) + events := generator.GetEventsWithEncoding(10, exeNodeEventEncodingVersion) validExecutorIdentities := flow.IdentityList{} setupStorage := func(n int) []*flow.Header { @@ -1038,7 +1038,7 @@ func (suite *Suite) TestGetEventsForBlockIDs() { } expected := make([]flow.BlockEvents, len(blockHeaders)) - expectedEvents := getEventsWithEncoding(10, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + expectedEvents := generator.GetEventsWithEncoding(10, entitiesproto.EventEncodingVersion_JSON_CDC_V0) for i := 0; i < len(blockHeaders); i++ { expected[i] = flow.BlockEvents{ BlockID: blockHeaders[i].ID(), @@ -1124,8 +1124,8 @@ func (suite *Suite) TestGetEventsForBlockIDs() { suite.Require().NoError(err) // execute request with an empty block id list and expect an empty list of events and no error - result, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), []flow.Identifier{}, entitiesproto.EventEncodingVersion_JSON_CDC_V0) - expectedResult := getEventsWithEncoding(1, version) + result, err := backend.GetEventsForBlockIDs(ctx, string(flow.EventAccountCreated), []flow.Identifier{}, version) + expectedResult := generator.GetEventsWithEncoding(1, version) suite.checkResponse(result, err) for _, blockEvent := range result { @@ -1346,14 +1346,14 @@ func (suite *Suite) TestGetEventsForHeightRange() { exeNodeEventEncodingVersion := entitiesproto.EventEncodingVersion_CCF_V0 for i, header := range blockHeaders { - events := getEventsWithEncoding(1, exeNodeEventEncodingVersion) + events := generator.GetEventsWithEncoding(1, exeNodeEventEncodingVersion) height := header.Height results[i] = flow.BlockEvents{ BlockID: header.ID(), BlockHeight: height, BlockTimestamp: header.Timestamp, - Events: getEventsWithEncoding(1, entitiesproto.EventEncodingVersion_JSON_CDC_V0), + Events: generator.GetEventsWithEncoding(1, entitiesproto.EventEncodingVersion_JSON_CDC_V0), } exeResults[i] = &execproto.GetEventsForBlockIDsResponse_Result{ @@ -1514,7 +1514,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { suite.Require().NoError(err) result, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, version) - expectedResult := getEventsWithEncoding(1, version) + expectedResult := generator.GetEventsWithEncoding(1, version) suite.checkResponse(result, err) for _, blockEvent := range result { @@ -1865,7 +1865,7 @@ func (suite *Suite) TestGetTransactionResultEventEncodingVersion() { suite.Require().NoError(err) exeNodeEventEncodingVersion := entitiesproto.EventEncodingVersion_CCF_V0 - events := getEventsWithEncoding(1, exeNodeEventEncodingVersion) + events := generator.GetEventsWithEncoding(1, exeNodeEventEncodingVersion) eventMessages := convert.EventsToMessages(events) for _, version := range eventEncodingVersions { @@ -1884,7 +1884,7 @@ func (suite *Suite) TestGetTransactionResultEventEncodingVersion() { Once() result, err := backend.GetTransactionResult(ctx, txId, blockId, flow.ZeroID, version) - expectedResult := getEventsWithEncoding(1, version) + expectedResult := generator.GetEventsWithEncoding(1, version) suite.checkResponse(result, err) suite.Assert().Equal(result.Events, expectedResult) @@ -1925,7 +1925,7 @@ func (suite *Suite) TestGetTransactionResultByIndexAndBlockIdEventEncodingVersio suite.Require().NoError(err) exeNodeEventEncodingVersion := entitiesproto.EventEncodingVersion_CCF_V0 - events := getEventsWithEncoding(1, exeNodeEventEncodingVersion) + events := generator.GetEventsWithEncoding(1, exeNodeEventEncodingVersion) eventMessages := convert.EventsToMessages(events) for _, version := range eventEncodingVersions { @@ -1946,7 +1946,7 @@ func (suite *Suite) TestGetTransactionResultByIndexAndBlockIdEventEncodingVersio result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, version) suite.checkResponse(result, err) - expectedResult := getEventsWithEncoding(1, version) + expectedResult := generator.GetEventsWithEncoding(1, version) suite.Assert().Equal(result.Events, expectedResult) }) @@ -1970,7 +1970,7 @@ func (suite *Suite) TestGetTransactionResultByIndexAndBlockIdEventEncodingVersio results, err := backend.GetTransactionResultsByBlockID(ctx, blockId, version) suite.checkResponse(results, err) - expectedResult := getEventsWithEncoding(1, version) + expectedResult := generator.GetEventsWithEncoding(1, version) for _, result := range results { suite.Assert().Equal(result.Events, expectedResult) } @@ -1978,24 +1978,6 @@ func (suite *Suite) TestGetTransactionResultByIndexAndBlockIdEventEncodingVersio } } -// getEventsWithEncoding generates a specified number of events with a given encoding version. -func getEventsWithEncoding(n int, version entitiesproto.EventEncodingVersion) []flow.Event { - var eventGenerator *generator.Events - switch version { - case entitiesproto.EventEncodingVersion_CCF_V0: - eventGenerator = generator.EventGenerator(generator.WithEncoding(generator.EncodingCCF)) - case entitiesproto.EventEncodingVersion_JSON_CDC_V0: - eventGenerator = generator.EventGenerator(generator.WithEncoding(generator.EncodingJSON)) - } - - events := make([]flow.Event, 0, n) - for i := 0; i < n; i++ { - events = append(events, eventGenerator.New()) - } - - return events -} - func (suite *Suite) assertAllExpectations() { suite.snapshot.AssertExpectations(suite.T()) suite.state.AssertExpectations(suite.T()) diff --git a/engine/access/state_stream/backend/handler_test.go b/engine/access/state_stream/backend/handler_test.go index e0dc9e34106..0ecc185c2c2 100644 --- a/engine/access/state_stream/backend/handler_test.go +++ b/engine/access/state_stream/backend/handler_test.go @@ -15,6 +15,7 @@ import ( pb "google.golang.org/genproto/googleapis/bytestream" "google.golang.org/grpc" + "github.com/onflow/cadence/encoding/ccf" "github.com/onflow/flow/protobuf/go/flow/entities" jsoncdc "github.com/onflow/cadence/encoding/json" @@ -58,17 +59,8 @@ func (fake *fakeReadServerImpl) Send(response *executiondata.SubscribeEventsResp func (s *HandlerTestSuite) SetupTest() { s.BackendExecutionDataSuite.SetupTest() - - config := Config{ - EventFilterConfig: state_stream.DefaultEventFilterConfig, - ClientSendTimeout: state_stream.DefaultSendTimeout, - ClientSendBufferSize: state_stream.DefaultSendBufferSize, - MaxGlobalStreams: 5, - HeartbeatInterval: state_stream.DefaultHeartbeatInterval, - } - chain := flow.MonotonicEmulator.Chain() - s.handler = NewHandler(s.backend, chain, config) + s.handler = NewHandler(s.backend, chain, makeConfig(5)) } // TestHeartbeatResponse tests the periodic heartbeat response. @@ -199,6 +191,75 @@ func (s *HandlerTestSuite) TestHeartbeatResponse() { }) } +// TestGetExecutionDataByBlockID tests the execution data by block id with different event encoding versions. +func TestGetExecutionDataByBlockID(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ccfEvents := generator.GetEventsWithEncoding(3, entities.EventEncodingVersion_CCF_V0) + jsonEvents := generator.GetEventsWithEncoding(3, entities.EventEncodingVersion_JSON_CDC_V0) + + tests := []struct { + eventVersion entities.EventEncodingVersion + expected []flow.Event + }{ + { + entities.EventEncodingVersion_JSON_CDC_V0, + jsonEvents, + }, + { + entities.EventEncodingVersion_CCF_V0, + ccfEvents, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("test %s event encoding version", test.eventVersion.String()), func(t *testing.T) { + result := unittest.BlockExecutionDataFixture( + unittest.WithChunkExecutionDatas( + unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(ccfEvents)), + unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(ccfEvents)), + ), + ) + blockID := result.BlockID + + api := ssmock.NewAPI(t) + api.On("GetExecutionDataByBlockID", mock.Anything, blockID).Return(result, nil) + + h := NewHandler(api, flow.Localnet.Chain(), makeConfig(1)) + + response, err := h.GetExecutionDataByBlockID(ctx, &executiondata.GetExecutionDataByBlockIDRequest{ + BlockId: blockID[:], + EventEncodingVersion: test.eventVersion, + }) + require.NoError(t, err) + require.NotNil(t, response) + + blockExecutionData := response.GetBlockExecutionData() + + require.Equal(t, blockID[:], blockExecutionData.GetBlockId()) + + convertedExecData, err := convert.MessageToBlockExecutionData(blockExecutionData, flow.Testnet.Chain()) + require.NoError(t, err) + + // Verify that the payload is valid + for _, chunk := range convertedExecData.ChunkExecutionDatas { + for i, e := range chunk.Events { + assert.Equal(t, test.expected[i], e) + + var err error + if test.eventVersion == entities.EventEncodingVersion_JSON_CDC_V0 { + _, err = jsoncdc.Decode(nil, e.Payload) + } else { + _, err = ccf.Decode(nil, e.Payload) + } + require.NoError(t, err) + } + } + }) + } +} + // TestExecutionDataStream tests the execution data stream with different event encoding versions. func TestExecutionDataStream(t *testing.T) { t.Parallel() @@ -206,38 +267,21 @@ func TestExecutionDataStream(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - api := ssmock.NewAPI(t) - stream := makeStreamMock[executiondata.SubscribeExecutionDataRequest, executiondata.SubscribeExecutionDataResponse](ctx) - - // generate some events with a payload to include - // generators will produce identical event payloads (before encoding) - ccfEventGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingCCF)) - jsonEventsGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingJSON)) - ccfInputEvents := make([]flow.Event, 0, 3) - jsonExpectedEvents := make([]flow.Event, 0, 3) - for i := 0; i < 3; i++ { - ccfInputEvents = append(ccfInputEvents, ccfEventGenerator.New()) - jsonExpectedEvents = append(jsonExpectedEvents, jsonEventsGenerator.New()) - } - // Send a single response. blockHeight := uint64(1) - config := Config{ - EventFilterConfig: state_stream.EventFilterConfig{}, - ClientSendTimeout: state_stream.DefaultSendTimeout, - ClientSendBufferSize: state_stream.DefaultSendBufferSize, - MaxGlobalStreams: 1, - HeartbeatInterval: state_stream.DefaultHeartbeatInterval, - } - // Helper function to perform a stream request and handle responses. - makeStreamRequest := func(request *executiondata.SubscribeExecutionDataRequest) { + makeStreamRequest := func( + stream *StreamMock[executiondata.SubscribeExecutionDataRequest, executiondata.SubscribeExecutionDataResponse], + api *ssmock.API, + request *executiondata.SubscribeExecutionDataRequest, + response *ExecutionDataResponse, + ) { sub := NewSubscription(1) api.On("SubscribeExecutionData", mock.Anything, flow.ZeroID, uint64(0), mock.Anything).Return(sub) - h := NewHandler(api, flow.Localnet.Chain(), config) + h := NewHandler(api, flow.Localnet.Chain(), makeConfig(1)) wg := sync.WaitGroup{} wg.Add(1) @@ -249,17 +293,7 @@ func TestExecutionDataStream(t *testing.T) { }() wg.Wait() - executionData := unittest.BlockExecutionDataFixture( - unittest.WithChunkExecutionDatas( - unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(ccfInputEvents)), - unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(ccfInputEvents)), - ), - ) - - err := sub.Send(ctx, &ExecutionDataResponse{ - Height: blockHeight, - ExecutionData: executionData, - }, 100*time.Millisecond) + err := sub.Send(ctx, response, 100*time.Millisecond) require.NoError(t, err) // Notify end of data. @@ -267,17 +301,20 @@ func TestExecutionDataStream(t *testing.T) { } // handleExecutionDataStreamResponses handles responses from the execution data stream. - handleExecutionDataStreamResponses := func(version entities.EventEncodingVersion, expectedEvents []flow.Event) { - receivedCount := 0 + handleExecutionDataStreamResponses := func( + stream *StreamMock[executiondata.SubscribeExecutionDataRequest, executiondata.SubscribeExecutionDataResponse], + version entities.EventEncodingVersion, + expectedEvents []flow.Event, + ) { var responses []*executiondata.SubscribeExecutionDataResponse for { + t.Log(len(responses)) resp, err := stream.RecvToClient() if err == io.EOF { break } require.NoError(t, err) responses = append(responses, resp) - receivedCount++ close(stream.sentFromServer) } @@ -287,136 +324,200 @@ func TestExecutionDataStream(t *testing.T) { assert.Equal(t, blockHeight, resp.GetBlockHeight()) + // only expect a single response + assert.Equal(t, 1, len(responses)) + // Verify that the payload is valid for _, chunk := range convertedExecData.ChunkExecutionDatas { for i, e := range chunk.Events { assert.Equal(t, expectedEvents[i], e) + var err error if version == entities.EventEncodingVersion_JSON_CDC_V0 { - _, err := jsoncdc.Decode(nil, e.Payload) - require.NoError(t, err) + _, err = jsoncdc.Decode(nil, e.Payload) + } else { + _, err = ccf.Decode(nil, e.Payload) } + require.NoError(t, err) } } - - // only expect a single response - assert.Equal(t, 1, receivedCount) } } + ccfEvents := generator.GetEventsWithEncoding(3, entities.EventEncodingVersion_CCF_V0) + jsonEvents := generator.GetEventsWithEncoding(3, entities.EventEncodingVersion_JSON_CDC_V0) + tests := []struct { eventVersion entities.EventEncodingVersion expected []flow.Event }{ { entities.EventEncodingVersion_JSON_CDC_V0, - jsonExpectedEvents, + jsonEvents, }, { entities.EventEncodingVersion_CCF_V0, - ccfInputEvents, + ccfEvents, }, } for _, test := range tests { - t.Run(fmt.Sprintf("test %s event encoding version add ."+ - "", test.eventVersion.String()), func(t *testing.T) { - makeStreamRequest(&executiondata.SubscribeExecutionDataRequest{ - EventEncodingVersion: test.eventVersion, - }) - handleExecutionDataStreamResponses(test.eventVersion, test.expected) + t.Run(fmt.Sprintf("test %s event encoding version", test.eventVersion.String()), func(t *testing.T) { + api := ssmock.NewAPI(t) + stream := makeStreamMock[executiondata.SubscribeExecutionDataRequest, executiondata.SubscribeExecutionDataResponse](ctx) + + makeStreamRequest( + stream, + api, + &executiondata.SubscribeExecutionDataRequest{ + EventEncodingVersion: test.eventVersion, + }, + &ExecutionDataResponse{ + Height: blockHeight, + ExecutionData: unittest.BlockExecutionDataFixture( + unittest.WithChunkExecutionDatas( + unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(ccfEvents)), + unittest.ChunkExecutionDataFixture(t, 1024, unittest.WithChunkEvents(ccfEvents)), + ), + ), + }, + ) + handleExecutionDataStreamResponses(stream, test.eventVersion, test.expected) }) } - } +// TestEventStream tests the event stream with different event encoding versions. func TestEventStream(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - api := ssmock.NewAPI(t) - stream := makeStreamMock[executiondata.SubscribeEventsRequest, executiondata.SubscribeEventsResponse](ctx) - sub := NewSubscription(1) + blockHeight := uint64(1) + blockID := unittest.IdentifierFixture() - config := Config{ - EventFilterConfig: state_stream.EventFilterConfig{}, - ClientSendTimeout: state_stream.DefaultSendTimeout, - ClientSendBufferSize: state_stream.DefaultSendBufferSize, - MaxGlobalStreams: 1, - HeartbeatInterval: state_stream.DefaultHeartbeatInterval, - } + // Helper function to perform a stream request and handle responses. + makeStreamRequest := func( + stream *StreamMock[executiondata.SubscribeEventsRequest, executiondata.SubscribeEventsResponse], + api *ssmock.API, + request *executiondata.SubscribeEventsRequest, + response *EventsResponse, + ) { + sub := NewSubscription(1) - // generate some events with a payload to include - // generators will produce identical event payloads (before encoding) - ccfEventGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingCCF)) - jsonEventsGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingJSON)) - inputEvents := make([]flow.Event, 0, 3) - expectedEvents := make([]flow.Event, 0, 3) - for i := 0; i < 3; i++ { - inputEvents = append(inputEvents, ccfEventGenerator.New()) - expectedEvents = append(expectedEvents, jsonEventsGenerator.New()) - } + api.On("SubscribeEvents", mock.Anything, flow.ZeroID, uint64(0), mock.Anything).Return(sub) - api.On("SubscribeEvents", mock.Anything, flow.ZeroID, uint64(0), mock.Anything).Return(sub) + h := NewHandler(api, flow.Localnet.Chain(), makeConfig(1)) - h := NewHandler(api, flow.Localnet.Chain(), config) + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + wg.Done() + err := h.SubscribeEvents(request, stream) + require.NoError(t, err) + t.Log("subscription closed") + }() + wg.Wait() - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - wg.Done() - err := h.SubscribeEvents(&executiondata.SubscribeEventsRequest{ - EventEncodingVersion: entities.EventEncodingVersion_JSON_CDC_V0, - }, stream) + // send a single response + err := sub.Send(ctx, response, 100*time.Millisecond) require.NoError(t, err) - t.Log("subscription closed") - }() - wg.Wait() - // send a single response - blockHeight := uint64(1) - blockID := unittest.IdentifierFixture() - err := sub.Send(ctx, &EventsResponse{ - BlockID: blockID, - Height: blockHeight, - Events: inputEvents, - }, 100*time.Millisecond) - require.NoError(t, err) - - // notify end of data - sub.Close() - - receivedCount := 0 - for { - t.Log(receivedCount) - resp, err := stream.RecvToClient() - if err == io.EOF { - break - } - require.NoError(t, err) + // notify end of data + sub.Close() + } - convertedEvents := convert.MessagesToEvents(resp.GetEvents()) + // handleExecutionDataStreamResponses handles responses from the execution data stream. + handleExecutionDataStreamResponses := func( + stream *StreamMock[executiondata.SubscribeEventsRequest, executiondata.SubscribeEventsResponse], + version entities.EventEncodingVersion, + expectedEvents []flow.Event, + ) { + var responses []*executiondata.SubscribeEventsResponse + for { + t.Log(len(responses)) + resp, err := stream.RecvToClient() + if err == io.EOF { + break + } + // make sure the payload is valid + require.NoError(t, err) + responses = append(responses, resp) - assert.Equal(t, blockHeight, resp.GetBlockHeight()) - assert.Equal(t, blockID, convert.MessageToIdentifier(resp.GetBlockId())) - assert.Equal(t, expectedEvents, convertedEvents) + // shutdown the stream after one response + close(stream.sentFromServer) + } - // make sure the payload is valid JSON-CDC - for _, e := range convertedEvents { - _, err := jsoncdc.Decode(nil, e.Payload) - require.NoError(t, err) + for _, resp := range responses { + convertedEvents := convert.MessagesToEvents(resp.GetEvents()) + + assert.Equal(t, blockHeight, resp.GetBlockHeight()) + assert.Equal(t, blockID, convert.MessageToIdentifier(resp.GetBlockId())) + assert.Equal(t, expectedEvents, convertedEvents) + // only expect a single response + assert.Equal(t, 1, len(responses)) + + for _, e := range convertedEvents { + var err error + if version == entities.EventEncodingVersion_JSON_CDC_V0 { + _, err = jsoncdc.Decode(nil, e.Payload) + } else { + _, err = ccf.Decode(nil, e.Payload) + } + require.NoError(t, err) + } } + } + + // generate events with a payload to include + // generators will produce identical event payloads (before encoding) + ccfEvents := generator.GetEventsWithEncoding(3, entities.EventEncodingVersion_CCF_V0) + jsonEvents := generator.GetEventsWithEncoding(3, entities.EventEncodingVersion_JSON_CDC_V0) - receivedCount++ + tests := []struct { + eventVersion entities.EventEncodingVersion + expected []flow.Event + }{ + { + entities.EventEncodingVersion_JSON_CDC_V0, + jsonEvents, + }, + { + entities.EventEncodingVersion_CCF_V0, + ccfEvents, + }, + } - // shutdown the stream after one response - close(stream.sentFromServer) + for _, test := range tests { + t.Run(fmt.Sprintf("test %s event encoding version", test.eventVersion.String()), func(t *testing.T) { + stream := makeStreamMock[executiondata.SubscribeEventsRequest, executiondata.SubscribeEventsResponse](ctx) + + makeStreamRequest( + stream, + ssmock.NewAPI(t), + &executiondata.SubscribeEventsRequest{ + EventEncodingVersion: test.eventVersion, + }, + &EventsResponse{ + BlockID: blockID, + Height: blockHeight, + Events: ccfEvents, + }, + ) + handleExecutionDataStreamResponses(stream, test.eventVersion, test.expected) + }) } +} - // only expect a single response - assert.Equal(t, 1, receivedCount) +func makeConfig(maxGlobalStreams uint32) Config { + return Config{ + EventFilterConfig: state_stream.DefaultEventFilterConfig, + ClientSendTimeout: state_stream.DefaultSendTimeout, + ClientSendBufferSize: state_stream.DefaultSendBufferSize, + MaxGlobalStreams: maxGlobalStreams, + HeartbeatInterval: state_stream.DefaultHeartbeatInterval, + } } func makeStreamMock[R, T any](ctx context.Context) *StreamMock[R, T] { diff --git a/engine/common/rpc/convert/execution_data_test.go b/engine/common/rpc/convert/execution_data_test.go index d281cb012ff..1f3f60d413c 100644 --- a/engine/common/rpc/convert/execution_data_test.go +++ b/engine/common/rpc/convert/execution_data_test.go @@ -21,15 +21,8 @@ import ( func TestConvertBlockExecutionDataEventPayloads(t *testing.T) { // generators will produce identical event payloads (before encoding) - ccfEventGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingCCF)) - jsonEventsGenerator := generator.EventGenerator(generator.WithEncoding(generator.EncodingJSON)) - - ccfEvents := make([]flow.Event, 0, 3) - jsonEvents := make([]flow.Event, 0, 3) - for i := 0; i < 3; i++ { - ccfEvents = append(ccfEvents, ccfEventGenerator.New()) - jsonEvents = append(jsonEvents, jsonEventsGenerator.New()) - } + ccfEvents := generator.GetEventsWithEncoding(3, entities.EventEncodingVersion_CCF_V0) + jsonEvents := generator.GetEventsWithEncoding(3, entities.EventEncodingVersion_JSON_CDC_V0) // generate BlockExecutionData with CCF encoded events executionData := unittest.BlockExecutionDataFixture( diff --git a/utils/unittest/generator/events.go b/utils/unittest/generator/events.go index e0878e9073c..ea5a86b9c93 100644 --- a/utils/unittest/generator/events.go +++ b/utils/unittest/generator/events.go @@ -7,25 +7,15 @@ import ( "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" "github.com/onflow/cadence/runtime/common" + "github.com/onflow/flow/protobuf/go/flow/entities" "github.com/onflow/flow-go/model/flow" ) -type EventEncoding string - -const ( - EncodingCCF EventEncoding = "ccf" - EncodingJSON EventEncoding = "json" -) - type EventGeneratorOption func(*Events) -func WithEncoding(encoding EventEncoding) EventGeneratorOption { +func WithEncoding(encoding entities.EventEncodingVersion) EventGeneratorOption { return func(g *Events) { - if encoding != EncodingCCF && encoding != EncodingJSON { - panic(fmt.Sprintf("unexpected encoding: %s", encoding)) - } - g.encoding = encoding } } @@ -33,14 +23,14 @@ func WithEncoding(encoding EventEncoding) EventGeneratorOption { type Events struct { count uint32 ids *Identifiers - encoding EventEncoding + encoding entities.EventEncodingVersion } func EventGenerator(opts ...EventGeneratorOption) *Events { g := &Events{ count: 1, ids: IdentifierGenerator(), - encoding: EncodingCCF, + encoding: entities.EventEncodingVersion_CCF_V0, } for _, opt := range opts { @@ -83,12 +73,12 @@ func (g *Events) New() flow.Event { var payload []byte switch g.encoding { - case EncodingCCF: + case entities.EventEncodingVersion_CCF_V0: payload, err = ccf.Encode(testEvent) if err != nil { panic(fmt.Sprintf("unexpected error while ccf encoding events: %s", err)) } - case EncodingJSON: + case entities.EventEncodingVersion_JSON_CDC_V0: payload, err = jsoncdc.Encode(testEvent) if err != nil { panic(fmt.Sprintf("unexpected error while json encoding events: %s", err)) @@ -107,3 +97,15 @@ func (g *Events) New() flow.Event { return event } + +// GetEventsWithEncoding generates a specified number of events with a given encoding version. +func GetEventsWithEncoding(n int, version entities.EventEncodingVersion) []flow.Event { + eventGenerator := EventGenerator(WithEncoding(version)) + + events := make([]flow.Event, 0, n) + for i := 0; i < n; i++ { + events = append(events, eventGenerator.New()) + } + + return events +} From 9e825ebb9f507478b3025ff88aaf571a4ccee47e Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Wed, 25 Oct 2023 16:19:54 +0300 Subject: [PATCH 25/27] Changed flow-emulator version --- integration/go.mod | 6 ++---- integration/go.sum | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/integration/go.mod b/integration/go.mod index 48edbb4748c..eb85f3b08e4 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -21,8 +21,8 @@ require ( github.com/onflow/cadence v0.42.1 github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20231016154253-a00dbf7c061f github.com/onflow/flow-core-contracts/lib/go/templates v1.2.4-0.20231016154253-a00dbf7c061f - github.com/onflow/flow-emulator v0.54.1 - github.com/onflow/flow-go v0.31.1-0.20230808172820-f074502a67e3 + github.com/onflow/flow-emulator v0.54.1-0.20231024204057-0273f8fe3807 + github.com/onflow/flow-go v0.32.3 github.com/onflow/flow-go-sdk v0.41.12 github.com/onflow/flow-go/crypto v0.24.9 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 @@ -345,5 +345,3 @@ require ( replace github.com/onflow/flow-go => ../ replace github.com/onflow/flow-go/insecure => ../insecure - -replace github.com/onflow/flow-emulator v0.54.1 => github.com/Guitarheroua/flow-emulator v0.0.0-20231018151849-2b2f7bcd6fa2 diff --git a/integration/go.sum b/integration/go.sum index 4778d28fcc3..da76c90b2b1 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -100,8 +100,6 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Guitarheroua/flow-emulator v0.0.0-20231018151849-2b2f7bcd6fa2 h1:WIoMCjOzGI2WyVWsTiVmWzTfnPKoaq0jGDj0XjuFBwE= -github.com/Guitarheroua/flow-emulator v0.0.0-20231018151849-2b2f7bcd6fa2/go.mod h1:m0BlKMdFBteEH3oepU1yeerU4kvpTctvO+6mkmF9nlU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= @@ -1436,6 +1434,8 @@ github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20231016154253-a github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20231016154253-a00dbf7c061f/go.mod h1:jM6GMAL+m0hjusUgiYDNrixPQ6b9s8xjoJQoEu5bHQI= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.4-0.20231016154253-a00dbf7c061f h1:Ep+Mpo2miWMe4pjPGIaEvEzshRep30dvNgxqk+//FrQ= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.4-0.20231016154253-a00dbf7c061f/go.mod h1:ZeLxwaBkzuSInESGjL8/IPZWezF+YOYsYbMrZlhN+q4= +github.com/onflow/flow-emulator v0.54.1-0.20231024204057-0273f8fe3807 h1:/4jZ2oELdhKubgL97NGqhiuO80oMH/M+fIQoNPfGg+g= +github.com/onflow/flow-emulator v0.54.1-0.20231024204057-0273f8fe3807/go.mod h1:Qq1YmTDYlfpzfuzrFH8gwMgzzv80LCKFiS1Kqm8vFcY= github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13 h1:B4ll7e3j+MqTJv2122Enq3RtDNzmIGRu9xjV7fo7un0= github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20230711213910-baad011d2b13/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= From 2beb56afb65ae8caba2b18f373abb3bdaa1864f2 Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Fri, 27 Oct 2023 11:51:01 +0300 Subject: [PATCH 26/27] Apply suggestions from code review Co-authored-by: Yurii Oleksyshyn --- engine/access/state_stream/backend/handler_test.go | 1 - utils/unittest/generator/events.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/engine/access/state_stream/backend/handler_test.go b/engine/access/state_stream/backend/handler_test.go index 0ecc185c2c2..b9a0c7aef5c 100644 --- a/engine/access/state_stream/backend/handler_test.go +++ b/engine/access/state_stream/backend/handler_test.go @@ -236,7 +236,6 @@ func TestGetExecutionDataByBlockID(t *testing.T) { require.NotNil(t, response) blockExecutionData := response.GetBlockExecutionData() - require.Equal(t, blockID[:], blockExecutionData.GetBlockId()) convertedExecData, err := convert.MessageToBlockExecutionData(blockExecutionData, flow.Testnet.Chain()) diff --git a/utils/unittest/generator/events.go b/utils/unittest/generator/events.go index ea5a86b9c93..ac86cd43136 100644 --- a/utils/unittest/generator/events.go +++ b/utils/unittest/generator/events.go @@ -101,11 +101,9 @@ func (g *Events) New() flow.Event { // GetEventsWithEncoding generates a specified number of events with a given encoding version. func GetEventsWithEncoding(n int, version entities.EventEncodingVersion) []flow.Event { eventGenerator := EventGenerator(WithEncoding(version)) - events := make([]flow.Event, 0, n) for i := 0; i < n; i++ { events = append(events, eventGenerator.New()) } - return events } From 36c93d8fb927e4e4827ec519c29cd56e4f27fb7b Mon Sep 17 00:00:00 2001 From: Andrii Slisarchuk Date: Fri, 27 Oct 2023 11:58:10 +0300 Subject: [PATCH 27/27] fixed import ordering --- engine/access/rpc/backend/retry_test.go | 3 +-- engine/access/state_stream/backend/handler_test.go | 3 +-- engine/common/rpc/convert/events_test.go | 3 +-- engine/common/rpc/convert/execution_data_test.go | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/engine/access/rpc/backend/retry_test.go b/engine/access/rpc/backend/retry_test.go index 01874185bbf..3b705049ff5 100644 --- a/engine/access/rpc/backend/retry_test.go +++ b/engine/access/rpc/backend/retry_test.go @@ -3,9 +3,8 @@ package backend import ( "context" - "github.com/onflow/flow/protobuf/go/flow/entities" - "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/onflow/flow/protobuf/go/flow/entities" "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/stretchr/testify/mock" "google.golang.org/grpc/codes" diff --git a/engine/access/state_stream/backend/handler_test.go b/engine/access/state_stream/backend/handler_test.go index b9a0c7aef5c..6c53c1b25f8 100644 --- a/engine/access/state_stream/backend/handler_test.go +++ b/engine/access/state_stream/backend/handler_test.go @@ -16,9 +16,8 @@ import ( "google.golang.org/grpc" "github.com/onflow/cadence/encoding/ccf" - "github.com/onflow/flow/protobuf/go/flow/entities" - jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/onflow/flow/protobuf/go/flow/entities" "github.com/onflow/flow/protobuf/go/flow/executiondata" "github.com/onflow/flow-go/engine/access/state_stream" diff --git a/engine/common/rpc/convert/events_test.go b/engine/common/rpc/convert/events_test.go index 420fad947bb..a40d8d24125 100644 --- a/engine/common/rpc/convert/events_test.go +++ b/engine/common/rpc/convert/events_test.go @@ -3,14 +3,13 @@ package convert_test import ( "testing" - "github.com/onflow/flow/protobuf/go/flow/entities" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/onflow/cadence" "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/onflow/flow/protobuf/go/flow/entities" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" diff --git a/engine/common/rpc/convert/execution_data_test.go b/engine/common/rpc/convert/execution_data_test.go index 1f3f60d413c..99744f9d8cc 100644 --- a/engine/common/rpc/convert/execution_data_test.go +++ b/engine/common/rpc/convert/execution_data_test.go @@ -3,13 +3,12 @@ package convert_test import ( "testing" - "github.com/onflow/flow/protobuf/go/flow/entities" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/onflow/flow/protobuf/go/flow/entities" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/ledger/common/testutils"