From 50a308d97d15b0833b7a5c08dd9c56996e051d31 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 7 Aug 2020 15:59:20 -0700 Subject: [PATCH 01/10] Move internal to pkg --- Makefile | 14 +++++++------- cmd/check_construction.go | 4 ++-- cmd/check_data.go | 4 ++-- cmd/configuration_create.go | 2 +- cmd/root.go | 2 +- cmd/utils_asserter_configuration.go | 2 +- cmd/view_account.go | 2 +- cmd/view_block.go | 2 +- configuration/configuration.go | 4 ++-- configuration/configuration_test.go | 2 +- mocks/constructor/handler.go | 14 -------------- mocks/constructor/helper.go | 16 +--------------- {internal => pkg}/constructor/constructor.go | 8 ++++---- .../constructor/constructor_test.go | 4 ++-- {internal => pkg}/logger/logger.go | 6 +++--- .../processor/balance_storage_handler.go | 4 ++-- .../processor/balance_storage_helper.go | 2 +- .../processor/balance_storage_helper_test.go | 0 .../processor/broadcast_storage_handler.go | 2 +- .../processor/broadcast_storage_helper.go | 2 +- .../processor/constructor_handler.go | 4 ++-- .../processor/constructor_helper.go | 6 +++--- .../processor/reconciler_handler.go | 4 ++-- {internal => pkg}/processor/reconciler_helper.go | 2 +- {internal => pkg}/scenario/scenario.go | 0 {internal => pkg}/scenario/scenario_test.go | 0 .../statefulsyncer/stateful_syncer.go | 4 ++-- {internal => pkg}/storage/badger_storage.go | 0 {internal => pkg}/storage/badger_storage_test.go | 2 +- {internal => pkg}/storage/balance_storage.go | 2 +- .../storage/balance_storage_test.go | 2 +- {internal => pkg}/storage/block_storage.go | 2 +- {internal => pkg}/storage/block_storage_test.go | 2 +- {internal => pkg}/storage/broadcast_storage.go | 0 .../storage/broadcast_storage_test.go | 2 +- {internal => pkg}/storage/coin_storage.go | 0 {internal => pkg}/storage/coin_storage_test.go | 2 +- {internal => pkg}/storage/counter_storage.go | 0 .../storage/counter_storage_test.go | 2 +- {internal => pkg}/storage/key_storage.go | 0 {internal => pkg}/storage/key_storage_test.go | 2 +- {internal => pkg}/storage/storage.go | 0 {internal => pkg}/storage/utils.go | 0 {internal => pkg}/tester/construction.go | 12 ++++++------ {internal => pkg}/tester/data.go | 10 +++++----- {internal => pkg}/utils/utils.go | 0 {internal => pkg}/utils/utils_test.go | 0 47 files changed, 64 insertions(+), 92 deletions(-) rename {internal => pkg}/constructor/constructor.go (99%) rename {internal => pkg}/constructor/constructor_test.go (99%) rename {internal => pkg}/logger/logger.go (98%) rename {internal => pkg}/processor/balance_storage_handler.go (96%) rename {internal => pkg}/processor/balance_storage_helper.go (98%) rename {internal => pkg}/processor/balance_storage_helper_test.go (100%) rename {internal => pkg}/processor/broadcast_storage_handler.go (98%) rename {internal => pkg}/processor/broadcast_storage_helper.go (98%) rename {internal => pkg}/processor/constructor_handler.go (94%) rename {internal => pkg}/processor/constructor_helper.go (97%) rename {internal => pkg}/processor/reconciler_handler.go (96%) rename {internal => pkg}/processor/reconciler_helper.go (97%) rename {internal => pkg}/scenario/scenario.go (100%) rename {internal => pkg}/scenario/scenario_test.go (100%) rename {internal => pkg}/statefulsyncer/stateful_syncer.go (98%) rename {internal => pkg}/storage/badger_storage.go (100%) rename {internal => pkg}/storage/badger_storage_test.go (98%) rename {internal => pkg}/storage/balance_storage.go (99%) rename {internal => pkg}/storage/balance_storage_test.go (99%) rename {internal => pkg}/storage/block_storage.go (99%) rename {internal => pkg}/storage/block_storage_test.go (99%) rename {internal => pkg}/storage/broadcast_storage.go (100%) rename {internal => pkg}/storage/broadcast_storage_test.go (99%) rename {internal => pkg}/storage/coin_storage.go (100%) rename {internal => pkg}/storage/coin_storage_test.go (99%) rename {internal => pkg}/storage/counter_storage.go (100%) rename {internal => pkg}/storage/counter_storage_test.go (97%) rename {internal => pkg}/storage/key_storage.go (100%) rename {internal => pkg}/storage/key_storage_test.go (98%) rename {internal => pkg}/storage/storage.go (100%) rename {internal => pkg}/storage/utils.go (100%) rename {internal => pkg}/tester/construction.go (95%) rename {internal => pkg}/tester/data.go (98%) rename {internal => pkg}/utils/utils.go (100%) rename {internal => pkg}/utils/utils_test.go (100%) diff --git a/Makefile b/Makefile index d4823f68..7d12b765 100644 --- a/Makefile +++ b/Makefile @@ -11,11 +11,11 @@ ADDLICENSE_CMD=go run github.com/google/addlicense ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v GOLINES_CMD=go run github.com/segmentio/golines GOVERALLS_CMD=go run github.com/mattn/goveralls -COVERAGE_TEST_DIRECTORIES=./configuration/... ./internal/constructor/... \ - ./internal/logger/... ./internal/scenario/... \ - ./internal/statefulsyncer/... ./internal/storage/... \ - ./internal/tester/... ./internal/utils/... -TEST_SCRIPT=go test -v ./internal/... ./configuration/... +COVERAGE_TEST_DIRECTORIES=./configuration/... ./pkg/constructor/... \ + ./pkg/logger/... ./pkg/scenario/... \ + ./pkg/statefulsyncer/... ./pkg/storage/... \ + ./pkg/tester/... ./pkg/utils/... +TEST_SCRIPT=go test -v ./pkg/... ./configuration/... COVERAGE_TEST_SCRIPT=go test -v ${COVERAGE_TEST_DIRECTORIES} deps: @@ -44,7 +44,7 @@ check-license: ${ADDLICENCE_SCRIPT} -check . shorten-lines: - ${GOLINES_CMD} -w --shorten-comments internal cmd configuration + ${GOLINES_CMD} -w --shorten-comments pkg cmd configuration salus: docker run --rm -t -v ${PWD}:/home/repo coinbase/salus @@ -59,4 +59,4 @@ install: mocks: rm -rf mocks; - mockery --dir internal/constructor --all --case underscore --outpkg constructor --output mocks/constructor; + mockery --dir pkg/constructor --all --case underscore --outpkg constructor --output mocks/constructor; diff --git a/cmd/check_construction.go b/cmd/check_construction.go index d6c42294..f219da99 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -20,8 +20,8 @@ import ( "os" "time" - "github.com/coinbase/rosetta-cli/internal/tester" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/tester" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/fatih/color" diff --git a/cmd/check_data.go b/cmd/check_data.go index c2359e68..277a2e6f 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -19,8 +19,8 @@ import ( "log" "time" - "github.com/coinbase/rosetta-cli/internal/tester" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/tester" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/spf13/cobra" diff --git a/cmd/configuration_create.go b/cmd/configuration_create.go index 4f4ddf30..e76ee48a 100644 --- a/cmd/configuration_create.go +++ b/cmd/configuration_create.go @@ -18,7 +18,7 @@ import ( "log" "github.com/coinbase/rosetta-cli/configuration" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/spf13/cobra" ) diff --git a/cmd/root.go b/cmd/root.go index 2c4e62a6..bc046abe 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -24,7 +24,7 @@ import ( "time" "github.com/coinbase/rosetta-cli/configuration" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/fatih/color" "github.com/spf13/cobra" diff --git a/cmd/utils_asserter_configuration.go b/cmd/utils_asserter_configuration.go index 5df7bc53..1f8ef02f 100644 --- a/cmd/utils_asserter_configuration.go +++ b/cmd/utils_asserter_configuration.go @@ -19,7 +19,7 @@ import ( "log" "time" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/spf13/cobra" diff --git a/cmd/view_account.go b/cmd/view_account.go index 54e2c30c..64b4a5cb 100644 --- a/cmd/view_account.go +++ b/cmd/view_account.go @@ -22,7 +22,7 @@ import ( "strconv" "time" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" diff --git a/cmd/view_block.go b/cmd/view_block.go index 6d34ac34..83d3a7c3 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -21,7 +21,7 @@ import ( "strconv" "time" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/parser" diff --git a/configuration/configuration.go b/configuration/configuration.go index 5bcaa638..8e743cc7 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -19,8 +19,8 @@ import ( "log" "math/big" - "github.com/coinbase/rosetta-cli/internal/scenario" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/scenario" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/types" diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 899bfcd4..b88279a4 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -19,7 +19,7 @@ import ( "os" "testing" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/types" "github.com/stretchr/testify/assert" diff --git a/mocks/constructor/handler.go b/mocks/constructor/handler.go index 8cef1737..4afe30ef 100644 --- a/mocks/constructor/handler.go +++ b/mocks/constructor/handler.go @@ -1,17 +1,3 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - // Code generated by mockery v1.0.0. DO NOT EDIT. package constructor diff --git a/mocks/constructor/helper.go b/mocks/constructor/helper.go index afad1945..ef940fb2 100644 --- a/mocks/constructor/helper.go +++ b/mocks/constructor/helper.go @@ -1,17 +1,3 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - // Code generated by mockery v1.0.0. DO NOT EDIT. package constructor @@ -25,7 +11,7 @@ import ( mock "github.com/stretchr/testify/mock" - storage "github.com/coinbase/rosetta-cli/internal/storage" + storage "github.com/coinbase/rosetta-cli/pkg/storage" types "github.com/coinbase/rosetta-sdk-go/types" ) diff --git a/internal/constructor/constructor.go b/pkg/constructor/constructor.go similarity index 99% rename from internal/constructor/constructor.go rename to pkg/constructor/constructor.go index fdb3cd9f..fa740e70 100644 --- a/internal/constructor/constructor.go +++ b/pkg/constructor/constructor.go @@ -23,10 +23,10 @@ import ( "time" "github.com/coinbase/rosetta-cli/configuration" - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/scenario" - "github.com/coinbase/rosetta-cli/internal/storage" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/logger" + "github.com/coinbase/rosetta-cli/pkg/scenario" + "github.com/coinbase/rosetta-cli/pkg/storage" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/keys" "github.com/coinbase/rosetta-sdk-go/parser" diff --git a/internal/constructor/constructor_test.go b/pkg/constructor/constructor_test.go similarity index 99% rename from internal/constructor/constructor_test.go rename to pkg/constructor/constructor_test.go index a508d9ef..f0d8f0c1 100644 --- a/internal/constructor/constructor_test.go +++ b/pkg/constructor/constructor_test.go @@ -22,9 +22,9 @@ import ( "time" "github.com/coinbase/rosetta-cli/configuration" - "github.com/coinbase/rosetta-cli/internal/scenario" - "github.com/coinbase/rosetta-cli/internal/storage" mocks "github.com/coinbase/rosetta-cli/mocks/constructor" + "github.com/coinbase/rosetta-cli/pkg/scenario" + "github.com/coinbase/rosetta-cli/pkg/storage" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/parser" diff --git a/internal/logger/logger.go b/pkg/logger/logger.go similarity index 98% rename from internal/logger/logger.go rename to pkg/logger/logger.go index a1c1691d..758fcfd4 100644 --- a/internal/logger/logger.go +++ b/pkg/logger/logger.go @@ -22,9 +22,9 @@ import ( "os" "path" - "github.com/coinbase/rosetta-cli/internal/scenario" - "github.com/coinbase/rosetta-cli/internal/storage" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/scenario" + "github.com/coinbase/rosetta-cli/pkg/storage" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/reconciler" diff --git a/internal/processor/balance_storage_handler.go b/pkg/processor/balance_storage_handler.go similarity index 96% rename from internal/processor/balance_storage_handler.go rename to pkg/processor/balance_storage_handler.go index a276e76d..72cf2dd7 100644 --- a/internal/processor/balance_storage_handler.go +++ b/pkg/processor/balance_storage_handler.go @@ -17,8 +17,8 @@ package processor import ( "context" - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/pkg/logger" + "github.com/coinbase/rosetta-cli/pkg/storage" "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/reconciler" diff --git a/internal/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go similarity index 98% rename from internal/processor/balance_storage_helper.go rename to pkg/processor/balance_storage_helper.go index e22fbd57..0bc91852 100644 --- a/internal/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -18,7 +18,7 @@ import ( "context" "fmt" - "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/pkg/storage" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" diff --git a/internal/processor/balance_storage_helper_test.go b/pkg/processor/balance_storage_helper_test.go similarity index 100% rename from internal/processor/balance_storage_helper_test.go rename to pkg/processor/balance_storage_helper_test.go diff --git a/internal/processor/broadcast_storage_handler.go b/pkg/processor/broadcast_storage_handler.go similarity index 98% rename from internal/processor/broadcast_storage_handler.go rename to pkg/processor/broadcast_storage_handler.go index 8c6514b1..f579e339 100644 --- a/internal/processor/broadcast_storage_handler.go +++ b/pkg/processor/broadcast_storage_handler.go @@ -20,7 +20,7 @@ import ( "math/big" "github.com/coinbase/rosetta-cli/configuration" - "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/pkg/storage" "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/types" diff --git a/internal/processor/broadcast_storage_helper.go b/pkg/processor/broadcast_storage_helper.go similarity index 98% rename from internal/processor/broadcast_storage_helper.go rename to pkg/processor/broadcast_storage_helper.go index 616b5c66..7052b482 100644 --- a/internal/processor/broadcast_storage_helper.go +++ b/pkg/processor/broadcast_storage_helper.go @@ -18,7 +18,7 @@ import ( "context" "fmt" - "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/pkg/storage" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" diff --git a/internal/processor/constructor_handler.go b/pkg/processor/constructor_handler.go similarity index 94% rename from internal/processor/constructor_handler.go rename to pkg/processor/constructor_handler.go index 802ee895..67c00951 100644 --- a/internal/processor/constructor_handler.go +++ b/pkg/processor/constructor_handler.go @@ -18,8 +18,8 @@ import ( "context" "math/big" - "github.com/coinbase/rosetta-cli/internal/constructor" - "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/pkg/constructor" + "github.com/coinbase/rosetta-cli/pkg/storage" "github.com/coinbase/rosetta-sdk-go/types" ) diff --git a/internal/processor/constructor_helper.go b/pkg/processor/constructor_helper.go similarity index 97% rename from internal/processor/constructor_helper.go rename to pkg/processor/constructor_helper.go index 15c785ca..c1e9af3c 100644 --- a/internal/processor/constructor_helper.go +++ b/pkg/processor/constructor_helper.go @@ -19,9 +19,9 @@ import ( "fmt" "math/big" - "github.com/coinbase/rosetta-cli/internal/constructor" - "github.com/coinbase/rosetta-cli/internal/storage" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/constructor" + "github.com/coinbase/rosetta-cli/pkg/storage" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/keys" diff --git a/internal/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go similarity index 96% rename from internal/processor/reconciler_handler.go rename to pkg/processor/reconciler_handler.go index 01ac8153..18fd81dd 100644 --- a/internal/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -19,8 +19,8 @@ import ( "errors" "math/big" - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/pkg/logger" + "github.com/coinbase/rosetta-cli/pkg/storage" "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/types" diff --git a/internal/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go similarity index 97% rename from internal/processor/reconciler_helper.go rename to pkg/processor/reconciler_helper.go index 304058de..85ef8bc9 100644 --- a/internal/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -18,7 +18,7 @@ import ( "context" "errors" - "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/pkg/storage" "github.com/coinbase/rosetta-sdk-go/types" ) diff --git a/internal/scenario/scenario.go b/pkg/scenario/scenario.go similarity index 100% rename from internal/scenario/scenario.go rename to pkg/scenario/scenario.go diff --git a/internal/scenario/scenario_test.go b/pkg/scenario/scenario_test.go similarity index 100% rename from internal/scenario/scenario_test.go rename to pkg/scenario/scenario_test.go diff --git a/internal/statefulsyncer/stateful_syncer.go b/pkg/statefulsyncer/stateful_syncer.go similarity index 98% rename from internal/statefulsyncer/stateful_syncer.go rename to pkg/statefulsyncer/stateful_syncer.go index e88d2ba3..f42f2d2c 100644 --- a/internal/statefulsyncer/stateful_syncer.go +++ b/pkg/statefulsyncer/stateful_syncer.go @@ -21,8 +21,8 @@ import ( "math/big" "time" - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/pkg/logger" + "github.com/coinbase/rosetta-cli/pkg/storage" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/syncer" diff --git a/internal/storage/badger_storage.go b/pkg/storage/badger_storage.go similarity index 100% rename from internal/storage/badger_storage.go rename to pkg/storage/badger_storage.go diff --git a/internal/storage/badger_storage_test.go b/pkg/storage/badger_storage_test.go similarity index 98% rename from internal/storage/badger_storage_test.go rename to pkg/storage/badger_storage_test.go index 5d71bbf4..8b44ac1d 100644 --- a/internal/storage/badger_storage_test.go +++ b/pkg/storage/badger_storage_test.go @@ -19,7 +19,7 @@ import ( "fmt" "testing" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/stretchr/testify/assert" ) diff --git a/internal/storage/balance_storage.go b/pkg/storage/balance_storage.go similarity index 99% rename from internal/storage/balance_storage.go rename to pkg/storage/balance_storage.go index 1564e996..aa58e779 100644 --- a/internal/storage/balance_storage.go +++ b/pkg/storage/balance_storage.go @@ -21,7 +21,7 @@ import ( "log" "math/big" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/parser" diff --git a/internal/storage/balance_storage_test.go b/pkg/storage/balance_storage_test.go similarity index 99% rename from internal/storage/balance_storage_test.go rename to pkg/storage/balance_storage_test.go index f1bfd259..88484033 100644 --- a/internal/storage/balance_storage_test.go +++ b/pkg/storage/balance_storage_test.go @@ -22,7 +22,7 @@ import ( "path" "testing" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/parser" diff --git a/internal/storage/block_storage.go b/pkg/storage/block_storage.go similarity index 99% rename from internal/storage/block_storage.go rename to pkg/storage/block_storage.go index 61edef66..61a5aae9 100644 --- a/internal/storage/block_storage.go +++ b/pkg/storage/block_storage.go @@ -20,7 +20,7 @@ import ( "fmt" "log" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" diff --git a/internal/storage/block_storage_test.go b/pkg/storage/block_storage_test.go similarity index 99% rename from internal/storage/block_storage_test.go rename to pkg/storage/block_storage_test.go index 1f8e20ca..63c4361b 100644 --- a/internal/storage/block_storage_test.go +++ b/pkg/storage/block_storage_test.go @@ -19,7 +19,7 @@ import ( "fmt" "testing" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/types" "github.com/stretchr/testify/assert" diff --git a/internal/storage/broadcast_storage.go b/pkg/storage/broadcast_storage.go similarity index 100% rename from internal/storage/broadcast_storage.go rename to pkg/storage/broadcast_storage.go diff --git a/internal/storage/broadcast_storage_test.go b/pkg/storage/broadcast_storage_test.go similarity index 99% rename from internal/storage/broadcast_storage_test.go rename to pkg/storage/broadcast_storage_test.go index b376eb33..99eb749b 100644 --- a/internal/storage/broadcast_storage_test.go +++ b/pkg/storage/broadcast_storage_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/types" "github.com/stretchr/testify/assert" diff --git a/internal/storage/coin_storage.go b/pkg/storage/coin_storage.go similarity index 100% rename from internal/storage/coin_storage.go rename to pkg/storage/coin_storage.go diff --git a/internal/storage/coin_storage_test.go b/pkg/storage/coin_storage_test.go similarity index 99% rename from internal/storage/coin_storage_test.go rename to pkg/storage/coin_storage_test.go index cfeec443..bf19c15a 100644 --- a/internal/storage/coin_storage_test.go +++ b/pkg/storage/coin_storage_test.go @@ -19,7 +19,7 @@ import ( "math/big" "testing" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/types" diff --git a/internal/storage/counter_storage.go b/pkg/storage/counter_storage.go similarity index 100% rename from internal/storage/counter_storage.go rename to pkg/storage/counter_storage.go diff --git a/internal/storage/counter_storage_test.go b/pkg/storage/counter_storage_test.go similarity index 97% rename from internal/storage/counter_storage_test.go rename to pkg/storage/counter_storage_test.go index a7f328af..565dd881 100644 --- a/internal/storage/counter_storage_test.go +++ b/pkg/storage/counter_storage_test.go @@ -19,7 +19,7 @@ import ( "math/big" "testing" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/stretchr/testify/assert" ) diff --git a/internal/storage/key_storage.go b/pkg/storage/key_storage.go similarity index 100% rename from internal/storage/key_storage.go rename to pkg/storage/key_storage.go diff --git a/internal/storage/key_storage_test.go b/pkg/storage/key_storage_test.go similarity index 98% rename from internal/storage/key_storage_test.go rename to pkg/storage/key_storage_test.go index 4109da86..f66c3dfe 100644 --- a/internal/storage/key_storage_test.go +++ b/pkg/storage/key_storage_test.go @@ -18,7 +18,7 @@ import ( "context" "testing" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/keys" "github.com/coinbase/rosetta-sdk-go/types" diff --git a/internal/storage/storage.go b/pkg/storage/storage.go similarity index 100% rename from internal/storage/storage.go rename to pkg/storage/storage.go diff --git a/internal/storage/utils.go b/pkg/storage/utils.go similarity index 100% rename from internal/storage/utils.go rename to pkg/storage/utils.go diff --git a/internal/tester/construction.go b/pkg/tester/construction.go similarity index 95% rename from internal/tester/construction.go rename to pkg/tester/construction.go index 5388b760..b3a081d4 100644 --- a/internal/tester/construction.go +++ b/pkg/tester/construction.go @@ -22,12 +22,12 @@ import ( "time" "github.com/coinbase/rosetta-cli/configuration" - "github.com/coinbase/rosetta-cli/internal/constructor" - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/processor" - "github.com/coinbase/rosetta-cli/internal/statefulsyncer" - "github.com/coinbase/rosetta-cli/internal/storage" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/constructor" + "github.com/coinbase/rosetta-cli/pkg/logger" + "github.com/coinbase/rosetta-cli/pkg/processor" + "github.com/coinbase/rosetta-cli/pkg/statefulsyncer" + "github.com/coinbase/rosetta-cli/pkg/storage" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/parser" diff --git a/internal/tester/data.go b/pkg/tester/data.go similarity index 98% rename from internal/tester/data.go rename to pkg/tester/data.go index 4f45b90a..5cc42405 100644 --- a/internal/tester/data.go +++ b/pkg/tester/data.go @@ -24,11 +24,11 @@ import ( "time" "github.com/coinbase/rosetta-cli/configuration" - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/processor" - "github.com/coinbase/rosetta-cli/internal/statefulsyncer" - "github.com/coinbase/rosetta-cli/internal/storage" - "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-cli/pkg/logger" + "github.com/coinbase/rosetta-cli/pkg/processor" + "github.com/coinbase/rosetta-cli/pkg/statefulsyncer" + "github.com/coinbase/rosetta-cli/pkg/storage" + "github.com/coinbase/rosetta-cli/pkg/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/reconciler" diff --git a/internal/utils/utils.go b/pkg/utils/utils.go similarity index 100% rename from internal/utils/utils.go rename to pkg/utils/utils.go diff --git a/internal/utils/utils_test.go b/pkg/utils/utils_test.go similarity index 100% rename from internal/utils/utils_test.go rename to pkg/utils/utils_test.go From 9a1de06310d53ba4e1be2332929705dc6a606cff Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 7 Aug 2020 16:05:16 -0700 Subject: [PATCH 02/10] Make logging of config file optional --- configuration/configuration.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 8e743cc7..0e1d84f6 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -358,6 +358,10 @@ type Configuration struct { // that uses more memory. DisableMemoryLimit bool `json:"disable_memory_limit"` + // LogConfiguration determines if the configuration settings + // should be printed to the console when a file is loaded. + LogConfiguration bool `json:"log_configuration"` + Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` } @@ -550,7 +554,9 @@ func LoadConfiguration(filePath string) (*Configuration, error) { filePath, ) - log.Println(types.PrettyPrintStruct(config)) + if config.LogConfiguration { + log.Println(types.PrettyPrintStruct(config)) + } return config, nil } From af45c89db8f502340f85a42117ea16deb459e075 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 7 Aug 2020 16:09:14 -0700 Subject: [PATCH 03/10] Fix license --- mocks/constructor/handler.go | 14 ++++++++++++++ mocks/constructor/helper.go | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/mocks/constructor/handler.go b/mocks/constructor/handler.go index 4afe30ef..8cef1737 100644 --- a/mocks/constructor/handler.go +++ b/mocks/constructor/handler.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by mockery v1.0.0. DO NOT EDIT. package constructor diff --git a/mocks/constructor/helper.go b/mocks/constructor/helper.go index ef940fb2..d29bd13b 100644 --- a/mocks/constructor/helper.go +++ b/mocks/constructor/helper.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by mockery v1.0.0. DO NOT EDIT. package constructor From 330589f6d3edec16018e902410b016b08cdc4107 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 7 Aug 2020 16:30:47 -0700 Subject: [PATCH 04/10] Return block identifier with coin storage --- mocks/constructor/handler.go | 14 -------- mocks/constructor/helper.go | 14 -------- pkg/storage/coin_storage.go | 42 +++++++++++++++++------- pkg/storage/coin_storage_test.go | 55 ++++++++++++++++++++++++-------- 4 files changed, 72 insertions(+), 53 deletions(-) diff --git a/mocks/constructor/handler.go b/mocks/constructor/handler.go index 8cef1737..4afe30ef 100644 --- a/mocks/constructor/handler.go +++ b/mocks/constructor/handler.go @@ -1,17 +1,3 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - // Code generated by mockery v1.0.0. DO NOT EDIT. package constructor diff --git a/mocks/constructor/helper.go b/mocks/constructor/helper.go index d29bd13b..ef940fb2 100644 --- a/mocks/constructor/helper.go +++ b/mocks/constructor/helper.go @@ -1,17 +1,3 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - // Code generated by mockery v1.0.0. DO NOT EDIT. package constructor diff --git a/pkg/storage/coin_storage.go b/pkg/storage/coin_storage.go index 8d4e6bf4..eabd1ea3 100644 --- a/pkg/storage/coin_storage.go +++ b/pkg/storage/coin_storage.go @@ -36,16 +36,31 @@ var _ BlockWorker = (*CoinStorage)(nil) type CoinStorage struct { db Database + helper CoinStorageHelper asserter *asserter.Asserter } +// CoinStorageHelper is used by CoinStorage to determine +// at which block a Coin set is valid. +type CoinStorageHelper interface { + // CurrentBlockIdentifier is called while fetching coins in a single + // database transaction to return the *types.BlockIdentifier where + // the Coin set is valid. + CurrentBlockIdentifier( + context.Context, + DatabaseTransaction, + ) (*types.BlockIdentifier, error) +} + // NewCoinStorage returns a new CoinStorage. func NewCoinStorage( db Database, + helper CoinStorageHelper, asserter *asserter.Asserter, ) *CoinStorage { return &CoinStorage{ db: db, + helper: helper, asserter: asserter, } } @@ -323,17 +338,22 @@ func (c *CoinStorage) RemovingBlock( func (c *CoinStorage) GetCoins( ctx context.Context, accountIdentifier *types.AccountIdentifier, -) ([]*Coin, error) { +) ([]*Coin, *types.BlockIdentifier, error) { transaction := c.db.NewDatabaseTransaction(ctx, false) defer transaction.Discard(ctx) accountExists, coins, err := getAndDecodeCoins(ctx, transaction, accountIdentifier) if err != nil { - return nil, fmt.Errorf("%w: unable to query account identifier", err) + return nil, nil, fmt.Errorf("%w: unable to query account identifier", err) + } + + headBlockIdentifier, err := c.helper.CurrentBlockIdentifier(ctx, transaction) + if err != nil { + return nil, nil, fmt.Errorf("%w: unable to get current block identifier", err) } if !accountExists { - return []*Coin{}, nil + return []*Coin{}, headBlockIdentifier, nil } coinArr := []*Coin{} @@ -344,17 +364,17 @@ func (c *CoinStorage) GetCoins( &types.CoinIdentifier{Identifier: coinIdentifier}, ) if err != nil { - return nil, fmt.Errorf("%w: unable to query coin", err) + return nil, nil, fmt.Errorf("%w: unable to query coin", err) } if !exists { - return nil, fmt.Errorf("%w: unable to get coin %s", err, coinIdentifier) + return nil, nil, fmt.Errorf("%w: unable to get coin %s", err, coinIdentifier) } coinArr = append(coinArr, coin) } - return coinArr, nil + return coinArr, headBlockIdentifier, nil } // GetLargestCoin returns the largest Coin for a @@ -364,10 +384,10 @@ func (c *CoinStorage) GetLargestCoin( ctx context.Context, accountIdentifier *types.AccountIdentifier, currency *types.Currency, -) (*big.Int, *types.CoinIdentifier, error) { - coins, err := c.GetCoins(ctx, accountIdentifier) +) (*big.Int, *types.CoinIdentifier, *types.BlockIdentifier, error) { + coins, blockIdentifier, err := c.GetCoins(ctx, accountIdentifier) if err != nil { - return nil, nil, fmt.Errorf( + return nil, nil, nil, fmt.Errorf( "%w: unable to get utxo balance for %s", err, accountIdentifier.Address, @@ -387,7 +407,7 @@ func (c *CoinStorage) GetLargestCoin( val, ok := new(big.Int).SetString(coin.Operation.Amount.Value, 10) if !ok { - return nil, nil, fmt.Errorf( + return nil, nil, nil, fmt.Errorf( "could not parse amount for coin %s", coin.Identifier.Identifier, ) @@ -399,5 +419,5 @@ func (c *CoinStorage) GetLargestCoin( } } - return bal, coinIdentifier, nil + return bal, coinIdentifier, blockIdentifier, nil } diff --git a/pkg/storage/coin_storage_test.go b/pkg/storage/coin_storage_test.go index bf19c15a..5e61a642 100644 --- a/pkg/storage/coin_storage_test.go +++ b/pkg/storage/coin_storage_test.go @@ -27,6 +27,10 @@ import ( ) var ( + blockIdentifier = &types.BlockIdentifier{ + Hash: "block", + Index: 1, + } account = &types.AccountIdentifier{ Address: "blah", } @@ -254,17 +258,21 @@ func TestCoinStorage(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, a) - c := NewCoinStorage(database, a) + mockHelper := &MockCoinStorageHelper{} + + c := NewCoinStorage(database, mockHelper, a) t.Run("get coins of unset account", func(t *testing.T) { - coins, err := c.GetCoins(ctx, account) + coins, block, err := c.GetCoins(ctx, account) assert.NoError(t, err) assert.Equal(t, []*Coin{}, coins) + assert.Equal(t, blockIdentifier, block) - bal, coinIdentifier, err := c.GetLargestCoin(ctx, account, currency) + bal, coinIdentifier, block, err := c.GetLargestCoin(ctx, account, currency) assert.NoError(t, err) assert.Equal(t, big.NewInt(0), bal) assert.Nil(t, coinIdentifier) + assert.Equal(t, blockIdentifier, block) }) t.Run("add block", func(t *testing.T) { @@ -274,9 +282,10 @@ func TestCoinStorage(t *testing.T) { assert.NoError(t, err) assert.NoError(t, tx.Commit(ctx)) - coins, err := c.GetCoins(ctx, account) + coins, block, err := c.GetCoins(ctx, account) assert.NoError(t, err) assert.Equal(t, accountCoins, coins) + assert.Equal(t, blockIdentifier, block) }) t.Run("add duplicate coin", func(t *testing.T) { @@ -286,9 +295,10 @@ func TestCoinStorage(t *testing.T) { assert.Error(t, err) tx.Discard(ctx) - coins, err := c.GetCoins(ctx, account) + coins, block, err := c.GetCoins(ctx, account) assert.NoError(t, err) assert.Equal(t, accountCoins, coins) + assert.Equal(t, blockIdentifier, block) }) t.Run("remove block", func(t *testing.T) { @@ -298,13 +308,15 @@ func TestCoinStorage(t *testing.T) { assert.NoError(t, err) assert.NoError(t, tx.Commit(ctx)) - coins, err := c.GetCoins(ctx, account) + coins, block, err := c.GetCoins(ctx, account) assert.NoError(t, err) assert.Equal(t, []*Coin{}, coins) + assert.Equal(t, blockIdentifier, block) - coins, err = c.GetCoins(ctx, account2) + coins, block, err = c.GetCoins(ctx, account2) assert.NoError(t, err) assert.Equal(t, account2Coins, coins) + assert.Equal(t, blockIdentifier, block) }) t.Run("spend coin", func(t *testing.T) { @@ -314,9 +326,10 @@ func TestCoinStorage(t *testing.T) { assert.NoError(t, err) assert.NoError(t, tx.Commit(ctx)) - coins, err := c.GetCoins(ctx, account) + coins, block, err := c.GetCoins(ctx, account) assert.NoError(t, err) assert.Equal(t, accountCoins, coins) + assert.Equal(t, blockIdentifier, block) tx = c.db.NewDatabaseTransaction(ctx, true) commitFunc, err = c.AddingBlock(ctx, coinBlock2, tx) @@ -324,13 +337,15 @@ func TestCoinStorage(t *testing.T) { assert.NoError(t, err) assert.NoError(t, tx.Commit(ctx)) - coins, err = c.GetCoins(ctx, account) + coins, block, err = c.GetCoins(ctx, account) assert.NoError(t, err) assert.Equal(t, []*Coin{}, coins) + assert.Equal(t, blockIdentifier, block) - coins, err = c.GetCoins(ctx, account2) + coins, block, err = c.GetCoins(ctx, account2) assert.NoError(t, err) assert.Equal(t, []*Coin{}, coins) + assert.Equal(t, blockIdentifier, block) }) t.Run("add block with multiple outputs for 1 account", func(t *testing.T) { @@ -340,22 +355,34 @@ func TestCoinStorage(t *testing.T) { assert.NoError(t, err) assert.NoError(t, tx.Commit(ctx)) - coins, err := c.GetCoins(ctx, account) + coins, block, err := c.GetCoins(ctx, account) assert.NoError(t, err) assert.Equal(t, []*Coin{}, coins) + assert.Equal(t, blockIdentifier, block) - coins, err = c.GetCoins(ctx, account3) + coins, block, err = c.GetCoins(ctx, account3) assert.NoError(t, err) assert.ElementsMatch(t, account3Coins, coins) + assert.Equal(t, blockIdentifier, block) - bal, coinIdentifier, err := c.GetLargestCoin(ctx, account3, currency) + bal, coinIdentifier, block, err := c.GetLargestCoin(ctx, account3, currency) assert.NoError(t, err) assert.Equal(t, big.NewInt(4), bal) assert.Equal(t, &types.CoinIdentifier{Identifier: "coin3"}, coinIdentifier) + assert.Equal(t, blockIdentifier, block) - bal, coinIdentifier, err = c.GetLargestCoin(ctx, account3, currency2) + bal, coinIdentifier, block, err = c.GetLargestCoin(ctx, account3, currency2) assert.NoError(t, err) assert.Equal(t, big.NewInt(6), bal) assert.Equal(t, &types.CoinIdentifier{Identifier: "coin4"}, coinIdentifier) + assert.Equal(t, blockIdentifier, block) }) } + +type MockCoinStorageHelper struct{} + +var _ CoinStorageHelper = (*MockCoinStorageHelper)(nil) + +func (h *MockCoinStorageHelper) CurrentBlockIdentifier(ctx context.Context, transaction DatabaseTransaction) (*types.BlockIdentifier, error) { + return blockIdentifier, nil +} From 11c10fe931250adbb7162317d4b5b4b67dafcd46 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 7 Aug 2020 16:32:09 -0700 Subject: [PATCH 05/10] Fix mocks to add license --- Makefile | 1 + mocks/constructor/handler.go | 14 ++++++++++++++ mocks/constructor/helper.go | 14 ++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/Makefile b/Makefile index 7d12b765..82ebae16 100644 --- a/Makefile +++ b/Makefile @@ -60,3 +60,4 @@ install: mocks: rm -rf mocks; mockery --dir pkg/constructor --all --case underscore --outpkg constructor --output mocks/constructor; + ${ADDLICENCE_SCRIPT} .; diff --git a/mocks/constructor/handler.go b/mocks/constructor/handler.go index 4afe30ef..8cef1737 100644 --- a/mocks/constructor/handler.go +++ b/mocks/constructor/handler.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by mockery v1.0.0. DO NOT EDIT. package constructor diff --git a/mocks/constructor/helper.go b/mocks/constructor/helper.go index ef940fb2..d29bd13b 100644 --- a/mocks/constructor/helper.go +++ b/mocks/constructor/helper.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by mockery v1.0.0. DO NOT EDIT. package constructor From e0f4bc3ebed481c47382c2ec7e6301327f54b449 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 7 Aug 2020 16:43:47 -0700 Subject: [PATCH 06/10] Return block identifier with coins --- pkg/processor/coin_storage_helper.go | 45 ++++++++++++++++++++++++++++ pkg/processor/constructor_helper.go | 8 ++++- pkg/storage/block_storage.go | 9 ++++++ pkg/storage/coin_storage_test.go | 5 +++- pkg/tester/construction.go | 3 +- 5 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 pkg/processor/coin_storage_helper.go diff --git a/pkg/processor/coin_storage_helper.go b/pkg/processor/coin_storage_helper.go new file mode 100644 index 00000000..6761cecd --- /dev/null +++ b/pkg/processor/coin_storage_helper.go @@ -0,0 +1,45 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package processor + +import ( + "context" + + "github.com/coinbase/rosetta-cli/pkg/storage" + + "github.com/coinbase/rosetta-sdk-go/types" +) + +var _ storage.CoinStorageHelper = (*CoinStorageHelper)(nil) + +// CoinStorageHelper implements the storage.CoinStorageHelper +// interface. +type CoinStorageHelper struct { + blockStorage *storage.BlockStorage +} + +// NewCoinStorageHelper returns a new *CoinStorageHelper. +func NewCoinStorageHelper(blockStorage *storage.BlockStorage) *CoinStorageHelper { + return &CoinStorageHelper{blockStorage: blockStorage} +} + +// CurrentBlockIdentifier returns the head *types.BlockIdentifier in +// the context of a storage.DatabaseTransaction. +func (c *CoinStorageHelper) CurrentBlockIdentifier( + ctx context.Context, + transaction storage.DatabaseTransaction, +) (*types.BlockIdentifier, error) { + return c.blockStorage.GetHeadBlockIdentifierTransactional(ctx, transaction) +} diff --git a/pkg/processor/constructor_helper.go b/pkg/processor/constructor_helper.go index c1e9af3c..1168ce8d 100644 --- a/pkg/processor/constructor_helper.go +++ b/pkg/processor/constructor_helper.go @@ -216,7 +216,13 @@ func (c *ConstructorHelper) CoinBalance( accountIdentifier *types.AccountIdentifier, currency *types.Currency, ) (*big.Int, *types.CoinIdentifier, error) { - return c.coinStorage.GetLargestCoin(ctx, accountIdentifier, currency) + coinValue, coinIdentifier, _, err := c.coinStorage.GetLargestCoin( + ctx, + accountIdentifier, + currency, + ) + + return coinValue, coinIdentifier, err } // LockedAddresses returns a slice of all addresses currently sending or receiving diff --git a/pkg/storage/block_storage.go b/pkg/storage/block_storage.go index 61a5aae9..61e9f298 100644 --- a/pkg/storage/block_storage.go +++ b/pkg/storage/block_storage.go @@ -129,6 +129,15 @@ func (b *BlockStorage) GetHeadBlockIdentifier( transaction := b.db.NewDatabaseTransaction(ctx, false) defer transaction.Discard(ctx) + return b.GetHeadBlockIdentifierTransactional(ctx, transaction) +} + +// GetHeadBlockIdentifierTransactional returns the head block identifier, +// if it exists, in the context of a DatabaseTransaction. +func (b *BlockStorage) GetHeadBlockIdentifierTransactional( + ctx context.Context, + transaction DatabaseTransaction, +) (*types.BlockIdentifier, error) { exists, block, err := transaction.Get(ctx, getHeadBlockKey()) if err != nil { return nil, err diff --git a/pkg/storage/coin_storage_test.go b/pkg/storage/coin_storage_test.go index 5e61a642..446b7c50 100644 --- a/pkg/storage/coin_storage_test.go +++ b/pkg/storage/coin_storage_test.go @@ -383,6 +383,9 @@ type MockCoinStorageHelper struct{} var _ CoinStorageHelper = (*MockCoinStorageHelper)(nil) -func (h *MockCoinStorageHelper) CurrentBlockIdentifier(ctx context.Context, transaction DatabaseTransaction) (*types.BlockIdentifier, error) { +func (h *MockCoinStorageHelper) CurrentBlockIdentifier( + ctx context.Context, + transaction DatabaseTransaction, +) (*types.BlockIdentifier, error) { return blockIdentifier, nil } diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index b3a081d4..9c098c43 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -84,7 +84,8 @@ func InitializeConstruction( blockStorage := storage.NewBlockStorage(localStore) keyStorage := storage.NewKeyStorage(localStore) - coinStorage := storage.NewCoinStorage(localStore, onlineFetcher.Asserter) + coinStorageHelper := processor.NewCoinStorageHelper(blockStorage) + coinStorage := storage.NewCoinStorage(localStore, coinStorageHelper, onlineFetcher.Asserter) balanceStorage := storage.NewBalanceStorage(localStore) balanceStorageHelper := processor.NewBalanceStorageHelper( From 3ad17f7a2ce04ea4fef8b2d665db65dbf89c90fc Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 7 Aug 2020 16:57:07 -0700 Subject: [PATCH 07/10] Track coins on check:data --- configuration/configuration.go | 6 ++++++ pkg/tester/data.go | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/configuration/configuration.go b/configuration/configuration.go index 0e1d84f6..8a2f4048 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -322,6 +322,12 @@ type DataConfiguration struct { // consistency. BalanceTrackingDisabled bool `json:"balance_tracking_disabled"` + // CoinTrackingDisabled is a boolean that indicates coin (or UTXO) tracking + // should not be attempted. When first testing an implemenation, it can be + // useful to just try to fetch all blocks before checking for coin + // consistency. + CoinTrackingDisabled bool `json:"coin_tracking_disabled"` + // EndCondition contains the conditions for the syncer to stop EndConditions *EndConditions `json:"end_conditions,omitempty"` } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 5cc42405..d0cbdcb9 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -225,6 +225,13 @@ func InitializeData( blockWorkers = append(blockWorkers, balanceStorage) } + if !config.Data.CoinTrackingDisabled { + coinStorageHelper := processor.NewCoinStorageHelper(blockStorage) + coinStorage := storage.NewCoinStorage(localStore, coinStorageHelper, fetcher.Asserter) + + blockWorkers = append(blockWorkers, coinStorage) + } + syncer := statefulsyncer.New( ctx, network, From dff2600064386651584a15b027b0eb9c497a5911 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 7 Aug 2020 17:20:46 -0700 Subject: [PATCH 08/10] Replicate block cache error --- pkg/storage/block_storage_test.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/storage/block_storage_test.go b/pkg/storage/block_storage_test.go index 63c4361b..4b066a3d 100644 --- a/pkg/storage/block_storage_test.go +++ b/pkg/storage/block_storage_test.go @@ -122,6 +122,18 @@ func simpleTransactionFactory( } var ( + genesisBlock = &types.Block{ + BlockIdentifier: &types.BlockIdentifier{ + Hash: "blah 0", + Index: 0, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "blah 0", + Index: 0, + }, + Timestamp: 1, + } + newBlock = &types.Block{ BlockIdentifier: &types.BlockIdentifier{ Hash: "blah 1", @@ -422,21 +434,21 @@ func TestCreateBlockCache(t *testing.T) { }) t.Run("1 block processed", func(t *testing.T) { - err = storage.AddBlock(ctx, newBlock) + err = storage.AddBlock(ctx, genesisBlock) assert.NoError(t, err) assert.Equal( t, - []*types.BlockIdentifier{newBlock.BlockIdentifier}, + []*types.BlockIdentifier{genesisBlock.BlockIdentifier}, storage.CreateBlockCache(ctx), ) }) t.Run("2 blocks processed", func(t *testing.T) { - err = storage.AddBlock(ctx, newBlock3) + err = storage.AddBlock(ctx, newBlock) assert.NoError(t, err) assert.Equal( t, - []*types.BlockIdentifier{newBlock.BlockIdentifier, newBlock3.BlockIdentifier}, + []*types.BlockIdentifier{genesisBlock.BlockIdentifier, newBlock.BlockIdentifier}, storage.CreateBlockCache(ctx), ) }) From ad0f7992961b482be820576c2ebfc7136f7befc5 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 7 Aug 2020 17:22:57 -0700 Subject: [PATCH 09/10] Populate cache correctly when hitting genesis --- pkg/storage/block_storage.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/storage/block_storage.go b/pkg/storage/block_storage.go index 61e9f298..ac5a59cf 100644 --- a/pkg/storage/block_storage.go +++ b/pkg/storage/block_storage.go @@ -382,6 +382,11 @@ func (b *BlockStorage) CreateBlockCache(ctx context.Context) []*types.BlockIdent cache = append([]*types.BlockIdentifier{block.BlockIdentifier}, cache...) head = block.ParentBlockIdentifier + + // We should break if we have reached genesis. + if head.Index == block.BlockIdentifier.Index { + break + } } return cache From 5e2bc38e3762f2f41ab2c9cb5162063a7f6407b1 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 7 Aug 2020 17:30:50 -0700 Subject: [PATCH 10/10] nits --- pkg/storage/block_storage_test.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pkg/storage/block_storage_test.go b/pkg/storage/block_storage_test.go index 4b066a3d..8f9a55e1 100644 --- a/pkg/storage/block_storage_test.go +++ b/pkg/storage/block_storage_test.go @@ -169,18 +169,6 @@ var ( }, } - newBlock3 = &types.Block{ - BlockIdentifier: &types.BlockIdentifier{ - Hash: "blah 2", - Index: 2, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "blah 1", - Index: 1, - }, - Timestamp: 1, - } - complexBlock = &types.Block{ BlockIdentifier: &types.BlockIdentifier{ Hash: "blah 3",