diff --git a/README.md b/README.md index 5d419915..de94a20a 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ and account balances. Before diving into the CLI, we recommend taking a look at the Rosetta API Docs: * [Overview](https://www.rosetta-api.org/docs/welcome.html) -* [Node API](https://www.rosetta-api.org/docs/node_api_introduction.html) -* [Wallet API (coming soon!)](https://www.rosetta-api.org/docs/wallet_api_introduction.html) +* [Data API](https://www.rosetta-api.org/docs/data_api_introduction.html) +* [Construction API](https://www.rosetta-api.org/docs/construction_api_introduction.html) ## Install ``` diff --git a/go.mod b/go.mod index ec7422c1..4f4b1ff1 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.6.1 + github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1 golang.org/x/net v0.0.0-20200513185701-a91f0712d120 // indirect golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect diff --git a/go.sum b/go.sum index 17e24c8e..b2b0fd20 100644 --- a/go.sum +++ b/go.sum @@ -65,6 +65,7 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -180,6 +181,12 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= +github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1 h1:d71/KA0LhvkrJ/Ok+Wx9qK7bU8meKA1Hk0jpVI5kJjk= +github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oTuqa03RjMwMfk/7/TCs+QI= +github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY= +github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -204,8 +211,10 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200513185701-a91f0712d120 h1:EZ3cVSzKOlJxAd8e8YAJ7no8nNypTxexh/YE/xW3ZEY= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -237,8 +246,10 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 h1:YTzHMGlqJu67/uEo1lBv0n3wBXhXNeUbB1XfN2vmTm0= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181127232545-e782529d0ddd/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -248,6 +259,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index 6d4c0373..3d02055d 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -17,15 +17,17 @@ package storage import ( "bytes" "context" - "encoding/gob" "encoding/json" "errors" "fmt" + "io" "io/ioutil" "log" "math/big" "path" + msgpack "github.com/vmihailenco/msgpack/v5" + "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/reconciler" @@ -115,6 +117,20 @@ func GetBalanceKey(account *types.AccountIdentifier, currency *types.Currency) [ ) } +func getEncoder(w io.Writer) *msgpack.Encoder { + enc := msgpack.NewEncoder(w) + enc.UseJSONTag(true) + + return enc +} + +func getDecoder(r io.Reader) *msgpack.Decoder { + dec := msgpack.NewDecoder(r) + dec.UseJSONTag(true) + + return dec +} + // Helper functions are used by BlockStorage to process blocks. Defining an // interface allows the client to determine if they wish to query the node for // certain information or use another datastore. @@ -177,9 +193,8 @@ func (b *BlockStorage) GetHeadBlockIdentifier( return nil, ErrHeadBlockNotFound } - dec := gob.NewDecoder(bytes.NewReader(block)) var blockIdentifier types.BlockIdentifier - err = dec.Decode(&blockIdentifier) + err = getDecoder(bytes.NewReader(block)).Decode(&blockIdentifier) if err != nil { return nil, err } @@ -195,7 +210,7 @@ func (b *BlockStorage) StoreHeadBlockIdentifier( blockIdentifier *types.BlockIdentifier, ) error { buf := new(bytes.Buffer) - err := gob.NewEncoder(buf).Encode(blockIdentifier) + err := getEncoder(buf).Encode(blockIdentifier) if err != nil { return err } @@ -221,7 +236,7 @@ func (b *BlockStorage) GetBlock( } var rosettaBlock types.Block - err = gob.NewDecoder(bytes.NewBuffer(block)).Decode(&rosettaBlock) + err = getDecoder(bytes.NewBuffer(block)).Decode(&rosettaBlock) if err != nil { return nil, err } @@ -271,7 +286,7 @@ func (b *BlockStorage) StoreBlock( transaction := b.newDatabaseTransaction(ctx, true) defer transaction.Discard(ctx) buf := new(bytes.Buffer) - err := gob.NewEncoder(buf).Encode(block) + err := getEncoder(buf).Encode(block) if err != nil { return nil, err } @@ -383,7 +398,7 @@ type balanceEntry struct { func serializeBalanceEntry(bal balanceEntry) ([]byte, error) { buf := new(bytes.Buffer) - err := gob.NewEncoder(buf).Encode(bal) + err := getEncoder(buf).Encode(bal) if err != nil { return nil, err } @@ -392,9 +407,8 @@ func serializeBalanceEntry(bal balanceEntry) ([]byte, error) { } func parseBalanceEntry(buf []byte) (*balanceEntry, error) { - dec := gob.NewDecoder(bytes.NewReader(buf)) var bal balanceEntry - err := dec.Decode(&bal) + err := getDecoder(bytes.NewReader(buf)).Decode(&bal) if err != nil { return nil, err } diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index 8df6c04a..9739f05a 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -176,6 +176,69 @@ var ( }, Timestamp: 1, } + + complexBlock = &types.Block{ + BlockIdentifier: &types.BlockIdentifier{ + Hash: "blah 3", + Index: 3, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "blah 2", + Index: 2, + }, + Timestamp: 1, + Transactions: []*types.Transaction{ + { + TransactionIdentifier: &types.TransactionIdentifier{ + Hash: "blahTx 2", + }, + Operations: []*types.Operation{ + { + OperationIdentifier: &types.OperationIdentifier{ + Index: 0, + }, + Type: "Transfer", + Status: "Success", + Account: &types.AccountIdentifier{ + Address: "addr1", + SubAccount: &types.SubAccountIdentifier{ + Address: "staking", + Metadata: map[string]interface{}{ + "other_complex_stuff": []interface{}{ + map[string]interface{}{ + "neat": "test", + "more complex": map[string]interface{}{ + "neater": "testier", + }, + }, + map[string]interface{}{ + "i love": "ice cream", + }, + }, + }, + }, + }, + Amount: &types.Amount{ + Value: "100", + Currency: &types.Currency{ + Symbol: "hello", + }, + }, + }, + }, + Metadata: map[string]interface{}{ + "other_stuff": []interface{}{"stuff"}, + "simple_stuff": "abc", + "super_complex_stuff": map[string]interface{}{ + "neat": "test", + "more complex": map[string]interface{}{ + "neater": "testier", + }, + }, + }, + }, + }, + } ) func TestBlock(t *testing.T) { @@ -247,6 +310,19 @@ func TestBlock(t *testing.T) { _, err = storage.StoreBlock(ctx, newBlock) assert.NoError(t, err) }) + + t.Run("Add block with complex metadata", func(t *testing.T) { + _, err := storage.StoreBlock(ctx, complexBlock) + assert.NoError(t, err) + + block, err := storage.GetBlock(ctx, complexBlock.BlockIdentifier) + assert.NoError(t, err) + assert.Equal(t, complexBlock, block) + + head, err := storage.GetHeadBlockIdentifier(ctx) + assert.NoError(t, err) + assert.Equal(t, complexBlock.BlockIdentifier, head) + }) } func TestBalance(t *testing.T) { @@ -298,7 +374,7 @@ func TestBalance(t *testing.T) { SubAccount: &types.SubAccountIdentifier{ Address: "stake", Metadata: map[string]interface{}{ - "cool": 10, + "cool": float64(10), }, }, } @@ -307,7 +383,7 @@ func TestBalance(t *testing.T) { SubAccount: &types.SubAccountIdentifier{ Address: "stake", Metadata: map[string]interface{}{ - "cool": 10, + "cool": float64(10), }, }, }