Skip to content

Commit

Permalink
Merge pull request #52 from coinbase/patrick/fix-gob-encoding-issue
Browse files Browse the repository at this point in the history 
Use msgpack for storage instead of gob
  • Loading branch information
@patrick-ogrady
patrick-ogrady authored Jul 13, 2020
2 parents 0ac81b3 + 897e3cd commit 7856d74
Show file tree
Hide file tree
Showing 5 changed files with 116 additions and 13 deletions.
4 changes: 2 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ and account balances.
Before diving into the CLI, we recommend taking a look at the Rosetta API Docs:

* [Overview](https://www.rosetta-api.org/docs/welcome.html)
* [Node API](https://www.rosetta-api.org/docs/node_api_introduction.html)
* [Wallet API (coming soon!)](https://www.rosetta-api.org/docs/wallet_api_introduction.html)
* [Data API](https://www.rosetta-api.org/docs/data_api_introduction.html)
* [Construction API](https://www.rosetta-api.org/docs/construction_api_introduction.html)

## Install
```
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ require (
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.6.1
github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1
golang.org/x/net v0.0.0-20200513185701-a91f0712d120 // indirect
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect
Expand Down
12 changes: 12 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
Expand Down Expand Up @@ -180,6 +181,12 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1 h1:d71/KA0LhvkrJ/Ok+Wx9qK7bU8meKA1Hk0jpVI5kJjk=
github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oTuqa03RjMwMfk/7/TCs+QI=
github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY=
github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=
github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
Expand All @@ -204,8 +211,10 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200513185701-a91f0712d120 h1:EZ3cVSzKOlJxAd8e8YAJ7no8nNypTxexh/YE/xW3ZEY=
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down Expand Up @@ -237,8 +246,10 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 h1:YTzHMGlqJu67/uEo1lBv0n3wBXhXNeUbB1XfN2vmTm0=
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181127232545-e782529d0ddd/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
Expand All @@ -248,6 +259,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
Expand Down
32 changes: 23 additions & 9 deletions internal/storage/block_storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,17 @@ package storage
import (
"bytes"
"context"
"encoding/gob"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"math/big"
"path"

msgpack "github.com/vmihailenco/msgpack/v5"

"github.com/coinbase/rosetta-sdk-go/asserter"
"github.com/coinbase/rosetta-sdk-go/parser"
"github.com/coinbase/rosetta-sdk-go/reconciler"
Expand Down Expand Up @@ -115,6 +117,20 @@ func GetBalanceKey(account *types.AccountIdentifier, currency *types.Currency) [
)
}

func getEncoder(w io.Writer) *msgpack.Encoder {
enc := msgpack.NewEncoder(w)
enc.UseJSONTag(true)

return enc
}

func getDecoder(r io.Reader) *msgpack.Decoder {
dec := msgpack.NewDecoder(r)
dec.UseJSONTag(true)

return dec
}

// Helper functions are used by BlockStorage to process blocks. Defining an
// interface allows the client to determine if they wish to query the node for
// certain information or use another datastore.
Expand Down Expand Up @@ -177,9 +193,8 @@ func (b *BlockStorage) GetHeadBlockIdentifier(
return nil, ErrHeadBlockNotFound
}

dec := gob.NewDecoder(bytes.NewReader(block))
var blockIdentifier types.BlockIdentifier
err = dec.Decode(&blockIdentifier)
err = getDecoder(bytes.NewReader(block)).Decode(&blockIdentifier)
if err != nil {
return nil, err
}
Expand All @@ -195,7 +210,7 @@ func (b *BlockStorage) StoreHeadBlockIdentifier(
blockIdentifier *types.BlockIdentifier,
) error {
buf := new(bytes.Buffer)
err := gob.NewEncoder(buf).Encode(blockIdentifier)
err := getEncoder(buf).Encode(blockIdentifier)
if err != nil {
return err
}
Expand All @@ -221,7 +236,7 @@ func (b *BlockStorage) GetBlock(
}

var rosettaBlock types.Block
err = gob.NewDecoder(bytes.NewBuffer(block)).Decode(&rosettaBlock)
err = getDecoder(bytes.NewBuffer(block)).Decode(&rosettaBlock)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -271,7 +286,7 @@ func (b *BlockStorage) StoreBlock(
transaction := b.newDatabaseTransaction(ctx, true)
defer transaction.Discard(ctx)
buf := new(bytes.Buffer)
err := gob.NewEncoder(buf).Encode(block)
err := getEncoder(buf).Encode(block)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -383,7 +398,7 @@ type balanceEntry struct {

func serializeBalanceEntry(bal balanceEntry) ([]byte, error) {
buf := new(bytes.Buffer)
err := gob.NewEncoder(buf).Encode(bal)
err := getEncoder(buf).Encode(bal)
if err != nil {
return nil, err
}
Expand All @@ -392,9 +407,8 @@ func serializeBalanceEntry(bal balanceEntry) ([]byte, error) {
}

func parseBalanceEntry(buf []byte) (*balanceEntry, error) {
dec := gob.NewDecoder(bytes.NewReader(buf))
var bal balanceEntry
err := dec.Decode(&bal)
err := getDecoder(bytes.NewReader(buf)).Decode(&bal)
if err != nil {
return nil, err
}
Expand Down
80 changes: 78 additions & 2 deletions internal/storage/block_storage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,69 @@ var (
},
Timestamp: 1,
}

complexBlock = &types.Block{
BlockIdentifier: &types.BlockIdentifier{
Hash: "blah 3",
Index: 3,
},
ParentBlockIdentifier: &types.BlockIdentifier{
Hash: "blah 2",
Index: 2,
},
Timestamp: 1,
Transactions: []*types.Transaction{
{
TransactionIdentifier: &types.TransactionIdentifier{
Hash: "blahTx 2",
},
Operations: []*types.Operation{
{
OperationIdentifier: &types.OperationIdentifier{
Index: 0,
},
Type: "Transfer",
Status: "Success",
Account: &types.AccountIdentifier{
Address: "addr1",
SubAccount: &types.SubAccountIdentifier{
Address: "staking",
Metadata: map[string]interface{}{
"other_complex_stuff": []interface{}{
map[string]interface{}{
"neat": "test",
"more complex": map[string]interface{}{
"neater": "testier",
},
},
map[string]interface{}{
"i love": "ice cream",
},
},
},
},
},
Amount: &types.Amount{
Value: "100",
Currency: &types.Currency{
Symbol: "hello",
},
},
},
},
Metadata: map[string]interface{}{
"other_stuff": []interface{}{"stuff"},
"simple_stuff": "abc",
"super_complex_stuff": map[string]interface{}{
"neat": "test",
"more complex": map[string]interface{}{
"neater": "testier",
},
},
},
},
},
}
)

func TestBlock(t *testing.T) {
Expand Down Expand Up @@ -247,6 +310,19 @@ func TestBlock(t *testing.T) {
_, err = storage.StoreBlock(ctx, newBlock)
assert.NoError(t, err)
})

t.Run("Add block with complex metadata", func(t *testing.T) {
_, err := storage.StoreBlock(ctx, complexBlock)
assert.NoError(t, err)

block, err := storage.GetBlock(ctx, complexBlock.BlockIdentifier)
assert.NoError(t, err)
assert.Equal(t, complexBlock, block)

head, err := storage.GetHeadBlockIdentifier(ctx)
assert.NoError(t, err)
assert.Equal(t, complexBlock.BlockIdentifier, head)
})
}

func TestBalance(t *testing.T) {
Expand Down Expand Up @@ -298,7 +374,7 @@ func TestBalance(t *testing.T) {
SubAccount: &types.SubAccountIdentifier{
Address: "stake",
Metadata: map[string]interface{}{
"cool": 10,
"cool": float64(10),
},
},
}
Expand All @@ -307,7 +383,7 @@ func TestBalance(t *testing.T) {
SubAccount: &types.SubAccountIdentifier{
Address: "stake",
Metadata: map[string]interface{}{
"cool": 10,
"cool": float64(10),
},
},
}
Expand Down

0 comments on commit 7856d74

Please sign in to comment.