From e38da3943594adf6d8685bb3f4b4116699969006 Mon Sep 17 00:00:00 2001 From: Frrist Date: Thu, 30 Jan 2020 16:08:12 -0800 Subject: [PATCH] feat: add generic store interface and impl (#70) * feat: add generic store interface and impl * remove gx --- .gx/lastpubver | 1 - Makefile | 21 ++++--- go.mod | 7 ++- go.sum | 28 ++++++++-- store.go | 147 +++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 186 insertions(+), 18 deletions(-) delete mode 100644 .gx/lastpubver create mode 100644 store.go diff --git a/.gx/lastpubver b/.gx/lastpubver deleted file mode 100644 index 4c8b569..0000000 --- a/.gx/lastpubver +++ /dev/null @@ -1 +0,0 @@ -1.5.10: QmVwsD9oGSAKcCoD4BGi3Hx8XRo4jNy6aMzsjDHVTgYv2D diff --git a/Makefile b/Makefile index 7811c09..0ad4560 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,14 @@ -gx: - go get github.com/whyrusleeping/gx - go get github.com/whyrusleeping/gx-go +all: build -covertools: - go get github.com/mattn/goveralls - go get golang.org/x/tools/cmd/cover +build: + go build ./... +.PHONY: build -deps: gx covertools - gx --verbose install --global - gx-go rewrite +test: + go test ./... +.PHONY: test -publish: - gx-go rewrite --undo +benchmark: + go test -bench=./... +.PHONY: benchmark diff --git a/go.mod b/go.mod index 80f8c27..51b8bd2 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,14 @@ module github.com/ipfs/go-ipld-cbor require ( github.com/ipfs/go-block-format v0.0.2 - github.com/ipfs/go-cid v0.0.2 + github.com/ipfs/go-cid v0.0.3 github.com/ipfs/go-ipfs-util v0.0.1 github.com/ipfs/go-ipld-format v0.0.1 - github.com/multiformats/go-multihash v0.0.1 + github.com/multiformats/go-multihash v0.0.10 github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa // indirect github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 // indirect + github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 ) + +go 1.13 diff --git a/go.sum b/go.sum index 9498720..2471a67 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,14 @@ +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gxed/hashland/keccakpg v0.0.1 h1:wrk3uMNaMxbXiHibbPO4S0ymqJMm41WiudyFSs7UnsU= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= -github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJu4ZbSc= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-cid v0.0.1 h1:GBjWPktLnNyX0JiQCNFpUuUSoMw5KMyqrsejHYlILBE= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.2 h1:tuuKaZPU1M6HcejsO3AcYWW8sZ8MTvyxfc4uqB4eFE8= -github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= +github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipld-format v0.0.1 h1:HCu4eB/Gh+KD/Q0M8u888RFkorTWNIL3da4oc5dwc80= @@ -20,23 +19,44 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16 h1:5W7KhL8HVF3XCFOweFD3BNESdnO8ewyYTFT2R+/b8FQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mr-tron/base58 v1.1.0 h1:Y51FGVJ91WBqCEabAi5OPUz38eAx8DakuAm5svLcsfQ= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1 h1:HHwN1K12I+XllBCrqKnhX949Orn4oawPkegHMu2vDqQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= +github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= +github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 h1:bzMe+2coZJYHnhGgVlcQKuRy4FSny4ds8dLQjw5P1XE= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 h1:WXhVOwj2USAXB5oMDwRl3piOux2XMV9TANaYxXHdkoE= +github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 h1:ng3VDlRp5/DHpSWl02R4rM9I+8M2rhmsuLwAMmkLQWE= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d h1:Z0Ahzd7HltpJtjAHHxX8QFP3j1yYgiuvjbjRzDj/KH0= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/store.go b/store.go new file mode 100644 index 0000000..ff655fe --- /dev/null +++ b/store.go @@ -0,0 +1,147 @@ +package cbornode + +import ( + "bytes" + "context" + "fmt" + + block "github.com/ipfs/go-block-format" + cid "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + recbor "github.com/polydawn/refmt/cbor" + atlas "github.com/polydawn/refmt/obj/atlas" + cbg "github.com/whyrusleeping/cbor-gen" +) + +type IpldStore interface { + Get(ctx context.Context, c cid.Cid, out interface{}) error + Put(ctx context.Context, v interface{}) (cid.Cid, error) +} + +type IpldBlockstore interface { + Get(cid.Cid) (block.Block, error) + Put(block.Block) error +} + +type BasicIpldStore struct { + Blocks IpldBlockstore + Atlas *atlas.Atlas +} + +var _ IpldStore = &BasicIpldStore{} + +func NewCborStore(bs IpldBlockstore) *BasicIpldStore { + return &BasicIpldStore{Blocks: bs} +} + +func (s *BasicIpldStore) Get(ctx context.Context, c cid.Cid, out interface{}) error { + blk, err := s.Blocks.Get(c) + if err != nil { + return err + } + + cu, ok := out.(cbg.CBORUnmarshaler) + if ok { + if err := cu.UnmarshalCBOR(bytes.NewReader(blk.RawData())); err != nil { + return NewSerializationError(err) + } + return nil + } + + if s.Atlas == nil { + return DecodeInto(blk.RawData(), out) + } else { + return recbor.UnmarshalAtlased(recbor.DecodeOptions{}, blk.RawData(), out, *s.Atlas) + } +} + +type cidProvider interface { + Cid() cid.Cid +} + +func (s *BasicIpldStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) { + mhType := uint64(mh.BLAKE2B_MIN + 31) + mhLen := -1 + codec := uint64(cid.DagCBOR) + + var expCid cid.Cid + if c, ok := v.(cidProvider); ok { + expCid := c.Cid() + pref := expCid.Prefix() + mhType = pref.MhType + mhLen = pref.MhLength + codec = pref.Codec + } + + cm, ok := v.(cbg.CBORMarshaler) + if ok { + buf := new(bytes.Buffer) + if err := cm.MarshalCBOR(buf); err != nil { + return cid.Undef, err + } + + pref := cid.Prefix{ + Codec: codec, + MhType: mhType, + MhLength: mhLen, + Version: 1, + } + c, err := pref.Sum(buf.Bytes()) + if err != nil { + return cid.Undef, err + } + + blk, err := block.NewBlockWithCid(buf.Bytes(), c) + if err != nil { + return cid.Undef, err + } + + if err := s.Blocks.Put(blk); err != nil { + return cid.Undef, err + } + + blkCid := blk.Cid() + if expCid != cid.Undef && blkCid != expCid { + return cid.Undef, fmt.Errorf("your object is not being serialized the way it expects to") + } + + return blkCid, nil + } + + nd, err := WrapObject(v, mhType, mhLen) + if err != nil { + return cid.Undef, err + } + + if err := s.Blocks.Put(nd); err != nil { + return cid.Undef, err + } + + ndCid := nd.Cid() + if expCid != cid.Undef && ndCid != expCid { + return cid.Undef, fmt.Errorf("your object is not being serialized the way it expects to") + } + + return ndCid, nil +} + +func NewSerializationError(err error) error { + return SerializationError{err} +} + +type SerializationError struct { + err error +} + +func (se SerializationError) Error() string { + return se.err.Error() +} + +func (se SerializationError) Unwrap() error { + return se.err +} + +func (se SerializationError) Is(o error) bool { + _, ok := o.(*SerializationError) + return ok +}