From 48877cac61281bafe6a08d720ce692b5168e2c3a Mon Sep 17 00:00:00 2001 From: Iaroslav Gridin Date: Mon, 14 Sep 2020 19:40:16 +0300 Subject: [PATCH] Named pins, storing pins in Datastore * Change pin structure to a tree stored in KV store * Adjust pin-related commands accordingly * Make pinning routines use meaningful default prefixes License: MIT Signed-off-by: Iaroslav Gridin --- assets/assets.go | 3 +- core/commands/add.go | 4 + core/commands/dag/dag.go | 22 +- core/commands/object/object.go | 11 +- core/commands/pin.go | 171 ++++------- core/commands/urlstore.go | 5 + core/coreapi/block.go | 9 +- core/coreapi/dag.go | 42 ++- core/coreapi/object.go | 8 +- core/coreapi/pin.go | 271 ++++++++---------- core/coreapi/unixfs.go | 1 + core/coreunix/add.go | 6 +- core/node/core.go | 22 +- fuse/ipns/common.go | 10 - gc/gc.go | 59 +--- go.mod | 15 +- namesys/publisher.go | 12 - test/sharness/t0010-basic-commands.sh | 4 +- test/sharness/t0025-datastores.sh | 2 +- test/sharness/t0040-add-and-cat.sh | 6 +- test/sharness/t0046-id-hash.sh | 2 +- test/sharness/t0050-block.sh | 9 +- test/sharness/t0080-repo.sh | 73 ++--- test/sharness/t0081-repo-pinning.sh | 30 +- test/sharness/t0085-pins.sh | 14 +- test/sharness/t0087-repo-robust-gc.sh | 8 +- test/sharness/t0250-files-api.sh | 2 +- test/sharness/t0252-files-gc.sh | 6 +- test/sharness/t0260-sharding.sh | 2 +- test/sharness/t0272-urlstore.sh | 4 +- test/sharness/t0276-cidv0v1.sh | 2 +- .../t0600-issues-and-regressions-online.sh | 6 +- test/sharness/x0601-pin-fail-test.sh | 4 +- 33 files changed, 337 insertions(+), 508 deletions(-) diff --git a/assets/assets.go b/assets/assets.go index 920488f4f2c..46506b14d5e 100644 --- a/assets/assets.go +++ b/assets/assets.go @@ -66,9 +66,10 @@ func addAssetList(nd *core.IpfsNode, l []string) (cid.Cid, error) { } } - if err := api.Pin().Add(nd.Context(), basePath); err != nil { + if err := api.Pin().Add(nd.Context(), "assets", basePath); err != nil { return cid.Cid{}, err } return basePath.Cid(), nil + } diff --git a/core/commands/add.go b/core/commands/add.go index df8dd5f9bbb..0a009918a3e 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -38,6 +38,7 @@ const ( onlyHashOptionName = "only-hash" chunkerOptionName = "chunker" pinOptionName = "pin" + pinPathOptionName = "pinpath" rawLeavesOptionName = "raw-leaves" noCopyOptionName = "nocopy" fstoreCacheOptionName = "fscache" @@ -133,6 +134,7 @@ only-hash, and progress/status related flags) will change the final hash. cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."), cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"), cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), + cmds.StringOption(pinPathOptionName, "P", "Pin object under this path.").WithDefault("added/"), cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"), cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"), cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"), @@ -173,6 +175,7 @@ only-hash, and progress/status related flags) will change the final hash. silent, _ := req.Options[silentOptionName].(bool) chunker, _ := req.Options[chunkerOptionName].(string) dopin, _ := req.Options[pinOptionName].(bool) + pinPath, _ := req.Options[pinPathOptionName].(string) rawblks, rbset := req.Options[rawLeavesOptionName].(bool) nocopy, _ := req.Options[noCopyOptionName].(bool) fscache, _ := req.Options[fstoreCacheOptionName].(bool) @@ -207,6 +210,7 @@ only-hash, and progress/status related flags) will change the final hash. options.Unixfs.Chunker(chunker), options.Unixfs.Pin(dopin), + options.Unixfs.PinPath(pinPath), options.Unixfs.HashOnly(hash), options.Unixfs.FsCache(fscache), options.Unixfs.Nocopy(nocopy), diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 4df6571cc83..fae93704d19 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -94,7 +94,7 @@ into an object of the specified format. Options: []cmds.Option{ cmds.StringOption("format", "f", "Format that the object will be added as.").WithDefault("cbor"), cmds.StringOption("input-enc", "Format that the input object will be.").WithDefault("json"), - cmds.BoolOption("pin", "Pin this object when adding."), + cmds.StringOption("pin", "Pin this object when adding.").WithDefault(""), cmds.StringOption("hash", "Hash function to use").WithDefault(""), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -106,7 +106,7 @@ into an object of the specified format. ienc, _ := req.Options["input-enc"].(string) format, _ := req.Options["format"].(string) hash, _ := req.Options["hash"].(string) - dopin, _ := req.Options["pin"].(bool) + pinpath, _ := req.Options["pin"].(string) // mhType tells inputParser which hash should be used. MaxUint64 means 'use // default hash' (sha256 for cbor, sha1 for git..) @@ -121,9 +121,6 @@ into an object of the specified format. } var adder ipld.NodeAdder = api.Dag() - if dopin { - adder = api.Dag().Pinning() - } b := ipld.NewBatch(req.Context, adder) it := req.Files.Entries() @@ -140,10 +137,13 @@ into an object of the specified format. return fmt.Errorf("no node returned from ParseInputs") } - for _, nd := range nds { - err := b.Add(req.Context, nd) - if err != nil { - return err + if pinpath != "" { + + for _, nd := range nds { + err := api.Pin().Add(req.Context, pinpath, path.IpfsPath(nd.Cid())) + if err != nil { + return err + } } } @@ -387,9 +387,7 @@ Maximum supported CAR version: 1 ret.PinErrorMsg = err.Error() } else if nd, err := ipld.Decode(block); err != nil { ret.PinErrorMsg = err.Error() - } else if err := node.Pinning.Pin(req.Context, nd, true); err != nil { - ret.PinErrorMsg = err.Error() - } else if err := node.Pinning.Flush(req.Context); err != nil { + } else if err := node.Pinning.Pin(req.Context, "car-import/", nd, true); err != nil { ret.PinErrorMsg = err.Error() } diff --git a/core/commands/object/object.go b/core/commands/object/object.go index b5858546f75..75e2f97c07d 100644 --- a/core/commands/object/object.go +++ b/core/commands/object/object.go @@ -404,6 +404,7 @@ And then run: cmds.StringOption(datafieldencOptionName, "Encoding type of the data field, either \"text\" or \"base64\".").WithDefault("text"), cmds.BoolOption(pinOptionName, "Pin this object when adding."), cmds.BoolOption(quietOptionName, "q", "Write minimal output."), + cmds.StringOption("pinpath", "Pin under this path").WithDefault("added/"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) @@ -431,7 +432,12 @@ And then run: return err } - dopin, _ := req.Options[pinOptionName].(bool) + pinpath, _ := req.Options["pinpath"].(string) + if err != nil { + return err + } + + pin, _ := req.Options["pin"].(bool) if err != nil { return err } @@ -439,7 +445,8 @@ And then run: p, err := api.Object().Put(req.Context, file, options.Object.DataType(datafieldenc), options.Object.InputEnc(inputenc), - options.Object.Pin(dopin)) + options.Object.Pin(pin), + options.Object.PinPath(pinpath)) if err != nil { return err } diff --git a/core/commands/pin.go b/core/commands/pin.go index 9d209040a59..bb252ca31f6 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -59,11 +59,12 @@ var addPinCmd = &cmds.Command{ }, Arguments: []cmds.Argument{ - cmds.StringArg("ipfs-path", true, true, "Path to object(s) to be pinned.").EnableStdin(), + cmds.StringArg("ipfs-path", true, true, "Path(es) to object(s) to be pinned.").EnableStdin(), }, Options: []cmds.Option{ - cmds.BoolOption(pinRecursiveOptionName, "r", "Recursively pin the object linked to by the specified object(s).").WithDefault(true), + cmds.BoolOption("direct", "d", "Pin the object directly"), cmds.BoolOption(pinProgressOptionName, "Show progress"), + cmds.StringOption("pinpath", "P", "Pin path.").WithDefault("default/"), }, Type: AddPinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -72,11 +73,11 @@ var addPinCmd = &cmds.Command{ return err } - // set recursive flag - recursive, _ := req.Options[pinRecursiveOptionName].(bool) + // set direct flag + direct, _ := req.Options["direct"].(bool) showProgress, _ := req.Options[pinProgressOptionName].(bool) - if err := req.ParseBodyArgs(); err != nil { + if err = req.ParseBodyArgs(); err != nil { return err } @@ -85,8 +86,13 @@ var addPinCmd = &cmds.Command{ return err } + pinPath, _ := req.Options["pinpath"].(string) + + toPin := req.Arguments + if !showProgress { - added, err := pinAddMany(req.Context, api, enc, req.Arguments, recursive) + added, err := pinAddMany(req.Context, api, enc, pinPath, toPin, !direct) + if err != nil { return err } @@ -104,7 +110,7 @@ var addPinCmd = &cmds.Command{ ch := make(chan pinResult, 1) go func() { - added, err := pinAddMany(ctx, api, enc, req.Arguments, recursive) + added, err := pinAddMany(ctx, api, enc, pinPath, toPin, !direct) ch <- pinResult{pins: added, err: err} }() @@ -136,9 +142,10 @@ var addPinCmd = &cmds.Command{ }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *AddPinOutput) error { - rec, found := req.Options["recursive"].(bool) + direct, found := req.Options["direct"].(bool) + var pintype string - if rec || !found { + if !direct || !found { pintype = "recursively" } else { pintype = "directly" @@ -180,7 +187,7 @@ var addPinCmd = &cmds.Command{ }, } -func pinAddMany(ctx context.Context, api coreiface.CoreAPI, enc cidenc.Encoder, paths []string, recursive bool) ([]string, error) { +func pinAddMany(ctx context.Context, api coreiface.CoreAPI, enc cidenc.Encoder, pinPath string, paths []string, recursive bool) ([]string, error) { added := make([]string, len(paths)) for i, b := range paths { rp, err := api.ResolvePath(ctx, path.New(b)) @@ -188,7 +195,7 @@ func pinAddMany(ctx context.Context, api coreiface.CoreAPI, enc cidenc.Encoder, return nil, err } - if err := api.Pin().Add(ctx, rp, options.Pin.Recursive(recursive)); err != nil { + if err := api.Pin().Add(ctx, pinPath, rp, options.Pin.Recursive(recursive)); err != nil { return nil, err } added[i] = enc.Encode(rp.Cid()) @@ -202,15 +209,15 @@ var rmPinCmd = &cmds.Command{ Tagline: "Remove pinned objects from local storage.", ShortDescription: ` Removes the pin from the given object allowing it to be garbage -collected if needed. (By default, recursively. Use -r=false for direct pins.) +collected if needed. By default, removes recursive pins. `, }, Arguments: []cmds.Argument{ - cmds.StringArg("ipfs-path", true, true, "Path to object(s) to be unpinned.").EnableStdin(), + cmds.StringArg("pin-path", true, true, "Pin paths").EnableStdin(), }, Options: []cmds.Option{ - cmds.BoolOption(pinRecursiveOptionName, "r", "Recursively unpin the object linked to by the specified object(s).").WithDefault(true), + cmds.BoolOption("direct", "d", "Unpins a direct pin").WithDefault(false), }, Type: PinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -220,32 +227,20 @@ collected if needed. (By default, recursively. Use -r=false for direct pins.) } // set recursive flag - recursive, _ := req.Options[pinRecursiveOptionName].(bool) + direct, _ := req.Options["direct"].(bool) + recursive := !direct if err := req.ParseBodyArgs(); err != nil { return err } - enc, err := cmdenv.GetCidEncoder(req) - if err != nil { - return err - } - - pins := make([]string, 0, len(req.Arguments)) for _, b := range req.Arguments { - rp, err := api.ResolvePath(req.Context, path.New(b)) - if err != nil { - return err - } - - id := enc.Encode(rp.Cid()) - pins = append(pins, id) - if err := api.Pin().Rm(req.Context, rp, options.Pin.RmRecursive(recursive)); err != nil { + if err := api.Pin().Rm(req.Context, b, options.Pin.RmRecursive(recursive)); err != nil { return err } } - return cmds.EmitOnce(res, &PinOutput{pins}) + return cmds.EmitOnce(res, &PinOutput{req.Arguments}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinOutput) error { @@ -314,6 +309,7 @@ Example: Options: []cmds.Option{ cmds.StringOption(pinTypeOptionName, "t", "The type of pinned keys to list. Can be \"direct\", \"indirect\", \"recursive\", or \"all\".").WithDefault("all"), cmds.BoolOption(pinQuietOptionName, "q", "Write just hashes of objects."), + cmds.BoolOption("recursive", "r", "List all pins under given prefix"), cmds.BoolOption(pinStreamOptionName, "s", "Enable streaming of pins as they are discovered."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -324,6 +320,7 @@ Example: typeStr, _ := req.Options[pinTypeOptionName].(string) stream, _ := req.Options[pinStreamOptionName].(bool) + recursive, _ := req.Options["recursive"].(bool) switch typeStr { case "all", "direct", "indirect", "recursive": @@ -332,22 +329,24 @@ Example: return err } + prefix := "" + if len(req.Arguments) == 1 { + prefix = req.Arguments[0] + } + // For backward compatibility, we accumulate the pins in the same output type as before. emit := res.Emit lgcList := map[string]PinLsType{} if !stream { emit = func(v interface{}) error { obj := v.(*PinLsOutputWrapper) - lgcList[obj.PinLsObject.Cid] = PinLsType{Type: obj.PinLsObject.Type} + lgcList[obj.PinLsObject.Cid] = PinLsType{Type: obj.PinLsObject.Type, PinPath: obj.PinLsObject.Path} return nil } } - if len(req.Arguments) > 0 { - err = pinLsKeys(req, typeStr, api, emit) - } else { - err = pinLsAll(req, typeStr, api, emit) - } + err = pinLsAll(req, prefix, typeStr, recursive, api, emit) + if err != nil { return err } @@ -381,7 +380,7 @@ Example: if quiet { fmt.Fprintf(w, "%s\n", out.PinLsObject.Cid) } else { - fmt.Fprintf(w, "%s %s\n", out.PinLsObject.Cid, out.PinLsObject.Type) + fmt.Fprintf(w, "%s %s %s\n", out.PinLsObject.Cid, out.PinLsObject.Type, out.PinLsObject.Path) } return nil } @@ -390,7 +389,7 @@ Example: if quiet { fmt.Fprintf(w, "%s\n", k) } else { - fmt.Fprintf(w, "%s %s\n", k, v.Type) + fmt.Fprintf(w, "%s %s %s\n", k, v.Type, v.PinPath) } } @@ -414,68 +413,18 @@ type PinLsList struct { // PinLsType contains the type of a pin type PinLsType struct { - Type string + Type string + PinPath string } // PinLsObject contains the description of a pin type PinLsObject struct { Cid string `json:",omitempty"` Type string `json:",omitempty"` + Path string `json:",omitempty"` } -func pinLsKeys(req *cmds.Request, typeStr string, api coreiface.CoreAPI, emit func(value interface{}) error) error { - enc, err := cmdenv.GetCidEncoder(req) - if err != nil { - return err - } - - switch typeStr { - case "all", "direct", "indirect", "recursive": - default: - return fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", typeStr) - } - - opt, err := options.Pin.IsPinned.Type(typeStr) - if err != nil { - panic("unhandled pin type") - } - - for _, p := range req.Arguments { - rp, err := api.ResolvePath(req.Context, path.New(p)) - if err != nil { - return err - } - - pinType, pinned, err := api.Pin().IsPinned(req.Context, rp, opt) - if err != nil { - return err - } - - if !pinned { - return fmt.Errorf("path '%s' is not pinned", p) - } - - switch pinType { - case "direct", "indirect", "recursive", "internal": - default: - pinType = "indirect through " + pinType - } - - err = emit(&PinLsOutputWrapper{ - PinLsObject: PinLsObject{ - Type: pinType, - Cid: enc.Encode(rp.Cid()), - }, - }) - if err != nil { - return err - } - } - - return nil -} - -func pinLsAll(req *cmds.Request, typeStr string, api coreiface.CoreAPI, emit func(value interface{}) error) error { +func pinLsAll(req *cmds.Request, prefix string, typeStr string, recursive bool, api coreiface.CoreAPI, emit func(value interface{}) error) error { enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err @@ -493,7 +442,9 @@ func pinLsAll(req *cmds.Request, typeStr string, api coreiface.CoreAPI, emit fun panic("unhandled pin type") } - pins, err := api.Pin().Ls(req.Context, opt) + recOpt := options.Pin.Ls.RecursiveList(recursive) + + pins, err := api.Pin().Ls(req.Context, prefix, opt, recOpt) if err != nil { return err } @@ -506,6 +457,7 @@ func pinLsAll(req *cmds.Request, typeStr string, api coreiface.CoreAPI, emit fun PinLsObject: PinLsObject{ Type: p.Type(), Cid: enc.Encode(p.Path().Cid()), + Path: p.PinPath(), }, }) if err != nil { @@ -537,41 +489,30 @@ pin. Arguments: []cmds.Argument{ cmds.StringArg("from-path", true, false, "Path to old object."), + cmds.StringArg("pinpath", true, false, "Pin path of the old object."), cmds.StringArg("to-path", true, false, "Path to a new object to be pinned."), }, - Options: []cmds.Option{ - cmds.BoolOption(pinUnpinOptionName, "Remove the old pin.").WithDefault(true), - }, - Type: PinOutput{}, + Options: []cmds.Option{}, + Type: PinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } - enc, err := cmdenv.GetCidEncoder(req) - if err != nil { - return err - } - - unpin, _ := req.Options[pinUnpinOptionName].(bool) + from := req.Arguments[0] - // Resolve the paths ahead of time so we can return the actual CIDs - from, err := api.ResolvePath(req.Context, path.New(req.Arguments[0])) - if err != nil { - return err - } to, err := api.ResolvePath(req.Context, path.New(req.Arguments[1])) if err != nil { return err } - err = api.Pin().Update(req.Context, from, to, options.Pin.Unpin(unpin)) + err = api.Pin().Update(req.Context, from, to) if err != nil { return err } - return cmds.EmitOnce(res, &PinOutput{Pins: []string{enc.Encode(from.Cid()), enc.Encode(to.Cid())}}) + return cmds.EmitOnce(res, &PinOutput{Pins: []string{from, to.String()}}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinOutput) error { @@ -666,9 +607,15 @@ func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts, enc ci bs := n.Blocks.Blockstore() DAG := dag.NewDAGService(bserv.New(bs, offline.Exchange(bs))) getLinks := dag.GetLinksWithDAG(DAG) - recPins, err := n.Pinning.RecursiveKeys(ctx) + + recPins, err := n.Pinning.PinnedCids(true) if err != nil { - return nil, err + status := PinStatus{Ok: false} + // TODO: do something about this error reporting + status.BadNodes = []BadNode{BadNode{Err: err.Error()}} + out := make(chan interface{}) + out <- &PinVerifyRes{"", status} + return out, err } var checkPin func(root cid.Cid) PinStatus diff --git a/core/commands/urlstore.go b/core/commands/urlstore.go index c71229b5a6e..59dd828676c 100644 --- a/core/commands/urlstore.go +++ b/core/commands/urlstore.go @@ -40,6 +40,7 @@ settings for 'ipfs add'. Options: []cmds.Option{ cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), + cmds.StringOption("pinpath", "P", "Pin object under this path.").WithDefault("added/"), }, Arguments: []cmds.Argument{ cmds.StringArg("url", true, false, "URL to add to IPFS"), @@ -71,9 +72,11 @@ settings for 'ipfs add'. useTrickledag, _ := req.Options[trickleOptionName].(bool) dopin, _ := req.Options[pinOptionName].(bool) + pinPath, _ := req.Options["pinpath"].(string) opts := []options.UnixfsAddOption{ options.Unixfs.Pin(dopin), + options.Unixfs.PinPath(pinPath), options.Unixfs.CidVersion(1), options.Unixfs.RawLeaves(true), options.Unixfs.Nocopy(true), @@ -89,7 +92,9 @@ settings for 'ipfs add'. if err != nil { return err } + size, _ := file.Size() + return cmds.EmitOnce(res, &BlockStat{ Key: enc.Encode(path.Cid()), Size: int(size), diff --git a/core/coreapi/block.go b/core/coreapi/block.go index 18d9fb763d6..27643b5748a 100644 --- a/core/coreapi/block.go +++ b/core/coreapi/block.go @@ -9,7 +9,6 @@ import ( blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" - pin "github.com/ipfs/go-ipfs-pinner" coreiface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" path "github.com/ipfs/interface-go-ipfs-core/path" @@ -45,18 +44,14 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc return nil, err } - if settings.Pin { - defer api.blockstore.PinLock().Unlock() - } - err = api.blocks.AddBlock(b) if err != nil { return nil, err } if settings.Pin { - api.pinning.PinWithMode(b.Cid(), pin.Recursive) - if err := api.pinning.Flush(ctx); err != nil { + err = api.pinning.AddPin(settings.PinPath, b.Cid(), true) + if err != nil { return nil, err } } diff --git a/core/coreapi/dag.go b/core/coreapi/dag.go index 7994842b8e5..5ff8a3b2d19 100644 --- a/core/coreapi/dag.go +++ b/core/coreapi/dag.go @@ -3,8 +3,6 @@ package coreapi import ( "context" - cid "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-pinner" ipld "github.com/ipfs/go-ipld-format" dag "github.com/ipfs/go-merkledag" ) @@ -18,34 +16,28 @@ type dagAPI struct { type pinningAdder CoreAPI func (adder *pinningAdder) Add(ctx context.Context, nd ipld.Node) error { - defer adder.blockstore.PinLock().Unlock() - if err := adder.dag.Add(ctx, nd); err != nil { - return err - } - - adder.pinning.PinWithMode(nd.Cid(), pin.Recursive) - - return adder.pinning.Flush(ctx) + return adder.dag.Add(ctx, nd) } func (adder *pinningAdder) AddMany(ctx context.Context, nds []ipld.Node) error { - defer adder.blockstore.PinLock().Unlock() - - if err := adder.dag.AddMany(ctx, nds); err != nil { - return err - } - - cids := cid.NewSet() - - for _, nd := range nds { - c := nd.Cid() - if cids.Visit(c) { - adder.pinning.PinWithMode(c, pin.Recursive) - } - } - return adder.pinning.Flush(ctx) + return adder.dag.AddMany(ctx, nds) // err != nil { + // return err + // } + // + // cids := cid.NewSet() + // + // for _, nd := range nds { + // c := nd.Cid() + // if cids.Visit(c) { + // err := adder.pinning.AddPin(pinPath, c, true) + // if err != nil { + // return err + // } + // } + // } + // return nil } func (api *dagAPI) Pinning() ipld.NodeAdder { diff --git a/core/coreapi/object.go b/core/coreapi/object.go index 413d7954973..58639942ed9 100644 --- a/core/coreapi/object.go +++ b/core/coreapi/object.go @@ -12,7 +12,6 @@ import ( "io/ioutil" cid "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-pinner" ipld "github.com/ipfs/go-ipld-format" dag "github.com/ipfs/go-merkledag" "github.com/ipfs/go-merkledag/dagutils" @@ -107,18 +106,13 @@ func (api *ObjectAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Obj return nil, err } - if options.Pin { - defer api.blockstore.PinLock().Unlock() - } - err = api.dag.Add(ctx, dagnode) if err != nil { return nil, err } if options.Pin { - api.pinning.PinWithMode(dagnode.Cid(), pin.Recursive) - err = api.pinning.Flush(ctx) + err = api.pinning.AddPin(options.PinPath, dagnode.Cid(), true) if err != nil { return nil, err } diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go index 3b6ae2d9395..d782a147b19 100644 --- a/core/coreapi/pin.go +++ b/core/coreapi/pin.go @@ -16,20 +16,20 @@ import ( type PinAPI CoreAPI -func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOption) error { - dagNode, err := api.core().ResolveNode(ctx, p) +func (api *PinAPI) Add(ctx context.Context, pinpath string, p path.Path, opts ...caopts.PinAddOption) error { + settings, err := caopts.PinAddOptions(opts...) if err != nil { - return fmt.Errorf("pin: %s", err) + return nil } - settings, err := caopts.PinAddOptions(opts...) + dagNode, err := api.core().ResolveNode(ctx, p) + if err != nil { - return err + return fmt.Errorf("pin: %s", err) } - defer api.blockstore.PinLock().Unlock() + err = api.pinning.Pin(ctx, pinpath, dagNode, settings.Recursive) - err = api.pinning.Pin(ctx, dagNode, settings.Recursive) if err != nil { return fmt.Errorf("pin: %s", err) } @@ -38,10 +38,10 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp return err } - return api.pinning.Flush(ctx) + return nil } -func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan coreiface.Pin, error) { +func (api *PinAPI) Ls(ctx context.Context, prefix string, opts ...caopts.PinLsOption) (<-chan coreiface.Pin, error) { settings, err := caopts.PinLsOptions(opts...) if err != nil { return nil, err @@ -53,81 +53,60 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan c return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.Type) } - return api.pinLsAll(ctx, settings.Type), nil -} - -func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.PinIsPinnedOption) (string, bool, error) { - resolved, err := api.core().ResolvePath(ctx, p) - if err != nil { - return "", false, fmt.Errorf("error resolving path: %s", err) - } - - settings, err := caopts.PinIsPinnedOptions(opts...) - if err != nil { - return "", false, err - } - - mode, ok := pin.StringToMode(settings.WithType) - if !ok { - return "", false, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.WithType) - } - - return api.pinning.IsPinnedWithType(ctx, resolved.Cid(), mode) + return api.pinLsAll(settings.Type, prefix, ctx, settings.Recursive), nil } // Rm pin rm api -func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOption) error { - rp, err := api.core().ResolvePath(ctx, p) - if err != nil { - return err - } +func (api *PinAPI) Rm(ctx context.Context, p string, opts ...caopts.PinRmOption) error { settings, err := caopts.PinRmOptions(opts...) if err != nil { return err } - // Note: after unpin the pin sets are flushed to the blockstore, so we need - // to take a lock to prevent a concurrent garbage collection - defer api.blockstore.PinLock().Unlock() + return api.pinning.Unpin(p, settings.Recursive) +} - if err = api.pinning.Unpin(ctx, rp.Cid(), settings.Recursive); err != nil { +func (api *PinAPI) Update(ctx context.Context, from string, to path.Path, opts ...caopts.PinUpdateOption) error { + tp, err := api.core().ResolvePath(ctx, to) + if err != nil { return err } - return api.pinning.Flush(ctx) + return api.pinning.Update(ctx, from, tp.Cid()) } -func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opts ...caopts.PinUpdateOption) error { - settings, err := caopts.PinUpdateOptions(opts...) - if err != nil { - return err - } +type pinStatus struct { + pinPath string + ok bool + badNodes []coreiface.BadPinNode +} - fp, err := api.core().ResolvePath(ctx, from) +func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.PinIsPinnedOption) (string, bool, error) { + resolved, err := api.core().ResolvePath(ctx, p) if err != nil { - return err + return "", false, fmt.Errorf("error resolving path: %s", err) } - tp, err := api.core().ResolvePath(ctx, to) + settings, err := caopts.PinIsPinnedOptions(opts...) if err != nil { - return err + return "", false, err } - defer api.blockstore.PinLock().Unlock() - - err = api.pinning.Update(ctx, fp.Cid(), tp.Cid(), settings.Unpin) + pinneds, err := api.pinning.CheckIfPinned(ctx, resolved.Cid()) if err != nil { - return err + return "", false, err } - return api.pinning.Flush(ctx) -} - -type pinStatus struct { - cid cid.Cid - ok bool - badNodes []coreiface.BadPinNode + for _, pinned := range pinneds { + if settings.WithType == "all" || + settings.WithType == "direct" && pinned.Mode == pin.Direct || + settings.WithType == "indirect" && pinned.Mode == pin.Indirect || + settings.WithType == "recursive" && pinned.Mode == pin.Recursive { + return pinned.String(), true, nil + } + } + return "not pinned", false, nil } // BadNode is used in PinVerifyRes @@ -140,6 +119,10 @@ func (s *pinStatus) Ok() bool { return s.ok } +func (s *pinStatus) PinPath() string { + return s.pinPath +} + func (s *pinStatus) BadNodes() []coreiface.BadPinNode { return s.badNodes } @@ -157,28 +140,30 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro bs := api.blockstore DAG := merkledag.NewDAGService(bserv.New(bs, offline.Exchange(bs))) getLinks := merkledag.GetLinksWithDAG(DAG) - recPins, err := api.pinning.RecursiveKeys(ctx) + recPins, err := api.pinning.PrefixedPins("", true) + if err != nil { return nil, err } - var checkPin func(root cid.Cid) *pinStatus - checkPin = func(root cid.Cid) *pinStatus { - if status, ok := visited[root]; ok { + var checkPin func(root cid.Cid, pinPath string) *pinStatus + checkPin = func(root cid.Cid, pinPath string) *pinStatus { + status, ok := visited[root] + if ok { return status } links, err := getLinks(ctx, root) if err != nil { - status := &pinStatus{ok: false, cid: root} + status := &pinStatus{ok: false, pinPath: pinPath} status.badNodes = []coreiface.BadPinNode{&badNode{path: path.IpldPath(root), err: err}} visited[root] = status return status } - status := &pinStatus{ok: true, cid: root} + status = &pinStatus{ok: true} for _, lnk := range links { - res := checkPin(lnk.Cid) + res := checkPin(lnk.Cid, pinPath) if !res.ok { status.ok = false status.badNodes = append(status.badNodes, res.badNodes...) @@ -192,8 +177,8 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro out := make(chan coreiface.PinStatus) go func() { defer close(out) - for _, c := range recPins { - out <- checkPin(c) + for path, c := range recPins { + out <- checkPin(c, path) } }() @@ -202,10 +187,15 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro type pinInfo struct { pinType string + pinPath string path path.Resolved err error } +func (p *pinInfo) PinPath() string { + return p.pinPath +} + func (p *pinInfo) Path() path.Resolved { return p.path } @@ -219,30 +209,47 @@ func (p *pinInfo) Err() error { } // pinLsAll is an internal function for returning a list of pins -func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string) <-chan coreiface.Pin { +func (api *PinAPI) pinLsAll(typeStr string, prefix string, ctx context.Context, recursive bool) <-chan coreiface.Pin { out := make(chan coreiface.Pin) - keys := cid.NewSet() + var recursiveMap map[string]cid.Cid + var directMap map[string]cid.Cid - AddToResultKeys := func(keyList []cid.Cid, typeStr string) error { - for _, c := range keyList { - if keys.Visit(c) { - select { - case out <- &pinInfo{ - pinType: typeStr, - path: path.IpldPath(c), - }: - case <-ctx.Done(): - return ctx.Err() - } - } - } - return nil + if api.nd == nil { + out <- &pinInfo{err: fmt.Errorf("cannot apply options to api without node")} + return out } - VisitKeys := func(keyList []cid.Cid) { - for _, c := range keyList { - keys.Visit(c) + n := api.nd + + if recursive { + var err error + recursiveMap, err = n.Pinning.PrefixedPins(prefix, true) + if err != nil { + out <- &pinInfo{err: err} + return out + } + directMap, err = n.Pinning.PrefixedPins(prefix, false) + if err != nil { + out <- &pinInfo{err: err} + return out + } + } else { + recursiveMap = make(map[string]cid.Cid) + c, err := n.Pinning.GetPin(prefix, true) + if err != pin.ErrNotPinned && err != nil { + out <- &pinInfo{err: err} + return out + } else if err != pin.ErrNotPinned { + recursiveMap[prefix] = *c + } + directMap = make(map[string]cid.Cid) + c, err = n.Pinning.GetPin(prefix, false) + if err != pin.ErrNotPinned && err != nil { + out <- &pinInfo{err: err} + return out + } else if err != pin.ErrNotPinned { + directMap[prefix] = *c } } @@ -250,85 +257,43 @@ func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string) <-chan coreifac defer close(out) if typeStr == "recursive" || typeStr == "all" { - rkeys, err := api.pinning.RecursiveKeys(ctx) - if err != nil { - out <- &pinInfo{err: err} - return - } - if err := AddToResultKeys(rkeys, "recursive"); err != nil { - out <- &pinInfo{err: err} - return + for pinPath, c := range recursiveMap { + out <- &pinInfo{ + pinType: "recursive", + path: path.IpldPath(c), + pinPath: pinPath, + } } } if typeStr == "direct" || typeStr == "all" { - dkeys, err := api.pinning.DirectKeys(ctx) - if err != nil { - out <- &pinInfo{err: err} - return - } - if err := AddToResultKeys(dkeys, "direct"); err != nil { - out <- &pinInfo{err: err} - return - } - } - if typeStr == "all" { - set := cid.NewSet() - rkeys, err := api.pinning.RecursiveKeys(ctx) - if err != nil { - out <- &pinInfo{err: err} - return - } - for _, k := range rkeys { - err := merkledag.Walk( - ctx, merkledag.GetLinksWithDAG(api.dag), k, - set.Visit, - merkledag.SkipRoot(), merkledag.Concurrent(), - ) - if err != nil { - out <- &pinInfo{err: err} - return + for pinPath, c := range directMap { + out <- &pinInfo{ + pinType: "direct", + path: path.IpldPath(c), + pinPath: pinPath, } } - if err := AddToResultKeys(set.Keys(), "indirect"); err != nil { - out <- &pinInfo{err: err} - return - } } - if typeStr == "indirect" { - // We need to first visit the direct pins that have priority - // without emitting them - - dkeys, err := api.pinning.DirectKeys(ctx) - if err != nil { - out <- &pinInfo{err: err} - return - } - VisitKeys(dkeys) - rkeys, err := api.pinning.RecursiveKeys(ctx) - if err != nil { - out <- &pinInfo{err: err} - return - } - VisitKeys(rkeys) - - set := cid.NewSet() - for _, k := range rkeys { - err := merkledag.Walk( - ctx, merkledag.GetLinksWithDAG(api.dag), k, - set.Visit, - merkledag.SkipRoot(), merkledag.Concurrent(), - ) + if typeStr == "indirect" || typeStr == "all" { + for pinPath, c := range recursiveMap { + set := cid.NewSet() + err := merkledag.Walk(ctx, merkledag.GetLinksWithDAG(api.dag), c, set.Visit, + merkledag.SkipRoot(), merkledag.Concurrent()) if err != nil { out <- &pinInfo{err: err} return } - } - if err := AddToResultKeys(set.Keys(), "indirect"); err != nil { - out <- &pinInfo{err: err} - return + for _, k := range set.Keys() { + out <- &pinInfo{ + pinType: "indirect", + pinPath: pinPath, + path: path.IpldPath(k), + } + } } } + }() return out diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 53d13a3a497..ed141855c1f 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -130,6 +130,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options fileAdder.Progress = settings.Progress } fileAdder.Pin = settings.Pin && !settings.OnlyHash + fileAdder.PinPath = settings.PinPath fileAdder.Silent = settings.Silent fileAdder.RawLeaves = settings.RawLeaves fileAdder.NoCopy = settings.NoCopy diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 68be08056ff..f6673d85121 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -69,6 +69,7 @@ type Adder struct { Out chan<- interface{} Progress bool Pin bool + PinPath string Trickle bool RawLeaves bool Silent bool @@ -171,15 +172,14 @@ func (adder *Adder) PinRoot(root ipld.Node) error { } if adder.tempRoot.Defined() { - err := adder.pinning.Unpin(adder.ctx, adder.tempRoot, true) + err := adder.pinning.UnpinCidUnderPrefix("tmp/temproot/", adder.tempRoot, true) if err != nil { return err } adder.tempRoot = rnk } - adder.pinning.PinWithMode(rnk, pin.Recursive) - return adder.pinning.Flush(adder.ctx) + return adder.pinning.AddPin(adder.PinPath, rnk, true) } func (adder *Adder) outputDirs(path string, fsn mfs.FSNode) error { diff --git a/core/node/core.go b/core/node/core.go index 4c522dbcfc3..8150cf3f0bb 100644 --- a/core/node/core.go +++ b/core/node/core.go @@ -12,7 +12,6 @@ import ( "github.com/ipfs/go-filestore" "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-ipfs-exchange-interface" - "github.com/ipfs/go-ipfs-exchange-offline" "github.com/ipfs/go-ipfs-pinner" "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" @@ -41,26 +40,7 @@ func BlockService(lc fx.Lifecycle, bs blockstore.Blockstore, rem exchange.Interf // Pinning creates new pinner which tells GC which blocks should be kept func Pinning(bstore blockstore.Blockstore, ds format.DAGService, repo repo.Repo) (pin.Pinner, error) { - internalDag := merkledag.NewDAGService(blockservice.New(bstore, offline.Exchange(bstore))) - rootDS := repo.Datastore() - - syncFn := func() error { - if err := rootDS.Sync(blockstore.BlockPrefix); err != nil { - return err - } - return rootDS.Sync(filestore.FilestorePrefix) - } - syncDs := &syncDagService{ds, syncFn} - syncInternalDag := &syncDagService{internalDag, syncFn} - - pinning, err := pin.LoadPinner(rootDS, syncDs, syncInternalDag) - if err != nil { - // TODO: we should move towards only running 'NewPinner' explicitly on - // node init instead of implicitly here as a result of the pinner keys - // not being found in the datastore. - // this is kinda sketchy and could cause data loss - pinning = pin.NewPinner(rootDS, syncDs, syncInternalDag) - } + pinning := pin.NewPinner(ds, repo.Datastore()) return pinning, nil } diff --git a/fuse/ipns/common.go b/fuse/ipns/common.go index 48129a92e98..68dbd4d4866 100644 --- a/fuse/ipns/common.go +++ b/fuse/ipns/common.go @@ -18,16 +18,6 @@ func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error { emptyDir := ft.EmptyDirNode() - err := n.Pinning.Pin(ctx, emptyDir, false) - if err != nil { - return err - } - - err = n.Pinning.Flush(ctx) - if err != nil { - return err - } - pub := nsys.NewIpnsPublisher(n.Routing, n.Repo.Datastore()) return pub.Publish(ctx, key, path.FromCid(emptyDir.Cid())) diff --git a/gc/gc.go b/gc/gc.go index 4b0e6c5d790..74bf4e9f67c 100644 --- a/gc/gc.go +++ b/gc/gc.go @@ -172,9 +172,10 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo // disk backed to conserve memory. errors := false gcs := cid.NewSet() - getLinks := func(ctx context.Context, cid cid.Cid) ([]*ipld.Link, error) { + + bestEffortGetLinks := func(ctx context.Context, cid cid.Cid) ([]*ipld.Link, error) { links, err := ipld.GetLinks(ctx, ng, cid) - if err != nil { + if err != nil && err != ipld.ErrNotFound { errors = true select { case output <- Result{Error: &CannotFetchLinksError{cid, err}}: @@ -184,62 +185,26 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo } return links, nil } - rkeys, err := pn.RecursiveKeys(ctx) - if err != nil { - return nil, err - } - err = Descendants(ctx, getLinks, gcs, rkeys) + recursiveKeys, err := pn.PinnedCids(true) if err != nil { errors = true - select { - case output <- Result{Error: err}: - case <-ctx.Done(): - return nil, ctx.Err() - } - } - - bestEffortGetLinks := func(ctx context.Context, cid cid.Cid) ([]*ipld.Link, error) { - links, err := ipld.GetLinks(ctx, ng, cid) - if err != nil && err != ipld.ErrNotFound { + output <- Result{Error: err} + } else { + err := Descendants(ctx, bestEffortGetLinks, gcs, recursiveKeys) + if err != nil { errors = true select { - case output <- Result{Error: &CannotFetchLinksError{cid, err}}: + case output <- Result{Error: err}: case <-ctx.Done(): return nil, ctx.Err() } } - return links, nil - } - err = Descendants(ctx, bestEffortGetLinks, gcs, bestEffortRoots) - if err != nil { - errors = true - select { - case output <- Result{Error: err}: - case <-ctx.Done(): - return nil, ctx.Err() - } } - dkeys, err := pn.DirectKeys(ctx) - if err != nil { - return nil, err - } - for _, k := range dkeys { - gcs.Add(k) - } + directKeys, err := pn.PinnedCids(false) - ikeys, err := pn.InternalPins(ctx) - if err != nil { - return nil, err - } - err = Descendants(ctx, getLinks, gcs, ikeys) - if err != nil { - errors = true - select { - case output <- Result{Error: err}: - case <-ctx.Done(): - return nil, ctx.Err() - } + for _, k := range directKeys { + gcs.Add(k) } if errors { diff --git a/go.mod b/go.mod index c22d959dada..3b50deeaba0 100644 --- a/go.mod +++ b/go.mod @@ -37,9 +37,9 @@ require ( github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 github.com/ipfs/go-ipfs-files v0.0.8 - github.com/ipfs/go-ipfs-pinner v0.0.4 + github.com/ipfs/go-ipfs-pinner v0.1.0-c5de08a github.com/ipfs/go-ipfs-posinfo v0.0.1 - github.com/ipfs/go-ipfs-provider v0.4.3 + github.com/ipfs/go-ipfs-provider v0.5.0-f4cd5b3 github.com/ipfs/go-ipfs-routing v0.1.0 github.com/ipfs/go-ipfs-util v0.0.2 github.com/ipfs/go-ipld-cbor v0.0.4 @@ -54,12 +54,14 @@ require ( github.com/ipfs/go-path v0.0.8 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 - github.com/ipfs/interface-go-ipfs-core v0.4.0 + github.com/ipfs/interface-go-ipfs-core v0.5.0-89fc7f2 github.com/ipld/go-car v0.1.1-0.20200429200904-c222d793c339 github.com/jbenet/go-is-domain v1.0.5 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 github.com/jbenet/goprocess v0.1.4 + github.com/jessevdk/go-flags v1.4.0 // indirect + github.com/kr/pty v1.1.3 // indirect github.com/libp2p/go-libp2p v0.11.0 github.com/libp2p/go-libp2p-circuit v0.3.1 github.com/libp2p/go-libp2p-connmgr v0.2.4 @@ -87,6 +89,7 @@ require ( github.com/libp2p/go-ws-transport v0.3.1 github.com/lucas-clemente/quic-go v0.18.0 github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-multiaddr v0.3.1 github.com/multiformats/go-multiaddr-dns v0.2.0 @@ -111,3 +114,9 @@ require ( ) go 1.13 + +replace github.com/ipfs/go-ipfs-provider => github.com/Voker57/go-ipfs-provider v0.5.0 + +replace github.com/ipfs/interface-go-ipfs-core => github.com/Voker57/interface-go-ipfs-core v0.1.1-0.20200919202922-89fc7f2a3083 + +replace github.com/ipfs/go-ipfs-pinner => github.com/Voker57/go-ipfs-pinner v0.1.0 diff --git a/namesys/publisher.go b/namesys/publisher.go index f558eaf2885..4b6802126da 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -288,18 +288,6 @@ func PublishEntry(ctx context.Context, r routing.ValueStore, ipnskey string, rec func InitializeKeyspace(ctx context.Context, pub Publisher, pins pin.Pinner, key ci.PrivKey) error { emptyDir := ft.EmptyDirNode() - // pin recursively because this might already be pinned - // and doing a direct pin would throw an error in that case - err := pins.Pin(ctx, emptyDir, true) - if err != nil { - return err - } - - err = pins.Flush(ctx) - if err != nil { - return err - } - return pub.Publish(ctx, key, path.FromCid(emptyDir.Cid())) } diff --git a/test/sharness/t0010-basic-commands.sh b/test/sharness/t0010-basic-commands.sh index 289c3ddc09f..71eba65883a 100755 --- a/test/sharness/t0010-basic-commands.sh +++ b/test/sharness/t0010-basic-commands.sh @@ -41,7 +41,7 @@ test_expect_success "ipfs version --all has all required fields" ' grep "Golang version" version_all.txt ' -test_expect_success "ipfs version deps succeeds" ' +test_expect_success "ipfs version deps fails" ' ipfs version deps >deps.txt ' @@ -139,7 +139,7 @@ test_expect_success "'ipfs commands --flags' succeeds" ' ' test_expect_success "'ipfs commands --flags' output looks good" ' - grep "ipfs pin add --recursive / ipfs pin add -r" commands.txt && + grep "ipfs pin add --direct / ipfs pin add -d" commands.txt && grep "ipfs id --format / ipfs id -f" commands.txt && grep "ipfs repo gc --quiet / ipfs repo gc -q" commands.txt ' diff --git a/test/sharness/t0025-datastores.sh b/test/sharness/t0025-datastores.sh index ec99accb5e7..34e0b7f4333 100755 --- a/test/sharness/t0025-datastores.sh +++ b/test/sharness/t0025-datastores.sh @@ -10,7 +10,7 @@ test_expect_success "'ipfs init --profile=badgerds' succeeds" ' ' test_expect_success "'ipfs pin ls' works" ' - ipfs pin ls | wc -l | grep 9 + ipfs pin ls -r | wc -l | grep 8 ' test_done diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index 365626bb76e..8676badc069 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -387,7 +387,7 @@ test_add_cat_5MB() { ' test_expect_success "remove hash" ' - ipfs pin rm "$EXP_HASH" && + ipfs pin rm "added/$EXP_HASH" && ipfs block rm "$EXP_HASH" ' @@ -509,7 +509,7 @@ test_add_named_pipe() { ( echo foo > named-pipe1 & echo "added $( echo foo | ipfs add -nq ) named-pipe1" > expected_named_pipes_add ) && mkfifo named-pipe2 && ( echo bar > named-pipe2 & echo "added $( echo bar | ipfs add -nq ) named-pipe2" >> expected_named_pipes_add ) && - ipfs add -n named-pipe1 named-pipe2 >actual_pipe_add && + ipfs add -P named-pipe -n named-pipe1 named-pipe2 >actual_pipe_add && rm named-pipe1 && rm named-pipe2 && test_cmp expected_named_pipes_add actual_pipe_add @@ -519,7 +519,7 @@ test_add_named_pipe() { mkdir -p named-pipe-dir && mkfifo named-pipe-dir/named-pipe && STAT=$(generic_stat named-pipe-dir/named-pipe) && - test_expect_code 1 ipfs add -r named-pipe-dir 2>actual && + test_expect_code 1 ipfs add -P named-pipe -r named-pipe-dir 2>actual && printf "Error: unrecognized file type for named-pipe-dir/named-pipe: $STAT\n" >expected && rm named-pipe-dir/named-pipe && rmdir named-pipe-dir && diff --git a/test/sharness/t0046-id-hash.sh b/test/sharness/t0046-id-hash.sh index d4c28f21507..65625cffd66 100755 --- a/test/sharness/t0046-id-hash.sh +++ b/test/sharness/t0046-id-hash.sh @@ -34,7 +34,7 @@ test_expect_success "but can fetch it anyway" ' ' test_expect_success "block rm does nothing" ' - ipfs pin rm $HASH && + ipfs pin rm added/$HASH && ipfs block rm $HASH ' diff --git a/test/sharness/t0050-block.sh b/test/sharness/t0050-block.sh index 70639f623e9..dbe95ec6a73 100755 --- a/test/sharness/t0050-block.sh +++ b/test/sharness/t0050-block.sh @@ -96,7 +96,6 @@ test_expect_success "add and pin directory" ' echo "file2" > adir/file2 && echo "file3" > adir/file3 && ipfs add -r adir - ipfs pin add -r $DIRHASH ' test_expect_success "can't remove pinned block" ' @@ -104,7 +103,7 @@ test_expect_success "can't remove pinned block" ' ' test_expect_success "can't remove pinned block: output looks good" ' - grep -q "$DIRHASH: pinned: recursive" block_rm_err + grep -q "$DIRHASH: pinned under added/$DIRHASH recursively" block_rm_err ' test_expect_success "can't remove indirectly pinned block" ' @@ -112,11 +111,11 @@ test_expect_success "can't remove indirectly pinned block" ' ' test_expect_success "can't remove indirectly pinned block: output looks good" ' - grep -q "$FILE1HASH: pinned via $DIRHASH" block_rm_err + grep -q "$FILE1HASH: pinned under added/$DIRHASH via $DIRHASH" block_rm_err ' test_expect_success "remove pin" ' - ipfs pin rm -r $DIRHASH + ipfs pin rm added/$DIRHASH ' test_expect_success "multi-block 'ipfs block rm' succeeds" ' @@ -148,7 +147,7 @@ test_expect_success "'add some blocks' succeeds" ' test_expect_success "add and pin directory" ' ipfs add -r adir - ipfs pin add -r $DIRHASH + ipfs pin add $DIRHASH ' HASH=QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk diff --git a/test/sharness/t0080-repo.sh b/test/sharness/t0080-repo.sh index 4e0d4800f72..44385f8e358 100755 --- a/test/sharness/t0080-repo.sh +++ b/test/sharness/t0080-repo.sh @@ -21,7 +21,7 @@ test_expect_success "'ipfs add afile' succeeds" ' ' test_expect_success "added file was pinned" ' - ipfs pin ls --type=recursive >actual && + ipfs pin ls -r added/ --type=recursive >actual && grep "$HASH" actual ' @@ -39,11 +39,11 @@ test_expect_success "'ipfs repo gc' doesn't remove file" ' ' test_expect_success "'ipfs pin rm' succeeds" ' - ipfs pin rm -r "$HASH" >actual1 + ipfs pin rm "added/$HASH" >actual1 ' test_expect_success "'ipfs pin rm' output looks good" ' - echo "unpinned $HASH" >expected1 && + echo "unpinned added/$HASH" >expected1 && test_cmp expected1 actual1 ' @@ -52,7 +52,7 @@ test_expect_success "ipfs repo gc fully reverse ipfs add (part 1)" ' random 100000 41 >gcfile && find "$IPFS_PATH/blocks" -type f -name "*.data" | sort -u > expected_blocks && hash=$(ipfs add -q gcfile) && - ipfs pin rm -r $hash && + ipfs pin rm added/$hash && ipfs repo gc ' @@ -66,45 +66,33 @@ test_expect_success "ipfs repo gc fully reverse ipfs add (part 2)" ' test_launch_ipfs_daemon --offline test_expect_success "file no longer pinned" ' - ipfs pin ls --type=recursive --quiet >actual2 && + ipfs pin ls -r --type=recursive --quiet >actual2 && test_expect_code 1 grep $HASH actual2 ' test_expect_success "recursively pin afile(default action)" ' - HASH=`ipfs add -q afile` && + HASH=`ipfs add -q --pin=false afile` && ipfs pin add "$HASH" ' test_expect_success "recursively pin rm afile (default action)" ' - ipfs pin rm "$HASH" + ipfs pin rm "default/$HASH" ' test_expect_success "recursively pin afile" ' - ipfs pin add -r "$HASH" -' - -test_expect_success "pinning directly should fail now" ' - echo "Error: pin: $HASH already pinned recursively" >expected3 && - test_must_fail ipfs pin add -r=false "$HASH" 2>actual3 && - test_cmp expected3 actual3 -' - -test_expect_success "'ipfs pin rm -r=false ' should fail" ' - echo "Error: $HASH is pinned recursively" >expected4 - test_must_fail ipfs pin rm -r=false "$HASH" 2>actual4 && - test_cmp expected4 actual4 + ipfs pin add "$HASH" ' -test_expect_success "remove recursive pin, add direct" ' - echo "unpinned $HASH" >expected5 && - ipfs pin rm -r "$HASH" >actual5 && - test_cmp expected5 actual5 && - ipfs pin add -r=false "$HASH" +test_expect_success "pinning directly should not fail either" ' + ipfs pin add -d "$HASH" ' +test_expect_success "remove recursive pin" ' + echo "unpinned default/$HASH" >expected6 && + ipfs pin rm "default/$HASH" >actual6 && test_expect_success "remove direct pin" ' - echo "unpinned $HASH" >expected6 && - ipfs pin rm "$HASH" >actual6 && + echo "unpinned default/$HASH" >expected6 && + ipfs pin rm -d "default/$HASH" >actual6 && test_cmp expected6 actual6 ' @@ -128,37 +116,38 @@ test_expect_success "adding multiblock random file succeeds" ' ' test_expect_success "'ipfs pin ls --type=indirect' is correct" ' - ipfs refs "$MBLOCKHASH" >refsout && - ipfs refs -r "$HASH_WELCOME_DOCS" >>refsout && - sed -i"~" "s/\(.*\)/\1 indirect/g" refsout && - ipfs pin ls --type=indirect >indirectpins && + ipfs refs "$MBLOCKHASH" >mbrefsout && + ipfs refs -r "$HASH_WELCOME_DOCS" >assetsrefsout && + sed -i"~" "s|\(.*\)|\1 indirect added/$MBLOCKHASH|g" mbrefsout && + sed -i"~" "s/\(.*\)/\1 indirect assets/g" assetsrefsout && + ipfs pin ls -r --type=indirect >indirectpins && + cat assetsrefsout mbrefsout > refsout && test_sort_cmp refsout indirectpins ' test_expect_success "pin something directly" ' echo "ipfs is so awesome" >awesome && DIRECTPIN=`ipfs add -q awesome` && - echo "unpinned $DIRECTPIN" >expected9 && - ipfs pin rm -r "$DIRECTPIN" >actual9 && + echo "unpinned added/$DIRECTPIN" >expected9 && + ipfs pin rm "added/$DIRECTPIN" >actual9 && test_cmp expected9 actual9 && echo "pinned $DIRECTPIN directly" >expected10 && - ipfs pin add -r=false "$DIRECTPIN" >actual10 && + ipfs pin add -d "$DIRECTPIN" >actual10 && test_cmp expected10 actual10 ' test_expect_success "'ipfs pin ls --type=direct' is correct" ' - echo "$DIRECTPIN direct" >directpinexpected && - ipfs pin ls --type=direct >directpinout && + echo "$DIRECTPIN direct default/$DIRECTPIN" >directpinexpected && + ipfs pin ls -r --type=direct >directpinout && test_sort_cmp directpinexpected directpinout ' test_expect_success "'ipfs pin ls --type=recursive' is correct" ' - echo "$MBLOCKHASH" >rp_expected && - echo "$HASH_WELCOME_DOCS" >>rp_expected && - echo "$EMPTY_DIR" >>rp_expected && - sed -i"~" "s/\(.*\)/\1 recursive/g" rp_expected && - ipfs pin ls --type=recursive >rp_actual && + echo "$MBLOCKHASH added/$MBLOCKHASH" >rp_expected && + echo "$HASH_WELCOME_DOCS assets" >>rp_expected && + sed -i"~" -E "s/^([^ ]+)/\1 recursive/g" rp_expected && + ipfs pin ls -r --type=recursive >rp_actual && test_sort_cmp rp_expected rp_actual ' @@ -167,7 +156,7 @@ test_expect_success "'ipfs pin ls --type=all --quiet' is correct" ' cat rp_actual >>allpins && cat indirectpins >>allpins && cut -f1 -d " " allpins | sort | uniq >> allpins_uniq_hashes && - ipfs pin ls --type=all --quiet >actual_allpins && + ipfs pin ls -r --type=all --quiet >actual_allpins && test_sort_cmp allpins_uniq_hashes actual_allpins ' diff --git a/test/sharness/t0081-repo-pinning.sh b/test/sharness/t0081-repo-pinning.sh index 9f1d3c85e20..2fcace4e97f 100755 --- a/test/sharness/t0081-repo-pinning.sh +++ b/test/sharness/t0081-repo-pinning.sh @@ -15,7 +15,7 @@ test_pin_flag() { echo "test_pin_flag" "$@" - if ipfs pin ls --type="$ptype" "$object" >actual + if ipfs pin ls -r --type="$ptype" | grep "^$object" >actual then test "$expect" = "true" && return test_fsh cat actual @@ -141,7 +141,7 @@ test_expect_success "added dir was NOT pinned indirectly" ' ' test_expect_success "nothing is pinned directly" ' - ipfs pin ls --type=direct >actual4 && + ipfs pin ls -r --type=direct >actual4 && test_must_be_empty actual4 ' @@ -166,8 +166,8 @@ test_expect_success "objects are still there" ' ' test_expect_success "remove dir recursive pin succeeds" ' - echo "unpinned $HASH_DIR1" >expected5 && - ipfs pin rm -r "$HASH_DIR1" >actual5 && + echo "unpinned added/$HASH_DIR1" >expected5 && + ipfs pin rm "added/$HASH_DIR1" >actual5 && test_cmp expected5 actual5 ' @@ -178,16 +178,16 @@ test_expect_success "none are pinned any more" ' test_pin "$HASH_FILE3" && test_pin "$HASH_FILE2" && test_pin "$HASH_FILE1" && - test_pin "$HASH_DIR3" && - test_pin "$HASH_DIR4" && - test_pin "$HASH_DIR2" && + test_pin "$HASH_DIR3" && + test_pin "$HASH_DIR4" && + test_pin "$HASH_DIR2" && test_pin "$HASH_DIR1" ' test_expect_success "pin some directly and indirectly" ' - ipfs pin add -r=false "$HASH_DIR1" >actual7 && - ipfs pin add -r=true "$HASH_DIR2" >>actual7 && - ipfs pin add -r=false "$HASH_FILE1" >>actual7 && + ipfs pin add -d "$HASH_DIR1" >actual7 && + ipfs pin add "$HASH_DIR2" >>actual7 && + ipfs pin add -d "$HASH_FILE1" >>actual7 && echo "pinned $HASH_DIR1 directly" >expected7 && echo "pinned $HASH_DIR2 recursively" >>expected7 && echo "pinned $HASH_FILE1 directly" >>expected7 && @@ -195,10 +195,10 @@ test_expect_success "pin some directly and indirectly" ' ' test_expect_success "pin lists look good" ' - test_pin $HASH_DIR1 direct && - test_pin $HASH_DIR2 recursive && - test_pin $HASH_DIR3 && - test_pin $HASH_DIR4 indirect && + test_pin $HASH_DIR1 direct && + test_pin $HASH_DIR2 recursive && + test_pin $HASH_DIR3 && + test_pin $HASH_DIR4 indirect && test_pin $HASH_FILE1 indirect direct && test_pin $HASH_FILE2 indirect && test_pin $HASH_FILE3 && @@ -226,7 +226,7 @@ test_expect_success "some objects are still there" ' ipfs cat "$HASH_FILE1" >>actual8 && ipfs ls "$HASH_DIR4" >>actual8 && ipfs ls "$HASH_DIR2" >>actual8 && - ipfs object links "$HASH_DIR1" >>actual8 && + ipfs object links "$HASH_DIR1" >>actual8 && test_cmp expected8 actual8 ' diff --git a/test/sharness/t0085-pins.sh b/test/sharness/t0085-pins.sh index e05f7dedbbd..069bab40943 100755 --- a/test/sharness/t0085-pins.sh +++ b/test/sharness/t0085-pins.sh @@ -93,18 +93,18 @@ test_pins() { ' test_expect_success "test pin update" ' - ipfs pin add "$HASH_A" && - ipfs pin ls $LS_ARGS $BASE_ARGS | tee before_update && + ipfs pin add -P test/updated_pin "$HASH_A" && + ipfs pin ls $LS_ARGS $BASE_ARGS -r > before_update && test_should_contain "$HASH_A" before_update && test_must_fail grep -q "$HASH_B" before_update && - ipfs pin update --unpin=true "$HASH_A" "$HASH_B" && - ipfs pin ls $LS_ARGS $BASE_ARGS > after_update && + ipfs pin update test/updated_pin "$HASH_B" && + ipfs pin ls $LS_ARGS $BASE_ARGS -r > after_update && test_must_fail grep -q "$HASH_A" after_update && test_should_contain "$HASH_B" after_update && - ipfs pin update --unpin=true "$HASH_B" "$HASH_B" && - ipfs pin ls $LS_ARGS $BASE_ARGS > after_idempotent_update && + ipfs pin update test/updated_pin "$HASH_B" && + ipfs pin ls $LS_ARGS $BASE_ARGS -r > after_idempotent_update && test_should_contain "$HASH_B" after_idempotent_update && - ipfs pin rm "$HASH_B" + ipfs pin rm test/updated_pin ' } diff --git a/test/sharness/t0087-repo-robust-gc.sh b/test/sharness/t0087-repo-robust-gc.sh index 6da96af6c44..8f3b1e1a88d 100755 --- a/test/sharness/t0087-repo-robust-gc.sh +++ b/test/sharness/t0087-repo-robust-gc.sh @@ -13,7 +13,7 @@ test_gc_robust_part1() { test_expect_success "add a 1MB file with --raw-leaves" ' random 1048576 56 > afile && - HASH1=`ipfs add --raw-leaves -q afile` + HASH1=`ipfs add --raw-leaves -q -P test/afile afile` ' HASH1FILE=.ipfs/blocks/L3/CIQNIPL4GP62ZMNNSLZ2G33Z3T5VAN3YHCJTGT5FG45XWH5FGZRXL3A.data @@ -56,7 +56,7 @@ test_gc_robust_part1() { ' test_expect_success "unpin the 1MB file" ' - ipfs pin rm $HASH1 + ipfs pin rm test/afile ' # make sure the permission problem is fixed on exit, otherwise cleanup @@ -91,7 +91,7 @@ test_gc_robust_part2() { test_expect_success "add 1MB file normally (i.e., without raw leaves)" ' random 1048576 56 > afile && - HASH2=`ipfs add -q afile` + HASH2=`ipfs add -q -P test/afile afile` ' LEAF1=QmSijovevteoY63Uj1uC5b8pkpDU5Jgyk2dYBqz3sMJUPc @@ -142,7 +142,7 @@ test_gc_robust_part2() { ' test_expect_success "unpin 1MB file" ' - ipfs pin rm $HASH2 + ipfs pin rm test/afile ' test_expect_success "'ipfs repo gc' should be fine now" ' diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index 5c146b550e0..3a383d2d1f1 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -92,7 +92,7 @@ test_sharding() { test_expect_success "can unpin a file from sharded directory $EXTRA" ' read -r _ HASH _ < pin_hash && - ipfs pin rm $HASH + ipfs pin rm default/$HASH ' test_expect_success "output object was really sharded and has correct hash $EXTRA" ' diff --git a/test/sharness/t0252-files-gc.sh b/test/sharness/t0252-files-gc.sh index 3666d7a481f..7dc67fbe920 100755 --- a/test/sharness/t0252-files-gc.sh +++ b/test/sharness/t0252-files-gc.sh @@ -29,12 +29,12 @@ test_expect_success "gc okay after adding incomplete node -- prep" ' ipfs files mkdir /adir && echo "file1" | ipfs files write --create /adir/file1 && echo "file2" | ipfs files write --create /adir/file2 && - ipfs pin add --recursive=false $ADIR_HASH && + ipfs pin add -d $ADIR_HASH && ipfs files rm -r /adir && ipfs repo gc && # will remove /adir/file1 and /adir/file2 but not /adir test_must_fail ipfs cat $FILE1_HASH && ipfs files cp /ipfs/$ADIR_HASH /adir && - ipfs pin rm $ADIR_HASH + ipfs pin rm -d default/$ADIR_HASH ' test_expect_success "gc okay after adding incomplete node" ' @@ -49,7 +49,7 @@ test_expect_success "add directory with direct pin" ' FILE_UNPINNED=$(ipfs add --pin=false -q -r mydir/hello.txt) && DIR_PINNED=$(ipfs add --pin=false -q -r mydir | tail -n1) && ipfs add --pin=false -r mydir && - ipfs pin add --recursive=false $DIR_PINNED && + ipfs pin add -d $DIR_PINNED && ipfs cat $FILE_UNPINNED ' diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh index 20be436c7a3..e70658a88d8 100755 --- a/test/sharness/t0260-sharding.sh +++ b/test/sharness/t0260-sharding.sh @@ -69,7 +69,7 @@ test_expect_success "'ipfs ls --resolve-type=false --size=false' admits missing ipfs ls "$SHARDED" | head -1 > first_file && ipfs ls --size=false "$SHARDED" | sort > sharded_out_nosize && read -r HASH _ NAME /dev/null ' @@ -131,7 +131,7 @@ EOF ' test_expect_success "remove broken files" ' - ipfs pin rm $HASH2 && + ipfs pin rm default/$HASH2 && ipfs repo gc > /dev/null ' diff --git a/test/sharness/t0276-cidv0v1.sh b/test/sharness/t0276-cidv0v1.sh index dfc6dce3876..8f26d19ebdf 100755 --- a/test/sharness/t0276-cidv0v1.sh +++ b/test/sharness/t0276-cidv0v1.sh @@ -59,7 +59,7 @@ test_expect_success "make sure the CIDv1 hash is not in the repo" ' ' test_expect_success "clean up" ' - ipfs pin rm $AHASHv0 && + ipfs pin rm added/$AHASHv0 && ipfs repo gc && ! ipfs refs local | grep -q $AHASHv0 ' diff --git a/test/sharness/t0600-issues-and-regressions-online.sh b/test/sharness/t0600-issues-and-regressions-online.sh index 93f51caeecb..a14a19e2ffa 100755 --- a/test/sharness/t0600-issues-and-regressions-online.sh +++ b/test/sharness/t0600-issues-and-regressions-online.sh @@ -35,14 +35,14 @@ test_expect_success "metrics work" ' test_expect_success "pin add api looks right - #3753" ' HASH=$(echo "foo" | ipfs add -q) && - curl -X POST "http://$API_ADDR/api/v0/pin/add/$HASH" > pinadd_out && + curl -X POST "http://$API_ADDR/api/v0/pin/add/$HASH?pinpath=pintest" > pinadd_out && echo "{\"Pins\":[\"QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6\"]}" > pinadd_exp && test_cmp pinadd_out pinadd_exp ' test_expect_success "pin add api looks right - #3753" ' - curl -X POST "http://$API_ADDR/api/v0/pin/rm/$HASH" > pinrm_out && - echo "{\"Pins\":[\"QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6\"]}" > pinrm_exp && + curl -X POST "http://$API_ADDR/api/v0/pin/rm/pintest" > pinrm_out && + echo "{\"Pins\":[\"pintest\"]}" > pinrm_exp && test_cmp pinrm_out pinrm_exp ' diff --git a/test/sharness/x0601-pin-fail-test.sh b/test/sharness/x0601-pin-fail-test.sh index ffab1062d07..301bc225f8d 100755 --- a/test/sharness/x0601-pin-fail-test.sh +++ b/test/sharness/x0601-pin-fail-test.sh @@ -14,7 +14,7 @@ test_launch_ipfs_daemon test_expect_success "pre-test setup" ' printf "" > pins && - ipfs pin ls --type=recursive -q > rec_pins_before + ipfs pin ls -r --type=recursive -q > rec_pins_before ' @@ -26,7 +26,7 @@ do done test_expect_success "get pinset afterwards" ' - ipfs pin ls --type=recursive -q | sort > rec_pins_after && + ipfs pin ls -r --type=recursive -q | sort > rec_pins_after && cat pins rec_pins_before | sort | uniq > exp_pins_after && test_cmp rec_pins_after exp_pins_after '