Named pins, storing pins in Datastore [WIP]

* Change pin structure to a tree stored in KV store
* Adjust pin-related commands accordingly
* Make pinning routines use meaningful default prefixes

License: MIT
Signed-off-by: Iaroslav Gridin <voker57@gmail.com>

# Conflicts:
#	assets/assets.go
#	core/builder.go
#	core/commands/add.go
#	core/commands/dag/dag.go
#	core/commands/object/object.go
#	core/commands/pin.go
#	core/commands/urlstore.go
#	core/coreapi/interface/options/object.go
#	core/coreapi/interface/options/pin.go
#	core/coreapi/interface/options/unixfs.go
#	core/coreapi/interface/pin.go
#	core/coreapi/interface/tests/pin.go
#	core/coreapi/interface/tests/unixfs.go
#	core/coreapi/pin.go
#	core/corerepo/pinning.go
#	core/coreunix/add.go
#	exchange/reprovide/providers.go
#	pin/pin.go
#	pin/pin_test.go
#	pin/set.go
#	pin/set_test.go
#	test/sharness/t0080-repo.sh
#	test/sharness/t0085-pins.sh
#	test/sharness/t0252-files-gc.sh
#	test/sharness/t0272-urlstore.sh

Author: iaroslav-gridin

assets/assets.go

@@ -78,9 +78,10 @@ func addAssetList(nd *core.IpfsNode, l []string) (cid.Cid, error) {
 		}
 	}
 
-	if err := api.Pin().Add(nd.Context(), basePath); err != nil {
+	if err := api.Pin().Add(nd.Context(), "assets", basePath); err != nil {
 		return cid.Cid{}, err
 	}
 
 	return basePath.Cid(), nil
+
 }

core/commands/add.go

@@ -38,6 +38,7 @@ const (
 	onlyHashOptionName    = "only-hash"
 	chunkerOptionName     = "chunker"
 	pinOptionName         = "pin"
+	pinPathOptionName     = "pinpath"
 	rawLeavesOptionName   = "raw-leaves"
 	noCopyOptionName      = "nocopy"
 	fstoreCacheOptionName = "fscache"
@@ -123,6 +124,7 @@ You can now check what blocks have been created by:
 		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes] or rabin-[min]-[avg]-[max]").WithDefault("size-262144"),
 		cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true),
 		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
+		cmds.StringOption(pinPathOptionName, "P", "Pin object under this path.").WithDefault("added/"),
 		cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
 		cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
 		cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. (experimental)"),
@@ -162,6 +164,7 @@ You can now check what blocks have been created by:
 		silent, _ := req.Options[silentOptionName].(bool)
 		chunker, _ := req.Options[chunkerOptionName].(string)
 		dopin, _ := req.Options[pinOptionName].(bool)
+		pinPath, _ := req.Options[pinPathOptionName].(string)
 		rawblks, rbset := req.Options[rawLeavesOptionName].(bool)
 		nocopy, _ := req.Options[noCopyOptionName].(bool)
 		fscache, _ := req.Options[fstoreCacheOptionName].(bool)
@@ -196,6 +199,7 @@ You can now check what blocks have been created by:
 			options.Unixfs.Chunker(chunker),
 
 			options.Unixfs.Pin(dopin),
+			options.Unixfs.PinPath(pinPath),
 			options.Unixfs.HashOnly(hash),
 			options.Unixfs.FsCache(fscache),
 			options.Unixfs.Nocopy(nocopy),

core/commands/dag/dag.go

@@ -61,7 +61,7 @@ into an object of the specified format.
 	Options: []cmds.Option{
 		cmds.StringOption("format", "f", "Format that the object will be added as.").WithDefault("cbor"),
 		cmds.StringOption("input-enc", "Format that the input object will be.").WithDefault("json"),
-		cmds.BoolOption("pin", "Pin this object when adding."),
+		cmds.StringOption("pin", "Pin this object when adding.").WithDefault(""),
 		cmds.StringOption("hash", "Hash function to use").WithDefault(""),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
@@ -73,7 +73,7 @@ into an object of the specified format.
 		ienc, _ := req.Options["input-enc"].(string)
 		format, _ := req.Options["format"].(string)
 		hash, _ := req.Options["hash"].(string)
-		dopin, _ := req.Options["pin"].(bool)
+		pinpath, _ := req.Options["pin"].(string)
 
 		// mhType tells inputParser which hash should be used. MaxUint64 means 'use
 		// default hash' (sha256 for cbor, sha1 for git..)
@@ -88,9 +88,6 @@ into an object of the specified format.
 		}
 
 		var adder ipld.NodeAdder = api.Dag()
-		if dopin {
-			adder = api.Dag().Pinning()
-		}
 		b := ipld.NewBatch(req.Context, adder)
 
 		it := req.Files.Entries()
@@ -107,10 +104,13 @@ into an object of the specified format.
 				return fmt.Errorf("no node returned from ParseInputs")
 			}
 
-			for _, nd := range nds {
-				err := b.Add(req.Context, nd)
-				if err != nil {
-					return err
+			if pinpath != "" {
+
+				for _, nd := range nds {
+					err := api.Pin().Add(req.Context, pinpath, path.IpfsPath(nd.Cid()))
+					if err != nil {
+						return err
+					}
 				}
 			}
 

core/commands/object/object.go

@@ -404,6 +404,7 @@ And then run:
 		cmds.StringOption(datafieldencOptionName, "Encoding type of the data field, either \"text\" or \"base64\".").WithDefault("text"),
 		cmds.BoolOption(pinOptionName, "Pin this object when adding."),
 		cmds.BoolOption(quietOptionName, "q", "Write minimal output."),
+		cmds.StringOption("pinpath", "Pin under this path").WithDefault("added/"),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		api, err := cmdenv.GetApi(env, req)
@@ -431,15 +432,21 @@ And then run:
 			return err
 		}
 
-		dopin, _ := req.Options[pinOptionName].(bool)
+		pinpath, _ := req.Options["pinpath"].(string)
+		if err != nil {
+			return err
+		}
+
+		pin, _ := req.Options["pin"].(bool)
 		if err != nil {
 			return err
 		}
 
 		p, err := api.Object().Put(req.Context, file,
 			options.Object.DataType(datafieldenc),
 			options.Object.InputEnc(inputenc),
-			options.Object.Pin(dopin))
+			options.Object.Pin(pin),
+			options.Object.PinPath(pinpath))
 		if err != nil {
 			return err
 		}