feat: support byte ranges

cmd/lassie/fetch.go

@@ -46,6 +46,19 @@ var fetchFlags = []cli.Flag{
 			return nil
 		},
 	},
+	&cli.StringFlag{
+		Name: "entity-bytes",
+		Usage: "describes the byte range to consider when selecting the blocks from a sharded file." +
+			" Valid values should be of the form from:to, where from and to are byte offsets and to may be '*'",
+		DefaultText: "defaults to the entire file, 0:*",
+		Action: func(cctx *cli.Context, v string) error {
+			if _, err := types.ParseByteRange(v); err != nil {
+				return fmt.Errorf("invalid entity-bytes parameter, must be of the form from:to," +
+					" where from and to are byte offsets and to may be '*'")
+			}
+			return nil
+		},
+	},
 	FlagIPNIEndpoint,
 	FlagEventRecorderAuth,
 	FlagEventRecorderInstanceId,
@@ -78,6 +91,7 @@ func fetchAction(cctx *cli.Context) error {
 	dataWriter := cctx.App.Writer
 
 	dagScope := cctx.String("dag-scope")
+	entityBytes := cctx.String("entity-bytes")
 	tempDir := cctx.String("tempdir")
 	progress := cctx.Bool("progress")
 
@@ -111,6 +125,7 @@ func fetchAction(cctx *cli.Context) error {
 		rootCid,
 		path,
 		dagScope,
+		entityBytes,
 		tempDir,
 		progress,
 		outfile,
@@ -188,6 +203,7 @@ type fetchRunFunc func(
 	rootCid cid.Cid,
 	path string,
 	dagScope string,
+	entityBytes string,
 	tempDir string,
 	progress bool,
 	outfile string,
@@ -207,6 +223,7 @@ func defaultFetchRun(
 	rootCid cid.Cid,
 	path string,
 	dagScope string,
+	entityBytes string,
 	tempDir string,
 	progress bool,
 	outfile string,
@@ -258,7 +275,12 @@ func defaultFetchRun(
 		}
 	}, false)
 
-	request, err := types.NewRequestForPath(carStore, rootCid, path, types.DagScope(dagScope))
+	byteRange, _ := types.ParseByteRange(entityBytes)
+	var br *types.ByteRange
+	if !byteRange.IsDefault() {
+		br = &byteRange
+	}
+	request, err := types.NewRequestForPath(carStore, rootCid, path, types.DagScope(dagScope), br)
 	if err != nil {
 		return err
 	}

cmd/lassie/fetch_test.go

@@ -28,11 +28,12 @@ func TestFetchCommandFlags(t *testing.T) {
 		{
 			name: "with default args",
 			args: []string{"fetch", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				// fetch specific params
 				require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", rootCid.String())
 				require.Equal(t, "", path)
 				require.Equal(t, string(types.DagScopeAll), dagScope)
+				require.Empty(t, entityBytes)
 				require.Equal(t, false, progress)
 				require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4.car", outfile)
 
@@ -70,7 +71,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"fetch",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4/birb.mp4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, "/birb.mp4", path)
 				return nil
 			},
@@ -83,7 +84,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"entity",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, string(types.DagScopeEntity), dagScope)
 				return nil
 			},
@@ -96,19 +97,58 @@ func TestFetchCommandFlags(t *testing.T) {
 				"block",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, string(types.DagScopeBlock), dagScope)
 				return nil
 			},
 		},
+		{
+			name: "with entity-bytes 0:*",
+			args: []string{
+				"fetch",
+				"--entity-bytes",
+				"0:*",
+				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
+			},
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
+				require.Equal(t, "0:*", entityBytes)
+				return nil
+			},
+		},
+		{
+			name: "with entity-bytes 0:10",
+			args: []string{
+				"fetch",
+				"--entity-bytes",
+				"0:10",
+				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
+			},
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
+				require.Equal(t, "0:10", entityBytes)
+				return nil
+			},
+		},
+		{
+			name: "with entity-bytes 1000:20000",
+			args: []string{
+				"fetch",
+				"--entity-bytes",
+				"1000:20000",
+				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
+			},
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
+				require.Equal(t, "1000:20000", entityBytes)
+				return nil
+			},
+		},
 		{
 			name: "with progress",
 			args: []string{
 				"fetch",
 				"--progress",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, true, progress)
 				return nil
 			},
@@ -121,7 +161,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"myfile",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, "myfile", outfile)
 				return nil
 			},
@@ -134,7 +174,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.IsType(t, &retriever.DirectCandidateFinder{}, lCfg.Finder, "finder should be a DirectCandidateFinder when providers are specified")
 				return nil
 			},
@@ -147,7 +187,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"https://cid.contact",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.IsType(t, &indexerlookup.IndexerCandidateFinder{}, lCfg.Finder, "finder should be an IndexerCandidateFinder when providing an ipni endpoint")
 				return nil
 			},
@@ -170,7 +210,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"/mytmpdir",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, "/mytmpdir", tempDir)
 				return nil
 			},
@@ -183,7 +223,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"30s",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, 30*time.Second, lCfg.ProviderTimeout)
 				return nil
 			},
@@ -196,7 +236,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"30s",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, 30*time.Second, lCfg.GlobalTimeout)
 				return nil
 			},
@@ -209,7 +249,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"bitswap,graphsync",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, []multicodec.Code{multicodec.TransportBitswap, multicodec.TransportGraphsyncFilecoinv1}, lCfg.Protocols)
 				return nil
 			},
@@ -222,7 +262,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4,12D3KooWPNbkEgjdBNeaCGpsgCrPRETe4uBZf1ShFXStobdN18ys",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				p1, err := peer.Decode("12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4")
 				require.NoError(t, err)
 				p2, err := peer.Decode("12D3KooWPNbkEgjdBNeaCGpsgCrPRETe4uBZf1ShFXStobdN18ys")
@@ -241,7 +281,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"10",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, 10, lCfg.BitswapConcurrency)
 				return nil
 			},
@@ -254,7 +294,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"https://myeventrecorder.com/v1/retrieval-events",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, "https://myeventrecorder.com/v1/retrieval-events", erCfg.EndpointURL)
 				return nil
 			},
@@ -267,7 +307,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"secret",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, "secret", erCfg.EndpointAuthorization)
 				return nil
 			},
@@ -280,7 +320,7 @@ func TestFetchCommandFlags(t *testing.T) {
 				"myinstanceid",
 				"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
 			},
-			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+			assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 				require.Equal(t, "myinstanceid", erCfg.InstanceID)
 				return nil
 			},
@@ -312,6 +352,6 @@ func TestFetchCommandFlags(t *testing.T) {
 	}
 }
 
-func noopRun(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
+func noopRun(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
 	return nil
 }

go.mod

@@ -22,10 +22,11 @@ require (
 	github.com/ipfs/go-ipfs-exchange-interface v0.2.0
 	github.com/ipfs/go-ipld-format v0.5.0
 	github.com/ipfs/go-log/v2 v2.5.1
-	github.com/ipfs/go-unixfsnode v1.7.1
+	github.com/ipfs/go-unixfsnode v1.7.2
 	github.com/ipld/go-car/v2 v2.10.1
 	github.com/ipld/go-codec-dagpb v1.6.0
-	github.com/ipld/go-ipld-prime v0.20.1-0.20230329011551-5056175565b0
+	github.com/ipld/go-ipld-prime v0.20.1-0.20230707090759-349deb22a1fd
+	github.com/ipld/ipld/specs v0.0.0-20230705075038-29da2e853cdb
 	github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff
 	github.com/libp2p/go-libp2p v0.27.1
 	github.com/libp2p/go-libp2p-routing-helpers v0.7.0
@@ -144,6 +145,7 @@ require (
 	github.com/raulk/go-watchdog v1.3.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/warpfork/go-testmark v0.12.1 // indirect
 	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
 	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect