From ee65443b2b9b3cccca4c890ccbbc286250639978 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 9 Jun 2023 16:51:35 +1000 Subject: [PATCH 01/15] feat: support verifying byte range CARs --- pkg/verifiedcar/verifiedcar.go | 14 +++++++++++--- pkg/verifiedcar/verifiedcar_test.go | 30 +++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/pkg/verifiedcar/verifiedcar.go b/pkg/verifiedcar/verifiedcar.go index 03a4c4e6..966b173a 100644 --- a/pkg/verifiedcar/verifiedcar.go +++ b/pkg/verifiedcar/verifiedcar.go @@ -55,8 +55,6 @@ type Config struct { MaxBlocks uint64 // set a budget for the traversal } -func visitNoop(p traversal.Progress, n datamodel.Node, r traversal.VisitReason) error { return nil } - // Verify reads a CAR from the provided reader, verifies the contents are // strictly what is specified by this Config and writes the blocks to the // provided BlockWriteOpener. It returns the number of blocks and bytes @@ -130,7 +128,17 @@ func (cfg Config) VerifyBlockStream(ctx context.Context, cbr BlockReader, lsys l if err != nil { return 0, 0, err } - if err := progress.WalkAdv(rootNode, sel, visitNoop); err != nil { + if err := progress.WalkMatching(rootNode, sel, func(p traversal.Progress, n datamodel.Node) error { + if lbn, ok := n.(datamodel.LargeBytesNode); ok { + rdr, err := lbn.AsLargeBytes() + if err != nil { + return err + } + _, err = io.Copy(io.Discard, rdr) + return err + } + return nil + }); err != nil { return 0, 0, traversalError(err) } diff --git a/pkg/verifiedcar/verifiedcar_test.go b/pkg/verifiedcar/verifiedcar_test.go index 40698131..6f9ee5c1 100644 --- a/pkg/verifiedcar/verifiedcar_test.go +++ b/pkg/verifiedcar/verifiedcar_test.go @@ -25,6 +25,7 @@ import ( "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/storage/memstore" "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/stretchr/testify/require" ) @@ -63,6 +64,17 @@ func TestVerifiedCar(t *testing.T) { unixfsFile := testutil.GenerateNoDupes(func() unixfs.DirEntry { return unixfs.GenerateFile(t, &lsys, rndReader, 4<<20) }) unixfsFileBlocks := testutil.ToBlocks(t, lsys, unixfsFile.Root, allSelector) + unixfsFileRange0_1048576Blocks := unixfsFileBlocks[0:6] + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + ss := ssb.ExploreInterpretAs("unixfs", ssb.MatcherSubset(0, 1<<20)) + unixfsFileRange0_1048576Selector := ss.Node() + + // need the root plus the byte range of 1M->2M, which happens to include the + // block of the 0->1M range because of overlapping data + unixfsFileRange1048576_2097152Blocks := append(append([]blocks.Block{}, unixfsFileBlocks[0]), unixfsFileBlocks[5:10]...) + ss = ssb.ExploreInterpretAs("unixfs", ssb.MatcherSubset(1<<20, 2<<20)) + unixfsFileRange1048576_2097152Selector := ss.Node() + unixfsFileWithDups := unixfs.GenerateFile(t, &lsys, testutil.ZeroReader{}, 4<<20) unixfsFileWithDupsBlocks := testutil.ToBlocks(t, lsys, unixfsFileWithDups.Root, allSelector) var unixfsDir unixfs.DirEntry @@ -514,6 +526,24 @@ func TestVerifiedCar(t *testing.T) { Selector: allSelector, }, }, + { + name: "unixfs: large sharded file byte range [0:1M]", + blocks: consumedBlocks(unixfsFileRange0_1048576Blocks), + roots: []cid.Cid{unixfsFile.Root}, + cfg: verifiedcar.Config{ + Root: unixfsFile.Root, + Selector: unixfsFileRange0_1048576Selector, + }, + }, + { + name: "unixfs: large sharded file byte range [1M:2M]", + blocks: consumedBlocks(unixfsFileRange1048576_2097152Blocks), + roots: []cid.Cid{unixfsFile.Root}, + cfg: verifiedcar.Config{ + Root: unixfsFile.Root, + Selector: unixfsFileRange1048576_2097152Selector, + }, + }, } for _, testCase := range testCases { From d80035d484e5eb9fdff397e805580f98470a88b4 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 16 Jun 2023 14:19:00 +1000 Subject: [PATCH 02/15] feat: remove unixfs-preload use --- go.mod | 5 +++ go.sum | 12 +++----- pkg/internal/itest/testpeer/generator.go | 4 +-- pkg/internal/testutil/toblocks.go | 3 +- pkg/retriever/bitswapretriever.go | 10 +++--- pkg/verifiedcar/verifiedcar.go | 39 ++++++++++++------------ pkg/verifiedcar/verifiedcar_test.go | 12 +++++++- 7 files changed, 47 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index e647c2d4..171a3611 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/ipfs/go-ipfs-delay v0.0.1 github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipld-format v0.5.0 + github.com/ipfs/go-libipfs v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-unixfsnode v1.7.1 github.com/ipld/go-car/v2 v2.10.1 @@ -166,3 +167,7 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect nhooyr.io/websocket v1.8.7 // indirect ) + +replace github.com/ipfs/go-unixfsnode => ../../ipfs/go-unixfsnode + +replace github.com/ipld/go-ipld-prime => ../../ipld/go-ipld-prime diff --git a/go.sum b/go.sum index 88cf9a7c..c2e72b7c 100644 --- a/go.sum +++ b/go.sum @@ -126,7 +126,7 @@ github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -292,10 +292,11 @@ github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNo github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= -github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6ZpusRg= +github.com/ipfs/go-ipfs-files v0.3.0 h1:fallckyc5PYjuMEitPNrjRfpwl7YFt69heCOUhsbGxQ= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= +github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9OWevc= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= @@ -311,6 +312,7 @@ github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAF github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= github.com/ipfs/go-libipfs v0.6.0 h1:3FuckAJEm+zdHbHbf6lAyk0QUzc45LsFcGw102oBCZM= +github.com/ipfs/go-libipfs v0.6.0/go.mod h1:UjjDIuehp2GzlNP0HEr5I9GfFT7zWgst+YfpUEIThtw= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= @@ -329,15 +331,11 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= -github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s= -github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipld/go-car/v2 v2.10.1 h1:MRDqkONNW9WRhB79u+Z3U5b+NoN7lYA5B8n8qI3+BoI= github.com/ipld/go-car/v2 v2.10.1/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= -github.com/ipld/go-ipld-prime v0.20.1-0.20230329011551-5056175565b0 h1:iJTl9tx5DEsnKpppX5PmfdoQ3ITuBmkh3yyEpHWY2SI= -github.com/ipld/go-ipld-prime v0.20.1-0.20230329011551-5056175565b0/go.mod h1:wmOtdy70ajP48iZITH8uLsGJVMqA4EJM61/bSfYYGhs= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff h1:xbKrIvnpQkbF8iHPk/HGcegsypCDpcXWHhzBCLyCWf8= github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff/go.mod h1:paYP9U4N3/vOzGCuN9kU972vtvw9JUcQjOKyiCFGwRk= @@ -616,7 +614,7 @@ github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU= github.com/urfave/cli/v2 v2.24.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= +github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= diff --git a/pkg/internal/itest/testpeer/generator.go b/pkg/internal/itest/testpeer/generator.go index 42d52d52..ae1d9648 100644 --- a/pkg/internal/itest/testpeer/generator.go +++ b/pkg/internal/itest/testpeer/generator.go @@ -454,7 +454,7 @@ func MockIpfsHandler(ctx context.Context, lsys linking.LinkSystem) func(http.Res } progress := traversal.Progress{Cfg: cfg} - err = progress.WalkAdv(rootNode, sel, visitNoop) + err = progress.WalkMatching(rootNode, sel, unixfsnode.BytesConsumingMatcher) if err != nil { // if we loaded the first block, we can't write headers any more return @@ -485,5 +485,3 @@ func RandTestPeerIdentity() (tnet.Identity, error) { } return nil, errors.New("failed to find an available port") } - -func visitNoop(p traversal.Progress, n datamodel.Node, vr traversal.VisitReason) error { return nil } diff --git a/pkg/internal/testutil/toblocks.go b/pkg/internal/testutil/toblocks.go index bbc43927..81061f8e 100644 --- a/pkg/internal/testutil/toblocks.go +++ b/pkg/internal/testutil/toblocks.go @@ -53,8 +53,7 @@ func ToBlocks(t *testing.T, lsys linking.LinkSystem, root cid.Cid, selNode datam LinkTargetNodePrototypeChooser: dagpb.AddSupportToChooser(basicnode.Chooser), }, } - vf := func(p traversal.Progress, n datamodel.Node, vr traversal.VisitReason) error { return nil } - err = prog.WalkAdv(rootNode, sel, vf) + err = prog.WalkMatching(rootNode, sel, unixfsnode.BytesConsumingMatcher) require.NoError(t, err) return traversedBlocks diff --git a/pkg/retriever/bitswapretriever.go b/pkg/retriever/bitswapretriever.go index f7224dba..ce89441e 100644 --- a/pkg/retriever/bitswapretriever.go +++ b/pkg/retriever/bitswapretriever.go @@ -18,6 +18,9 @@ import ( "github.com/ipfs/boxo/bitswap/network" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/bitswap/client" + "github.com/ipfs/go-libipfs/bitswap/network" + "github.com/ipfs/go-unixfsnode" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" @@ -295,10 +298,6 @@ func loaderForSession(retrievalID types.RetrievalID, inProgressCids InProgressCi } } -func noopVisitor(prog traversal.Progress, n datamodel.Node, reason traversal.VisitReason) error { - return nil -} - func easyTraverse( ctx context.Context, root datamodel.Link, @@ -340,7 +339,8 @@ func easyTraverse( if err != nil { return err } - if err := progress.WalkAdv(node, compiledSelector, noopVisitor); err != nil { + + if err := progress.WalkMatching(node, compiledSelector, unixfsnode.BytesConsumingMatcher); err != nil { return err } return ecr.Error diff --git a/pkg/verifiedcar/verifiedcar.go b/pkg/verifiedcar/verifiedcar.go index 966b173a..d3bf7ef6 100644 --- a/pkg/verifiedcar/verifiedcar.go +++ b/pkg/verifiedcar/verifiedcar.go @@ -118,34 +118,19 @@ func (cfg Config) VerifyBlockStream(ctx context.Context, cbr BlockReader, lsys l NodeBudget: math.MaxInt64, } } - lc := linking.LinkContext{Ctx: ctx} - lnk := cidlink.Link{Cid: cfg.Root} - proto, err := protoChooser(lnk, lc) + + rootNode, err := loadNode(ctx, cfg.Root, lsys) if err != nil { - return 0, 0, err + return 0, 0, fmt.Errorf("failed to load root node: %w", err) } - rootNode, err := lsys.Load(lc, lnk, proto) - if err != nil { - return 0, 0, err - } - if err := progress.WalkMatching(rootNode, sel, func(p traversal.Progress, n datamodel.Node) error { - if lbn, ok := n.(datamodel.LargeBytesNode); ok { - rdr, err := lbn.AsLargeBytes() - if err != nil { - return err - } - _, err = io.Copy(io.Discard, rdr) - return err - } - return nil - }); err != nil { + if err := progress.WalkMatching(rootNode, sel, unixfsnode.BytesConsumingMatcher); err != nil { return 0, 0, traversalError(err) } if nbls.Error != nil { // capture any errors not bubbled up through the traversal, i.e. see // https://github.com/ipld/go-ipld-prime/pull/524 - return 0, 0, nbls.Error + return 0, 0, fmt.Errorf("block load failed during traversal: %w", nbls.Error) } // make sure we don't have any extraneous data beyond what the traversal needs @@ -160,6 +145,20 @@ func (cfg Config) VerifyBlockStream(ctx context.Context, cbr BlockReader, lsys l return bt.blocks, bt.bytes, nil } +func loadNode(ctx context.Context, rootCid cid.Cid, lsys linking.LinkSystem) (datamodel.Node, error) { + lnk := cidlink.Link{Cid: rootCid} + lnkCtx := linking.LinkContext{Ctx: ctx} + proto, err := protoChooser(lnk, lnkCtx) + if err != nil { + return nil, fmt.Errorf("failed to choose prototype for CID %s: %w", rootCid.String(), err) + } + rootNode, err := lsys.Load(lnkCtx, lnk, proto) + if err != nil { + return nil, fmt.Errorf("failed to load root CID: %w", err) + } + return rootNode, nil +} + type NextBlockLinkSystem struct { Error error } diff --git a/pkg/verifiedcar/verifiedcar_test.go b/pkg/verifiedcar/verifiedcar_test.go index 6f9ee5c1..7ae45015 100644 --- a/pkg/verifiedcar/verifiedcar_test.go +++ b/pkg/verifiedcar/verifiedcar_test.go @@ -25,6 +25,7 @@ import ( "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/storage/memstore" "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/stretchr/testify/require" @@ -101,6 +102,11 @@ func TestVerifiedCar(t *testing.T) { unixfsWrappedPathSelector := unixfsnode.UnixFSPathSelectorBuilder(wrapPath, unixfsnode.ExploreAllRecursivelySelector, false) unixfsWrappedPreloadPathSelector := unixfsnode.UnixFSPathSelectorBuilder(wrapPath, unixfsnode.MatchUnixFSPreloadSelector, false) + preloadSubst := ssb.ExploreInterpretAs("unixfs", ssb.ExploreRecursive( + selector.RecursionLimitDepth(1), + ssb.ExploreAll(ssb.ExploreRecursiveEdge()), + )) + unixfsWrappedPreloadPathSelectorSubst := unixfsnode.UnixFSPathSelectorBuilder(wrapPath, preloadSubst, false) unixfsWrappedFile := testutil.GenerateNoDupes(func() unixfs.DirEntry { return unixfs.WrapContent(t, rndReader, &lsys, unixfsFile, wrapPath, false) }) unixfsWrappedFileBlocks := testutil.ToBlocks(t, lsys, unixfsWrappedFile.Root, allSelector) @@ -127,6 +133,7 @@ func TestVerifiedCar(t *testing.T) { mismatchedCidBlk, _ := blocks.NewBlockWithCid(extraneousByts, allBlocks[99].Cid()) testCases := []struct { name string + skip bool blocks []expectedBlock roots []cid.Cid carv2 bool @@ -448,7 +455,7 @@ func TestVerifiedCar(t *testing.T) { roots: []cid.Cid{unixfsExclusiveWrappedShardedDir.Root}, cfg: verifiedcar.Config{ Root: unixfsExclusiveWrappedShardedDir.Root, - Selector: unixfsWrappedPreloadPathSelector, + Selector: unixfsWrappedPreloadPathSelectorSubst, }, }, { @@ -549,6 +556,9 @@ func TestVerifiedCar(t *testing.T) { for _, testCase := range testCases { testCase := testCase t.Run(testCase.name, func(t *testing.T) { + if testCase.skip { + t.Skip() + } t.Parallel() ctx, cancel := context.WithTimeout(ctx, 2*time.Second) From 1448930ea3225a66852ae41a33ef62977d81c101 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Tue, 27 Jun 2023 17:50:19 +1000 Subject: [PATCH 03/15] chore: test fixtures for various unixfs queries --- go.mod | 3 +- go.sum | 5 +- pkg/internal/fixtures/testcases.go | 94 +++++ pkg/internal/fixtures/unixfs_20m_variety.go | 75 ++++ pkg/internal/testdata/unixfs_20m_variety.md | 364 ++++++++++++++++++++ pkg/retriever/bitswapretriever.go | 2 - pkg/server/http/util.go | 22 +- pkg/types/types.go | 59 ++++ pkg/verifiedcar/verifiedcar_test.go | 72 ++++ 9 files changed, 671 insertions(+), 25 deletions(-) create mode 100644 pkg/internal/fixtures/testcases.go create mode 100644 pkg/internal/fixtures/unixfs_20m_variety.go create mode 100644 pkg/internal/testdata/unixfs_20m_variety.md diff --git a/go.mod b/go.mod index 171a3611..5c6e6cbd 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/ipfs/go-ipfs-delay v0.0.1 github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipld-format v0.5.0 - github.com/ipfs/go-libipfs v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-unixfsnode v1.7.1 github.com/ipld/go-car/v2 v2.10.1 @@ -37,6 +36,7 @@ require ( github.com/multiformats/go-multihash v0.2.3 github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.24.4 + github.com/warpfork/go-testmark v0.12.1 go.opentelemetry.io/otel v1.14.0 go.opentelemetry.io/otel/trace v1.14.0 go.uber.org/multierr v1.11.0 @@ -84,6 +84,7 @@ require ( github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect + github.com/ipfs/go-ipfs-files v0.3.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect diff --git a/go.sum b/go.sum index c2e72b7c..65562672 100644 --- a/go.sum +++ b/go.sum @@ -293,10 +293,10 @@ github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-files v0.3.0 h1:fallckyc5PYjuMEitPNrjRfpwl7YFt69heCOUhsbGxQ= +github.com/ipfs/go-ipfs-files v0.3.0/go.mod h1:xAUtYMwB+iu/dtf6+muHNSFQCJG2dSiStR2P6sn9tIM= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= -github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9OWevc= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= @@ -312,7 +312,6 @@ github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAF github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= github.com/ipfs/go-libipfs v0.6.0 h1:3FuckAJEm+zdHbHbf6lAyk0QUzc45LsFcGw102oBCZM= -github.com/ipfs/go-libipfs v0.6.0/go.mod h1:UjjDIuehp2GzlNP0HEr5I9GfFT7zWgst+YfpUEIThtw= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= @@ -614,7 +613,9 @@ github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU= github.com/urfave/cli/v2 v2.24.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/warpfork/go-fsx v0.3.0/go.mod h1:oTACCMj+Zle+vgVa5SAhGAh7WksYpLgGUCKEAVc+xPg= github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= +github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= diff --git a/pkg/internal/fixtures/testcases.go b/pkg/internal/fixtures/testcases.go new file mode 100644 index 00000000..896eb897 --- /dev/null +++ b/pkg/internal/fixtures/testcases.go @@ -0,0 +1,94 @@ +package fixtures + +import ( + "errors" + "fmt" + "net/url" + "strings" + + "github.com/filecoin-project/lassie/pkg/types" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" +) + +type TestCase struct { + Name string + Root cid.Cid + Path string + Scope types.DagScope + Duplicates bool + ByteRange *types.ByteRange + ExpectedCids []cid.Cid +} + +func (tc TestCase) AsQuery() string { + pp := ipld.ParsePath(tc.Path).String() + if pp != "" { + pp = "/" + pp + } + br := "" + if tc.ByteRange != nil && !tc.ByteRange.IsDefault() { + br = fmt.Sprintf("&entity-bytes=%s", tc.ByteRange.String()) + } + dup := "" + if tc.Duplicates { + dup = "&dups=y" + } + return fmt.Sprintf("/ipfs/%s%s?dag-scope=%s%s%s", tc.Root, pp, tc.Scope, br, dup) +} + +func ParseCase(name, spec, exec string) (TestCase, error) { + lines := strings.Split(exec, "\n") + for len(lines) > 0 && strings.TrimSpace(lines[0]) == "" { + lines = lines[1:] + } + for len(lines) > 0 && strings.TrimSpace(lines[len(lines)-1]) == "" { + lines = lines[:len(lines)-1] + } + specParts := strings.Split(strings.TrimSpace(spec), "?") + if len(specParts) != 2 { + return TestCase{}, errors.New("invalid spec") + } + spec = specParts[0] + query, err := url.ParseQuery(specParts[1]) + if err != nil { + return TestCase{}, err + } + specParts = strings.Split(spec, "/") + if specParts[0] != "" && specParts[1] != "ipfs" { + return TestCase{}, errors.New("invalid spec") + } + root, err := cid.Parse(specParts[2]) + if err != nil { + return TestCase{}, err + } + path := "/" + ipld.ParsePath(strings.Join(specParts[3:], "/")).String() + scope, err := types.ParseDagScope(query.Get("dag-scope")) // required + if err != nil { + return TestCase{}, err + } + duplicates := query.Get("dups") == "y" + var byteRange *types.ByteRange + if query.Get("byte-range") != "" { + if br, err := types.ParseByteRange(query.Get("byte-range")); err != nil { + return TestCase{}, err + } else { + byteRange = &br + } + } + expectedCids := make([]cid.Cid, 0, len(lines)) + for _, line := range lines { + la := strings.Split(line, "|") + c := cid.MustParse(strings.TrimSpace(la[0])) + expectedCids = append(expectedCids, c) + } + return TestCase{ + Name: name, + Root: root, + Path: path, + Scope: scope, + Duplicates: duplicates, + ByteRange: byteRange, + ExpectedCids: expectedCids, + }, nil +} diff --git a/pkg/internal/fixtures/unixfs_20m_variety.go b/pkg/internal/fixtures/unixfs_20m_variety.go new file mode 100644 index 00000000..8937842a --- /dev/null +++ b/pkg/internal/fixtures/unixfs_20m_variety.go @@ -0,0 +1,75 @@ +package fixtures + +import ( + "fmt" + "io" + "os" + "strings" + + "github.com/ipfs/go-cid" + carstorage "github.com/ipld/go-car/v2/storage" + "github.com/ipld/go-ipld-prime/storage" + "github.com/warpfork/go-testmark" +) + +var Unixfs20mVarietyRoot = cid.MustParse("bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq") + +const file = "internal/testdata/unixfs_20m_variety." + +func filepath(typ string) (string, error) { + wd, err := os.Getwd() + if err != nil { + return "", err + } + rootInd := strings.LastIndex(wd, "/lassie/pkg/") + if rootInd == -1 { + return "", fmt.Errorf("could not find root of lassie package") + } + filename := wd[:rootInd] + "/lassie/pkg/" + file + typ + fmt.Println("Using", filename) + return filename, nil +} + +func Unixfs20mVarietyReadableStorage() (storage.ReadableStorage, io.Closer, error) { + file, err := filepath("car") + if err != nil { + return nil, nil, err + } + carFile, err := os.Open(file) + if err != nil { + return nil, nil, err + } + reader, err := carstorage.OpenReadable(carFile) + if err != nil { + carFile.Close() + return nil, nil, err + } + return reader, carFile, nil +} + +func Unixfs20mVarietyCases() ([]TestCase, error) { + file, err := filepath("md") + if err != nil { + return nil, err + } + doc, err := testmark.ReadFile(file) + if err != nil { + return nil, err + } + doc.BuildDirIndex() + testCases := make([]TestCase, 0) + for _, test := range doc.DirEnt.Children["test"].ChildrenList { + for _, scope := range test.ChildrenList { + tc, err := ParseCase(test.Name+"/"+scope.Name, dstr(scope, "query"), dstr(scope, "execution")) + if err != nil { + return nil, err + } + testCases = append(testCases, tc) + } + } + return testCases, nil +} + +func dstr(dir *testmark.DirEnt, ch string) string { + return string(dir.Children[ch].Hunk.Body) +} diff --git a/pkg/internal/testdata/unixfs_20m_variety.md b/pkg/internal/testdata/unixfs_20m_variety.md new file mode 100644 index 00000000..7ed330f1 --- /dev/null +++ b/pkg/internal/testdata/unixfs_20m_variety.md @@ -0,0 +1,364 @@ +# unixfs_20m_variety (test fixture) + + * 20 MB of files with a variety of UnixFS features across 1,103 blocks + * unixfs_20m_variety.car is a CARv1 in strict dfs order with a single root. + +## Root CID +[testmark]:# (root) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq +``` + +## Test Cases + +### Small file in directory + +Same result regardless of scope. + +#### all + +[testmark]:# (test/small_file_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/pi?dag-scope=all +``` + +[testmark]:# (test/small_file_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1701] +``` + +#### entity + +[testmark]:# (test/small_file_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/pi?dag-scope=entity +``` + +[testmark]:# (test/small_file_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1701] +``` + +#### block + +[testmark]:# (test/small_file_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/pi?dag-scope=block +``` + +[testmark]:# (test/small_file_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1701] +``` + +### Sharded file in directory + +All and entity are the same but block should just get the root File block + +#### all + +[testmark]:# (test/sharded_file_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/želva.xml?dag-scope=all +``` + +[testmark]:# (test/sharded_file_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352050] +bafkreigtwsisgpg6x752y5md2z2r4jhuhohh46y4x3mvxrbkcubo7mqlgi | RawLeaf | ↳ /želva.xml[0:256144] +bafkreigahqispwg55yvqwobavwlheongcyhk63eufsaqutqgjiwfwesfau | RawLeaf | /želva.xml[256144:512288] +bafkreic4kgh44v2ung3wspd7y6wigcxake45ztpfx3c5ibfwmqe2kt7uay | RawLeaf | /želva.xml[512288:768432] +bafkreibdjsvoyftwgcywb3xylnfod2wififs2tj7pww4zkvaba7z5aigtm | RawLeaf | /želva.xml[768432:1024576] +bafkreidtv6frnbb4o4yobyu3xbtd5onzne67dhgngpd3vwrbdcneapy6fa | RawLeaf | /želva.xml[1024576:1280720] +bafkreidy2ntimx4u22b5uy6tjh7du5ex5lhgu7comxwcjfkzfbdsdunbou | RawLeaf | /želva.xml[1280720:1352050] +``` + +#### entity + +[testmark]:# (test/sharded_file_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/želva.xml?dag-scope=entity +``` + +[testmark]:# (test/sharded_file_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352050] +bafkreigtwsisgpg6x752y5md2z2r4jhuhohh46y4x3mvxrbkcubo7mqlgi | RawLeaf | ↳ /želva.xml[0:256144] +bafkreigahqispwg55yvqwobavwlheongcyhk63eufsaqutqgjiwfwesfau | RawLeaf | /želva.xml[256144:512288] +bafkreic4kgh44v2ung3wspd7y6wigcxake45ztpfx3c5ibfwmqe2kt7uay | RawLeaf | /želva.xml[512288:768432] +bafkreibdjsvoyftwgcywb3xylnfod2wififs2tj7pww4zkvaba7z5aigtm | RawLeaf | /želva.xml[768432:1024576] +bafkreidtv6frnbb4o4yobyu3xbtd5onzne67dhgngpd3vwrbdcneapy6fa | RawLeaf | /želva.xml[1024576:1280720] +bafkreidy2ntimx4u22b5uy6tjh7du5ex5lhgu7comxwcjfkzfbdsdunbou | RawLeaf | /želva.xml[1280720:1352050] +``` + +#### block + +[testmark]:# (test/sharded_file_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/želva.xml?dag-scope=block +``` + +[testmark]:# (test/sharded_file_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352050] +``` + +### Sharded file in directory in directory + +Aame as above but one extra level of nesting. + +#### all + +[testmark]:# (test/sharded_file_in_directory_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Flibbertigibbet5/eorþscyld.pdf?dag-scope=all +``` + +[testmark]:# (test/sharded_file_in_directory_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeicqlqnzd2tvs5vlo4c2dz72ryvouwmjwpu2mvxrocuj2u26joxeam | Directory | ↳ /Flibbertigibbet5 +bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558478] +bafkreidz265issniggrhdjbwp3c5o4fnmg7xsa6hbn3kmyo7bsafc3jjsy | RawLeaf | ↳ /eorþscyld.pdf[0:256144] +bafkreiepissrnjsgcanpe6hzen5kv66glcr5x5vw5vhferkjscxdj2uv3y | RawLeaf | /eorþscyld.pdf[256144:512288] +bafkreifohr4rlzjpskbdcntrjzolglq6e5uf7n4mgdiw7wl2yhovhx46ve | RawLeaf | /eorþscyld.pdf[512288:768432] +bafkreihiulihaonenbbtrbptmcmzcg2hvk6dypdxvmposz7ansd6h5mkla | RawLeaf | /eorþscyld.pdf[768432:1024576] +bafkreidbhmgdt3ajzgoeywwxdbdcquone7buyenwhirl6af2z3gmftiys4 | RawLeaf | /eorþscyld.pdf[1024576:1280720] +bafkreib6iawulrjvzet7dxbqdqneqa7kpy2kacvq7fawjvmlgveoakuzzy | RawLeaf | /eorþscyld.pdf[1280720:1536864] +bafkreicbbl7whtfqsoz6tu663qicdisdyvzp4pp4ylggcttc44j63bdhxy | RawLeaf | /eorþscyld.pdf[1536864:1793008] +bafkreifxe2irzoxtzk3ltpunioehodovu4pwzdtjr2lqtmkg7hyh3o5r2e | RawLeaf | /eorþscyld.pdf[1793008:2049152] +bafkreihr2m4zfx4qvpwkp2fnihn7gbjq6gyr2gv5nbibt2nuajnp2k6pvq | RawLeaf | /eorþscyld.pdf[2049152:2305296] +bafkreibhyumtv62kh3d4rvwpxkdlr52uur5thgmbtyuyxas3c424od7cta | RawLeaf | /eorþscyld.pdf[2305296:2561440] +bafkreiatyvemaol2uxceo4suktpmusl53dqgg5bcqi2rkfhlxnnvpigkci | RawLeaf | /eorþscyld.pdf[2561440:2817584] +bafkreias6ygzx2pnowi3hxj5kmed2rm74hgigwijiandz7vamvrohtkpje | RawLeaf | /eorþscyld.pdf[2817584:3073728] +bafkreihf5g2wmsblx67664w2k3m7hj2bg2wnpumlnty7ssw4gj46oj3bxa | RawLeaf | /eorþscyld.pdf[3073728:3329872] +bafkreib7gr6yi6lhl2p5izuxrxpknf5tc5lhkf5tdhzaxw5g66jxqichra | RawLeaf | /eorþscyld.pdf[3329872:3586016] +bafkreigmnnfehab7c4tblwqgydbaoc76y34rkjfetayb6ytql2eywqnq5y | RawLeaf | /eorþscyld.pdf[3586016:3842160] +bafkreiax5uhfktinmfo3ovm7geaf2266u5ya2slq4bob6eeginffbewszu | RawLeaf | /eorþscyld.pdf[3842160:4098304] +bafkreic5chuwrtofd6ymxawcaxulacfewzfi3zroakmkwjkfmadrxrmln4 | RawLeaf | /eorþscyld.pdf[4098304:4354448] +bafkreich7pcb4lnheypmqb4sikk5m6cuqqbxbjzuk767cx47djxac45o64 | RawLeaf | /eorþscyld.pdf[4354448:4558478] +``` + +#### entity + +[testmark]:# (test/sharded_file_in_directory_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Flibbertigibbet5/eorþscyld.pdf?dag-scope=entity +``` + +[testmark]:# (test/sharded_file_in_directory_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeicqlqnzd2tvs5vlo4c2dz72ryvouwmjwpu2mvxrocuj2u26joxeam | Directory | ↳ /Flibbertigibbet5 +bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558478] +bafkreidz265issniggrhdjbwp3c5o4fnmg7xsa6hbn3kmyo7bsafc3jjsy | RawLeaf | ↳ /eorþscyld.pdf[0:256144] +bafkreiepissrnjsgcanpe6hzen5kv66glcr5x5vw5vhferkjscxdj2uv3y | RawLeaf | /eorþscyld.pdf[256144:512288] +bafkreifohr4rlzjpskbdcntrjzolglq6e5uf7n4mgdiw7wl2yhovhx46ve | RawLeaf | /eorþscyld.pdf[512288:768432] +bafkreihiulihaonenbbtrbptmcmzcg2hvk6dypdxvmposz7ansd6h5mkla | RawLeaf | /eorþscyld.pdf[768432:1024576] +bafkreidbhmgdt3ajzgoeywwxdbdcquone7buyenwhirl6af2z3gmftiys4 | RawLeaf | /eorþscyld.pdf[1024576:1280720] +bafkreib6iawulrjvzet7dxbqdqneqa7kpy2kacvq7fawjvmlgveoakuzzy | RawLeaf | /eorþscyld.pdf[1280720:1536864] +bafkreicbbl7whtfqsoz6tu663qicdisdyvzp4pp4ylggcttc44j63bdhxy | RawLeaf | /eorþscyld.pdf[1536864:1793008] +bafkreifxe2irzoxtzk3ltpunioehodovu4pwzdtjr2lqtmkg7hyh3o5r2e | RawLeaf | /eorþscyld.pdf[1793008:2049152] +bafkreihr2m4zfx4qvpwkp2fnihn7gbjq6gyr2gv5nbibt2nuajnp2k6pvq | RawLeaf | /eorþscyld.pdf[2049152:2305296] +bafkreibhyumtv62kh3d4rvwpxkdlr52uur5thgmbtyuyxas3c424od7cta | RawLeaf | /eorþscyld.pdf[2305296:2561440] +bafkreiatyvemaol2uxceo4suktpmusl53dqgg5bcqi2rkfhlxnnvpigkci | RawLeaf | /eorþscyld.pdf[2561440:2817584] +bafkreias6ygzx2pnowi3hxj5kmed2rm74hgigwijiandz7vamvrohtkpje | RawLeaf | /eorþscyld.pdf[2817584:3073728] +bafkreihf5g2wmsblx67664w2k3m7hj2bg2wnpumlnty7ssw4gj46oj3bxa | RawLeaf | /eorþscyld.pdf[3073728:3329872] +bafkreib7gr6yi6lhl2p5izuxrxpknf5tc5lhkf5tdhzaxw5g66jxqichra | RawLeaf | /eorþscyld.pdf[3329872:3586016] +bafkreigmnnfehab7c4tblwqgydbaoc76y34rkjfetayb6ytql2eywqnq5y | RawLeaf | /eorþscyld.pdf[3586016:3842160] +bafkreiax5uhfktinmfo3ovm7geaf2266u5ya2slq4bob6eeginffbewszu | RawLeaf | /eorþscyld.pdf[3842160:4098304] +bafkreic5chuwrtofd6ymxawcaxulacfewzfi3zroakmkwjkfmadrxrmln4 | RawLeaf | /eorþscyld.pdf[4098304:4354448] +bafkreich7pcb4lnheypmqb4sikk5m6cuqqbxbjzuk767cx47djxac45o64 | RawLeaf | /eorþscyld.pdf[4354448:4558478] +``` + +#### block + +[testmark]:# (test/sharded_file_in_directory_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Flibbertigibbet5/eorþscyld.pdf?dag-scope=block +``` + +[testmark]:# (test/sharded_file_in_directory_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeicqlqnzd2tvs5vlo4c2dz72ryvouwmjwpu2mvxrocuj2u26joxeam | Directory | ↳ /Flibbertigibbet5 +bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558478] +``` + +### Sharded file in hamt in directory + +Same as above but the inner directory is a HAMT and we have an intermediate block. + +#### all + +[testmark]:# (test/sharded_file_in_hamt_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/O/Othello.png?dag-scope=all +``` + +[testmark]:# (test/sharded_file_in_hamt_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiahh6giyfzxpsp6b4y6j6r34xjgsb2si7n6dszwoevnjxdcuee5yq | HAMTShard | ↳ /O +bafybeics7ohtet4wfuzszfchbttrgotqzpuruq4bencpdtc443vawbcbni | HAMTShard | ↳ +bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977731] +bafkreibyblakon7atsv2hwzyzrweeb5uriksuscfxjtlzj3l56xmrouwke | RawLeaf | ↳ /Othello.png[0:256144] +bafkreic7nx6e7zezkdz5lp2xcwarmgavr62cyrmtepyagvm2ivuo3jesqe | RawLeaf | /Othello.png[256144:512288] +bafkreicylwcsi7ovreggex5yc7jolt5zrwf74ida3sprzsw37oxpzkwvte | RawLeaf | /Othello.png[512288:768432] +bafkreiainpsn6gadltlhqzgakxvdbxrz5n2cstizn3afhsr2nwefus7pqm | RawLeaf | /Othello.png[768432:1024576] +bafkreihuwjz4av7scz3fdjejtrx3k6ycdjudjusuz6sjzbbsewpgslosxi | RawLeaf | /Othello.png[1024576:1280720] +bafkreiexmzld3vi72v665bbqn473mfc5smmtd6nnndl7dzcapzlxdwzhq4 | RawLeaf | /Othello.png[1280720:1536864] +bafkreignp6prdhvi4mxoef5ybl4c644eyeikd42w3z2atw5mgbzw5p4374 | RawLeaf | /Othello.png[1536864:1793008] +bafkreia6nxplcal74ycukjwfmwmaaraeg6qwclfvb6qja2iaayukzsedoa | RawLeaf | /Othello.png[1793008:2049152] +bafkreiae6iay4fqiethqcvva4qazdxb7szb4dh77dsxiskvewhm5sepwfm | RawLeaf | /Othello.png[2049152:2305296] +bafkreieprkc6i3qzg47k5y3zxejsoglyl5fwrk3fnslbmdmau7surwiocy | RawLeaf | /Othello.png[2305296:2561440] +bafkreidwsenghahefqcfw5ikxd6osi6a35koyki4gibw3sunlmqrroqzc4 | RawLeaf | /Othello.png[2561440:2817584] +bafkreicn24jr4rwjcptuzu677u2aeqyirf7weradcafcnkq5hzo7wickkq | RawLeaf | /Othello.png[2817584:2977731] +``` + +#### entity + +[testmark]:# (test/sharded_file_in_hamt_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/O/Othello.png?dag-scope=entity +``` + +[testmark]:# (test/sharded_file_in_hamt_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiahh6giyfzxpsp6b4y6j6r34xjgsb2si7n6dszwoevnjxdcuee5yq | HAMTShard | ↳ /O +bafybeics7ohtet4wfuzszfchbttrgotqzpuruq4bencpdtc443vawbcbni | HAMTShard | ↳ +bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977731] +bafkreibyblakon7atsv2hwzyzrweeb5uriksuscfxjtlzj3l56xmrouwke | RawLeaf | ↳ /Othello.png[0:256144] +bafkreic7nx6e7zezkdz5lp2xcwarmgavr62cyrmtepyagvm2ivuo3jesqe | RawLeaf | /Othello.png[256144:512288] +bafkreicylwcsi7ovreggex5yc7jolt5zrwf74ida3sprzsw37oxpzkwvte | RawLeaf | /Othello.png[512288:768432] +bafkreiainpsn6gadltlhqzgakxvdbxrz5n2cstizn3afhsr2nwefus7pqm | RawLeaf | /Othello.png[768432:1024576] +bafkreihuwjz4av7scz3fdjejtrx3k6ycdjudjusuz6sjzbbsewpgslosxi | RawLeaf | /Othello.png[1024576:1280720] +bafkreiexmzld3vi72v665bbqn473mfc5smmtd6nnndl7dzcapzlxdwzhq4 | RawLeaf | /Othello.png[1280720:1536864] +bafkreignp6prdhvi4mxoef5ybl4c644eyeikd42w3z2atw5mgbzw5p4374 | RawLeaf | /Othello.png[1536864:1793008] +bafkreia6nxplcal74ycukjwfmwmaaraeg6qwclfvb6qja2iaayukzsedoa | RawLeaf | /Othello.png[1793008:2049152] +bafkreiae6iay4fqiethqcvva4qazdxb7szb4dh77dsxiskvewhm5sepwfm | RawLeaf | /Othello.png[2049152:2305296] +bafkreieprkc6i3qzg47k5y3zxejsoglyl5fwrk3fnslbmdmau7surwiocy | RawLeaf | /Othello.png[2305296:2561440] +bafkreidwsenghahefqcfw5ikxd6osi6a35koyki4gibw3sunlmqrroqzc4 | RawLeaf | /Othello.png[2561440:2817584] +bafkreicn24jr4rwjcptuzu677u2aeqyirf7weradcafcnkq5hzo7wickkq | RawLeaf | /Othello.png[2817584:2977731] +``` + +#### block + +[testmark]:# (test/sharded_file_in_hamt_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/O/Othello.png?dag-scope=block +``` + +[testmark]:# (test/sharded_file_in_hamt_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiahh6giyfzxpsp6b4y6j6r34xjgsb2si7n6dszwoevnjxdcuee5yq | HAMTShard | ↳ /O +bafybeics7ohtet4wfuzszfchbttrgotqzpuruq4bencpdtc443vawbcbni | HAMTShard | ↳ +bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977731] +``` + +### Sharded file in directory in hamt in directory + +Same as above but with an extra directory layer under the HAMT. + +#### all + +[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/Throttlebottom/supercalifragilisticexpialidocious.txt?dag-scope=all +``` + +[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | ↳ +bafybeifjkcls323ddq3t5ov22yl7i6ks36a7jzv33pjy4stqkils6jzvqe | Directory | ↳ /Throttlebottom +bafybeify427noacqiu6sxaaunk5uw2xhelnkcwktkvh6woi6ahlipsz7em | File | ↳ /supercalifragilisticexpialidocious.txt[0:568521] +bafkreicdwgfhxwnzq7i34cuhnqqqxz6d6jtirupjy5kq3uaphopyw5e2ky | RawLeaf | ↳ /supercalifragilisticexpialidocious.txt[0:256144] +bafkreifmk6qfl6ucap7btfu35rjd37bh7uuefavq6nsuungxjl3or7bz2e | RawLeaf | /supercalifragilisticexpialidocious.txt[256144:512288] +bafkreifk2vcldftxe57ml2cxyfcwb34ukkhaopm46kv3as5vo26ht63fci | RawLeaf | /supercalifragilisticexpialidocious.txt[512288:568521] +``` + +#### entity + +[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/Throttlebottom?dag-scope=entity +``` + +[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | ↳ +bafybeifjkcls323ddq3t5ov22yl7i6ks36a7jzv33pjy4stqkils6jzvqe | Directory | ↳ /Throttlebottom +``` + +#### block + +[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/Throttlebottom?dag-scope=block +``` + +[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | ↳ +bafybeifjkcls323ddq3t5ov22yl7i6ks36a7jzv33pjy4stqkils6jzvqe | Directory | ↳ /Throttlebottom +``` + +### Hamt in directory + +"All" is too large to be useful, so we'll just do "entity" and "block". + +#### entity + +[testmark]:# (test/hamt_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious?dag-scope=entity +``` + +[testmark]:# (test/hamt_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +bafybeiggzqpxmfbve7sutrmcwrlwpx4tkno536d7qdgd6hnhxf74t7gzee | HAMTShard | ↳ /Zigzagumptious +bafybeigfwz54atwgnklb72rfqqn2a3ywoontgf3b3kekxwoxalptx7gc3e | HAMTShard | ↳ /Zigzagumptious +bafybeifr3w3eyq2oi6hxwbfoblor7s5pfvu3m4xyyzovxlk4g5xgslsqui | HAMTShard | /Zigzagumptious +bafybeigivdbrxmcixaqfap3rrs7ti5ycr7vj4xfbkjougq67k7ac5oz55a | HAMTShard | /Zigzagumptious +bafybeid3cny2p7fwt4znzm3wjncsesdk2rdze5fm5rzxjpbethy4657j6m | HAMTShard | /Zigzagumptious +bafybeidivwpxv37o6p4ton3xcpmg64pwk4zqlkuge7isafgsgjsvb6ce2u | HAMTShard | /Zigzagumptious +bafybeib5ptplpmtmoyif4lxt5erw5diivp6so4s57lvgfigs53d7zpc4uu | HAMTShard | /Zigzagumptious +bafybeiafmairlfkvgqgticndcs4uomkll4h5xotvnnsw2j7cx3rjwrbh7i | HAMTShard | ↳ /Zigzagumptious +bafybeifjo6xz3onvav6cnpjjuroa7vbzegoxgg5jn6smfvgfcpzwzdaipa | HAMTShard | /Zigzagumptious +bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | /Zigzagumptious +bafybeiar4b5hfriv2pmmbj7npemto6bev7sqkn4jpstwqtaudexpetpg4m | HAMTShard | /Zigzagumptious +bafybeifcekpohkm7qaczfwrrzcdz6j3ykwntawdnp7b7fjfech75vxpwf4 | HAMTShard | ↳ /Zigzagumptious +bafybeihnejxifhvlunhcswhghhrbues366h7o4xqp3ziyeicrdt5473za4 | HAMTShard | /Zigzagumptious +bafybeihlijmc2xyfmomw4fx53mlz3gpt6svgfu3vfokm4wmyd4dagfup24 | HAMTShard | /Zigzagumptious +bafybeihjbh7kptdfaoanrkzhdpx2jwjacg5ncdlcbbwqlum6vwqfga5fii | HAMTShard | /Zigzagumptious +bafybeidaz5aob6tdurpnuj2xeqsjkarqudtono6x2kgbzrzv46ugp3g4ju | HAMTShard | /Zigzagumptious +``` + +#### block + +[testmark]:# (test/hamt_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious?dag-scope=block +``` + +[testmark]:# (test/hamt_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +``` diff --git a/pkg/retriever/bitswapretriever.go b/pkg/retriever/bitswapretriever.go index ce89441e..bdd2fcf0 100644 --- a/pkg/retriever/bitswapretriever.go +++ b/pkg/retriever/bitswapretriever.go @@ -18,8 +18,6 @@ import ( "github.com/ipfs/boxo/bitswap/network" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/bitswap/client" - "github.com/ipfs/go-libipfs/bitswap/network" "github.com/ipfs/go-unixfsnode" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime/datamodel" diff --git a/pkg/server/http/util.go b/pkg/server/http/util.go index 13be0574..f5c7e9b5 100644 --- a/pkg/server/http/util.go +++ b/pkg/server/http/util.go @@ -14,29 +14,11 @@ import ( // parameter is not one of the supported values. func ParseScope(req *http.Request) (types.DagScope, error) { if req.URL.Query().Has("dag-scope") { - switch req.URL.Query().Get("dag-scope") { - case "all": - return types.DagScopeAll, nil - case "entity": - return types.DagScopeEntity, nil - case "block": - return types.DagScopeBlock, nil - default: - return types.DagScopeAll, errors.New("invalid dag-scope parameter") - } + return types.ParseDagScope(req.URL.Query().Get("dag-scope")) } // check for legacy param name -- to do -- delete once we confirm this isn't used any more if req.URL.Query().Has("car-scope") { - switch req.URL.Query().Get("car-scope") { - case "all": - return types.DagScopeAll, nil - case "file": - return types.DagScopeEntity, nil - case "block": - return types.DagScopeBlock, nil - default: - return types.DagScopeAll, errors.New("invalid car-scope parameter") - } + return types.ParseDagScope(req.URL.Query().Get("car-scope")) } return types.DagScopeAll, nil } diff --git a/pkg/types/types.go b/pkg/types/types.go index 383ce1bf..fd83f8bc 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -4,7 +4,10 @@ import ( "context" "errors" "fmt" + "math" "net/url" + "strconv" + "strings" "time" "github.com/filecoin-project/go-state-types/abi" @@ -280,6 +283,62 @@ func (ds DagScope) TerminalSelectorSpec() builder.SelectorSpec { panic(fmt.Sprintf("unknown DagScope: [%s]", string(ds))) } +func ParseDagScope(s string) (DagScope, error) { + switch s { + case "all": + return DagScopeAll, nil + case "entity": + return DagScopeEntity, nil + case "block": + return DagScopeBlock, nil + default: + return DagScopeAll, errors.New("invalid dag-scope") + } +} + func (ds DagScope) AcceptHeader() string { return "application/vnd.ipld.car;version=1;order=dfs;dups=y" } + +type ByteRange struct { + From int64 + To int64 +} + +func (br ByteRange) String() string { + to := strconv.FormatInt(br.To, 10) + if br.To == math.MaxInt64 { + to = "*" + } + return fmt.Sprintf("%d:%s", br.From, to) +} + +func (br ByteRange) IsDefault() bool { + return br.From == 0 && br.To == math.MaxInt64 +} + +func ParseByteRange(s string) (ByteRange, error) { + br := ByteRange{From: 0, To: math.MaxInt64} + if s == "" { + return br, nil + } + parts := strings.Split(s, ":") + if len(parts) != 2 { + return br, fmt.Errorf("invalid byte range: %s", s) + } + var err error + br.From, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return br, err + } + if br.From < 0 { + return br, fmt.Errorf("invalid byte range: %s", s) + } + if parts[1] != "*" { + br.To, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return br, err + } + } + return br, nil +} diff --git a/pkg/verifiedcar/verifiedcar_test.go b/pkg/verifiedcar/verifiedcar_test.go index 7ae45015..890089de 100644 --- a/pkg/verifiedcar/verifiedcar_test.go +++ b/pkg/verifiedcar/verifiedcar_test.go @@ -10,7 +10,9 @@ import ( "testing" "time" + "github.com/filecoin-project/lassie/pkg/internal/fixtures" "github.com/filecoin-project/lassie/pkg/internal/testutil" + "github.com/filecoin-project/lassie/pkg/types" "github.com/filecoin-project/lassie/pkg/verifiedcar" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -31,6 +33,76 @@ import ( "github.com/stretchr/testify/require" ) +func TestUnixfs20mVariety(t *testing.T) { + req := require.New(t) + + testCases, err := fixtures.Unixfs20mVarietyCases() + req.NoError(err) + storage, closer, err := fixtures.Unixfs20mVarietyReadableStorage() + req.NoError(err) + defer closer.Close() + + lsys := cidlink.DefaultLinkSystem() + lsys.TrustedStorage = true + unixfsnode.AddUnixFSReificationToLinkSystem(&lsys) + lsys.SetReadStorage(storage) + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + req := require.New(t) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + t.Logf("query=%s, blocks=%d", tc.AsQuery(), len(tc.ExpectedCids)) + + // tc.ExpectedCids is in the order we expect to see them in a properly + // formed trustless CAR for the given query. So we build our list of + // expected blocks in that order and feed it through makeCarStream to + // produce the expected CAR. + expectedBlocks := make([]expectedBlock, len(tc.ExpectedCids)) + for ii, ec := range tc.ExpectedCids { + byt, err := lsys.LoadRaw(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: ec}) + req.NoError(err) + blk, err := blocks.NewBlockWithCid(byt, ec) + req.NoError(err) + expectedBlocks[ii] = expectedBlock{blk, false} + } + + carStream := makeCarStream(t, ctx, []cid.Cid{tc.Root}, expectedBlocks, false, false, false) + + lsys := cidlink.DefaultLinkSystem() + var writeCounter int + lsys.StorageWriteOpener = func(lc linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) { + var buf bytes.Buffer + return &buf, func(l datamodel.Link) error { + req.Equal(expectedBlocks[writeCounter].Cid().String(), l.(cidlink.Link).Cid.String(), "block %d", writeCounter) + req.Equal(expectedBlocks[writeCounter].RawData(), buf.Bytes(), "block %d", writeCounter) + writeCounter++ + return nil + }, nil + } + + // Run the verifier over the CAR stream to see if we end up with + // the same query. + cfg := verifiedcar.Config{ + Root: tc.Root, + Selector: types.PathScopeSelector(tc.Path, tc.Scope), + } + blockCount, byteCount, err := cfg.VerifyCar(ctx, carStream, lsys) + + req.NoError(err) + req.Equal(count(expectedBlocks), blockCount) + req.Equal(sizeOf(expectedBlocks), byteCount) + req.Equal(int(count(expectedBlocks)), writeCounter) + + // Make sure we consumed the entire stream. + byt, err := io.ReadAll(carStream) + req.NoError(err) + req.Equal(0, len(byt)) + }) + } +} + func TestVerifiedCar(t *testing.T) { ctx := context.Background() From 46adc794e3d69f1842f7b0bdd451f499123b5fea Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 28 Jun 2023 16:48:38 +1000 Subject: [PATCH 04/15] feat: make byte ranges work, lots of tests --- go.mod | 8 +- go.sum | 4 + pkg/internal/fixtures/testcases.go | 4 +- pkg/internal/testdata/unixfs_20m_variety.md | 696 +++++++++++++++++--- pkg/storage/duplicateaddercar.go | 2 +- pkg/types/request.go | 29 +- pkg/types/types.go | 6 +- pkg/verifiedcar/verifiedcar_test.go | 60 +- 8 files changed, 695 insertions(+), 114 deletions(-) diff --git a/go.mod b/go.mod index 5c6e6cbd..2e9a86c2 100644 --- a/go.mod +++ b/go.mod @@ -22,10 +22,10 @@ require ( github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipfs/go-unixfsnode v1.7.1 + github.com/ipfs/go-unixfsnode v1.7.2-0.20230622024455-de5cbf74cbe0 github.com/ipld/go-car/v2 v2.10.1 github.com/ipld/go-codec-dagpb v1.6.0 - github.com/ipld/go-ipld-prime v0.20.1-0.20230329011551-5056175565b0 + github.com/ipld/go-ipld-prime v0.20.1-0.20230628064445-0955c324b41a github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff github.com/libp2p/go-libp2p v0.27.1 github.com/libp2p/go-libp2p-routing-helpers v0.7.0 @@ -168,7 +168,3 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect nhooyr.io/websocket v1.8.7 // indirect ) - -replace github.com/ipfs/go-unixfsnode => ../../ipfs/go-unixfsnode - -replace github.com/ipld/go-ipld-prime => ../../ipld/go-ipld-prime diff --git a/go.sum b/go.sum index 65562672..9caa707d 100644 --- a/go.sum +++ b/go.sum @@ -330,11 +330,15 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= +github.com/ipfs/go-unixfsnode v1.7.2-0.20230622024455-de5cbf74cbe0 h1:XTKKTDz0rbJ53ff/7borHTvN64zJK74wZDvrYfdN0Fg= +github.com/ipfs/go-unixfsnode v1.7.2-0.20230622024455-de5cbf74cbe0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipld/go-car/v2 v2.10.1 h1:MRDqkONNW9WRhB79u+Z3U5b+NoN7lYA5B8n8qI3+BoI= github.com/ipld/go-car/v2 v2.10.1/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= +github.com/ipld/go-ipld-prime v0.20.1-0.20230628064445-0955c324b41a h1:o5zc8GsB8u1UMp4Etmb8VBMJun1sEC5SLYye9iLTXr0= +github.com/ipld/go-ipld-prime v0.20.1-0.20230628064445-0955c324b41a/go.mod h1:PRQpXNcJypaPiiSdarsrJABPkYrBvafwDl0B9HjujZ8= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff h1:xbKrIvnpQkbF8iHPk/HGcegsypCDpcXWHhzBCLyCWf8= github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff/go.mod h1:paYP9U4N3/vOzGCuN9kU972vtvw9JUcQjOKyiCFGwRk= diff --git a/pkg/internal/fixtures/testcases.go b/pkg/internal/fixtures/testcases.go index 896eb897..689fedd0 100644 --- a/pkg/internal/fixtures/testcases.go +++ b/pkg/internal/fixtures/testcases.go @@ -69,8 +69,8 @@ func ParseCase(name, spec, exec string) (TestCase, error) { } duplicates := query.Get("dups") == "y" var byteRange *types.ByteRange - if query.Get("byte-range") != "" { - if br, err := types.ParseByteRange(query.Get("byte-range")); err != nil { + if query.Get("entity-bytes") != "" { + if br, err := types.ParseByteRange(query.Get("entity-bytes")); err != nil { return TestCase{}, err } else { byteRange = &br diff --git a/pkg/internal/testdata/unixfs_20m_variety.md b/pkg/internal/testdata/unixfs_20m_variety.md index 7ed330f1..aad131ed 100644 --- a/pkg/internal/testdata/unixfs_20m_variety.md +++ b/pkg/internal/testdata/unixfs_20m_variety.md @@ -25,7 +25,7 @@ Same result regardless of scope. [testmark]:# (test/small_file_in_directory/all/execution) ``` bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1701] +bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1700] (1701 B) ``` #### entity @@ -38,7 +38,7 @@ bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /p [testmark]:# (test/small_file_in_directory/entity/execution) ``` bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1701] +bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1700] (1701 B) ``` #### block @@ -51,7 +51,56 @@ bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /p [testmark]:# (test/small_file_in_directory/block/execution) ``` bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1701] +bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1700] (1701 B) +``` + +### Small file in directory in directory in directory + +Same result regardless of scope. + +#### all + +[testmark]:# (test/small_file_in_directory_in_directory_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/by/ğħōšţ/Whatchamacallit.json?dag-scope=all +``` + +[testmark]:# (test/small_file_in_directory_in_directory_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeigdgcs3nulbgu6vtnyesfgsaau2grbxzbxxdtpgolsfhtrpjx6mw4 | Directory | ↳ /by +bafybeidbjnqmuizharp3piroes43frsqip37qafovv4cjok667hbdy7vhq | Directory | ↳ /ğħōšţ +bafkreicgd36kzadnbwjhat2eyvcbvv63a36l5oufslvryixcymnd342oei | RawLeaf | ↳ /Whatchamacallit.json[0:1204] (1205 B) +``` + +#### entity + +[testmark]:# (test/small_file_in_directory_in_directory_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/by/ğħōšţ/Whatchamacallit.json?dag-scope=entity +``` + +[testmark]:# (test/small_file_in_directory_in_directory_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeigdgcs3nulbgu6vtnyesfgsaau2grbxzbxxdtpgolsfhtrpjx6mw4 | Directory | ↳ /by +bafybeidbjnqmuizharp3piroes43frsqip37qafovv4cjok667hbdy7vhq | Directory | ↳ /ğħōšţ +bafkreicgd36kzadnbwjhat2eyvcbvv63a36l5oufslvryixcymnd342oei | RawLeaf | ↳ /Whatchamacallit.json[0:1204] (1205 B) +``` + +#### block + +[testmark]:# (test/small_file_in_directory_in_directory_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/by/ğħōšţ/Whatchamacallit.json?dag-scope=entity +``` + +[testmark]:# (test/small_file_in_directory_in_directory_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeigdgcs3nulbgu6vtnyesfgsaau2grbxzbxxdtpgolsfhtrpjx6mw4 | Directory | ↳ /by +bafybeidbjnqmuizharp3piroes43frsqip37qafovv4cjok667hbdy7vhq | Directory | ↳ /ğħōšţ +bafkreicgd36kzadnbwjhat2eyvcbvv63a36l5oufslvryixcymnd342oei | RawLeaf | ↳ /Whatchamacallit.json[0:1204] (1205 B) ``` ### Sharded file in directory @@ -68,13 +117,13 @@ All and entity are the same but block should just get the root File block [testmark]:# (test/sharded_file_in_directory/all/execution) ``` bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352050] -bafkreigtwsisgpg6x752y5md2z2r4jhuhohh46y4x3mvxrbkcubo7mqlgi | RawLeaf | ↳ /želva.xml[0:256144] -bafkreigahqispwg55yvqwobavwlheongcyhk63eufsaqutqgjiwfwesfau | RawLeaf | /želva.xml[256144:512288] -bafkreic4kgh44v2ung3wspd7y6wigcxake45ztpfx3c5ibfwmqe2kt7uay | RawLeaf | /želva.xml[512288:768432] -bafkreibdjsvoyftwgcywb3xylnfod2wififs2tj7pww4zkvaba7z5aigtm | RawLeaf | /želva.xml[768432:1024576] -bafkreidtv6frnbb4o4yobyu3xbtd5onzne67dhgngpd3vwrbdcneapy6fa | RawLeaf | /želva.xml[1024576:1280720] -bafkreidy2ntimx4u22b5uy6tjh7du5ex5lhgu7comxwcjfkzfbdsdunbou | RawLeaf | /želva.xml[1280720:1352050] +bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352049] (1352050 B) +bafkreigtwsisgpg6x752y5md2z2r4jhuhohh46y4x3mvxrbkcubo7mqlgi | RawLeaf | ↳ /želva.xml[0:256143] (256144 B) +bafkreigahqispwg55yvqwobavwlheongcyhk63eufsaqutqgjiwfwesfau | RawLeaf | /želva.xml[256144:512287] (256144 B) +bafkreic4kgh44v2ung3wspd7y6wigcxake45ztpfx3c5ibfwmqe2kt7uay | RawLeaf | /želva.xml[512288:768431] (256144 B) +bafkreibdjsvoyftwgcywb3xylnfod2wififs2tj7pww4zkvaba7z5aigtm | RawLeaf | /želva.xml[768432:1024575] (256144 B) +bafkreidtv6frnbb4o4yobyu3xbtd5onzne67dhgngpd3vwrbdcneapy6fa | RawLeaf | /želva.xml[1024576:1280719] (256144 B) +bafkreidy2ntimx4u22b5uy6tjh7du5ex5lhgu7comxwcjfkzfbdsdunbou | RawLeaf | /želva.xml[1280720:1352049] (71330 B) ``` #### entity @@ -87,13 +136,13 @@ bafkreidy2ntimx4u22b5uy6tjh7du5ex5lhgu7comxwcjfkzfbdsdunbou | RawLeaf | / [testmark]:# (test/sharded_file_in_directory/entity/execution) ``` bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352050] -bafkreigtwsisgpg6x752y5md2z2r4jhuhohh46y4x3mvxrbkcubo7mqlgi | RawLeaf | ↳ /želva.xml[0:256144] -bafkreigahqispwg55yvqwobavwlheongcyhk63eufsaqutqgjiwfwesfau | RawLeaf | /želva.xml[256144:512288] -bafkreic4kgh44v2ung3wspd7y6wigcxake45ztpfx3c5ibfwmqe2kt7uay | RawLeaf | /želva.xml[512288:768432] -bafkreibdjsvoyftwgcywb3xylnfod2wififs2tj7pww4zkvaba7z5aigtm | RawLeaf | /želva.xml[768432:1024576] -bafkreidtv6frnbb4o4yobyu3xbtd5onzne67dhgngpd3vwrbdcneapy6fa | RawLeaf | /želva.xml[1024576:1280720] -bafkreidy2ntimx4u22b5uy6tjh7du5ex5lhgu7comxwcjfkzfbdsdunbou | RawLeaf | /želva.xml[1280720:1352050] +bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352049] (1352050 B) +bafkreigtwsisgpg6x752y5md2z2r4jhuhohh46y4x3mvxrbkcubo7mqlgi | RawLeaf | ↳ /želva.xml[0:256143] (256144 B) +bafkreigahqispwg55yvqwobavwlheongcyhk63eufsaqutqgjiwfwesfau | RawLeaf | /želva.xml[256144:512287] (256144 B) +bafkreic4kgh44v2ung3wspd7y6wigcxake45ztpfx3c5ibfwmqe2kt7uay | RawLeaf | /želva.xml[512288:768431] (256144 B) +bafkreibdjsvoyftwgcywb3xylnfod2wififs2tj7pww4zkvaba7z5aigtm | RawLeaf | /želva.xml[768432:1024575] (256144 B) +bafkreidtv6frnbb4o4yobyu3xbtd5onzne67dhgngpd3vwrbdcneapy6fa | RawLeaf | /želva.xml[1024576:1280719] (256144 B) +bafkreidy2ntimx4u22b5uy6tjh7du5ex5lhgu7comxwcjfkzfbdsdunbou | RawLeaf | /želva.xml[1280720:1352049] (71330 B) ``` #### block @@ -106,7 +155,7 @@ bafkreidy2ntimx4u22b5uy6tjh7du5ex5lhgu7comxwcjfkzfbdsdunbou | RawLeaf | / [testmark]:# (test/sharded_file_in_directory/block/execution) ``` bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352050] +bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352049] (1352050 B) ``` ### Sharded file in directory in directory @@ -124,25 +173,25 @@ Aame as above but one extra level of nesting. ``` bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / bafybeicqlqnzd2tvs5vlo4c2dz72ryvouwmjwpu2mvxrocuj2u26joxeam | Directory | ↳ /Flibbertigibbet5 -bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558478] -bafkreidz265issniggrhdjbwp3c5o4fnmg7xsa6hbn3kmyo7bsafc3jjsy | RawLeaf | ↳ /eorþscyld.pdf[0:256144] -bafkreiepissrnjsgcanpe6hzen5kv66glcr5x5vw5vhferkjscxdj2uv3y | RawLeaf | /eorþscyld.pdf[256144:512288] -bafkreifohr4rlzjpskbdcntrjzolglq6e5uf7n4mgdiw7wl2yhovhx46ve | RawLeaf | /eorþscyld.pdf[512288:768432] -bafkreihiulihaonenbbtrbptmcmzcg2hvk6dypdxvmposz7ansd6h5mkla | RawLeaf | /eorþscyld.pdf[768432:1024576] -bafkreidbhmgdt3ajzgoeywwxdbdcquone7buyenwhirl6af2z3gmftiys4 | RawLeaf | /eorþscyld.pdf[1024576:1280720] -bafkreib6iawulrjvzet7dxbqdqneqa7kpy2kacvq7fawjvmlgveoakuzzy | RawLeaf | /eorþscyld.pdf[1280720:1536864] -bafkreicbbl7whtfqsoz6tu663qicdisdyvzp4pp4ylggcttc44j63bdhxy | RawLeaf | /eorþscyld.pdf[1536864:1793008] -bafkreifxe2irzoxtzk3ltpunioehodovu4pwzdtjr2lqtmkg7hyh3o5r2e | RawLeaf | /eorþscyld.pdf[1793008:2049152] -bafkreihr2m4zfx4qvpwkp2fnihn7gbjq6gyr2gv5nbibt2nuajnp2k6pvq | RawLeaf | /eorþscyld.pdf[2049152:2305296] -bafkreibhyumtv62kh3d4rvwpxkdlr52uur5thgmbtyuyxas3c424od7cta | RawLeaf | /eorþscyld.pdf[2305296:2561440] -bafkreiatyvemaol2uxceo4suktpmusl53dqgg5bcqi2rkfhlxnnvpigkci | RawLeaf | /eorþscyld.pdf[2561440:2817584] -bafkreias6ygzx2pnowi3hxj5kmed2rm74hgigwijiandz7vamvrohtkpje | RawLeaf | /eorþscyld.pdf[2817584:3073728] -bafkreihf5g2wmsblx67664w2k3m7hj2bg2wnpumlnty7ssw4gj46oj3bxa | RawLeaf | /eorþscyld.pdf[3073728:3329872] -bafkreib7gr6yi6lhl2p5izuxrxpknf5tc5lhkf5tdhzaxw5g66jxqichra | RawLeaf | /eorþscyld.pdf[3329872:3586016] -bafkreigmnnfehab7c4tblwqgydbaoc76y34rkjfetayb6ytql2eywqnq5y | RawLeaf | /eorþscyld.pdf[3586016:3842160] -bafkreiax5uhfktinmfo3ovm7geaf2266u5ya2slq4bob6eeginffbewszu | RawLeaf | /eorþscyld.pdf[3842160:4098304] -bafkreic5chuwrtofd6ymxawcaxulacfewzfi3zroakmkwjkfmadrxrmln4 | RawLeaf | /eorþscyld.pdf[4098304:4354448] -bafkreich7pcb4lnheypmqb4sikk5m6cuqqbxbjzuk767cx47djxac45o64 | RawLeaf | /eorþscyld.pdf[4354448:4558478] +bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558477] (4558478 B) +bafkreidz265issniggrhdjbwp3c5o4fnmg7xsa6hbn3kmyo7bsafc3jjsy | RawLeaf | ↳ /eorþscyld.pdf[0:256143] (256144 B) +bafkreiepissrnjsgcanpe6hzen5kv66glcr5x5vw5vhferkjscxdj2uv3y | RawLeaf | /eorþscyld.pdf[256144:512287] (256144 B) +bafkreifohr4rlzjpskbdcntrjzolglq6e5uf7n4mgdiw7wl2yhovhx46ve | RawLeaf | /eorþscyld.pdf[512288:768431] (256144 B) +bafkreihiulihaonenbbtrbptmcmzcg2hvk6dypdxvmposz7ansd6h5mkla | RawLeaf | /eorþscyld.pdf[768432:1024575] (256144 B) +bafkreidbhmgdt3ajzgoeywwxdbdcquone7buyenwhirl6af2z3gmftiys4 | RawLeaf | /eorþscyld.pdf[1024576:1280719] (256144 B) +bafkreib6iawulrjvzet7dxbqdqneqa7kpy2kacvq7fawjvmlgveoakuzzy | RawLeaf | /eorþscyld.pdf[1280720:1536863] (256144 B) +bafkreicbbl7whtfqsoz6tu663qicdisdyvzp4pp4ylggcttc44j63bdhxy | RawLeaf | /eorþscyld.pdf[1536864:1793007] (256144 B) +bafkreifxe2irzoxtzk3ltpunioehodovu4pwzdtjr2lqtmkg7hyh3o5r2e | RawLeaf | /eorþscyld.pdf[1793008:2049151] (256144 B) +bafkreihr2m4zfx4qvpwkp2fnihn7gbjq6gyr2gv5nbibt2nuajnp2k6pvq | RawLeaf | /eorþscyld.pdf[2049152:2305295] (256144 B) +bafkreibhyumtv62kh3d4rvwpxkdlr52uur5thgmbtyuyxas3c424od7cta | RawLeaf | /eorþscyld.pdf[2305296:2561439] (256144 B) +bafkreiatyvemaol2uxceo4suktpmusl53dqgg5bcqi2rkfhlxnnvpigkci | RawLeaf | /eorþscyld.pdf[2561440:2817583] (256144 B) +bafkreias6ygzx2pnowi3hxj5kmed2rm74hgigwijiandz7vamvrohtkpje | RawLeaf | /eorþscyld.pdf[2817584:3073727] (256144 B) +bafkreihf5g2wmsblx67664w2k3m7hj2bg2wnpumlnty7ssw4gj46oj3bxa | RawLeaf | /eorþscyld.pdf[3073728:3329871] (256144 B) +bafkreib7gr6yi6lhl2p5izuxrxpknf5tc5lhkf5tdhzaxw5g66jxqichra | RawLeaf | /eorþscyld.pdf[3329872:3586015] (256144 B) +bafkreigmnnfehab7c4tblwqgydbaoc76y34rkjfetayb6ytql2eywqnq5y | RawLeaf | /eorþscyld.pdf[3586016:3842159] (256144 B) +bafkreiax5uhfktinmfo3ovm7geaf2266u5ya2slq4bob6eeginffbewszu | RawLeaf | /eorþscyld.pdf[3842160:4098303] (256144 B) +bafkreic5chuwrtofd6ymxawcaxulacfewzfi3zroakmkwjkfmadrxrmln4 | RawLeaf | /eorþscyld.pdf[4098304:4354447] (256144 B) +bafkreich7pcb4lnheypmqb4sikk5m6cuqqbxbjzuk767cx47djxac45o64 | RawLeaf | /eorþscyld.pdf[4354448:4558477] (204030 B) ``` #### entity @@ -156,25 +205,25 @@ bafkreich7pcb4lnheypmqb4sikk5m6cuqqbxbjzuk767cx47djxac45o64 | RawLeaf | ``` bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / bafybeicqlqnzd2tvs5vlo4c2dz72ryvouwmjwpu2mvxrocuj2u26joxeam | Directory | ↳ /Flibbertigibbet5 -bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558478] -bafkreidz265issniggrhdjbwp3c5o4fnmg7xsa6hbn3kmyo7bsafc3jjsy | RawLeaf | ↳ /eorþscyld.pdf[0:256144] -bafkreiepissrnjsgcanpe6hzen5kv66glcr5x5vw5vhferkjscxdj2uv3y | RawLeaf | /eorþscyld.pdf[256144:512288] -bafkreifohr4rlzjpskbdcntrjzolglq6e5uf7n4mgdiw7wl2yhovhx46ve | RawLeaf | /eorþscyld.pdf[512288:768432] -bafkreihiulihaonenbbtrbptmcmzcg2hvk6dypdxvmposz7ansd6h5mkla | RawLeaf | /eorþscyld.pdf[768432:1024576] -bafkreidbhmgdt3ajzgoeywwxdbdcquone7buyenwhirl6af2z3gmftiys4 | RawLeaf | /eorþscyld.pdf[1024576:1280720] -bafkreib6iawulrjvzet7dxbqdqneqa7kpy2kacvq7fawjvmlgveoakuzzy | RawLeaf | /eorþscyld.pdf[1280720:1536864] -bafkreicbbl7whtfqsoz6tu663qicdisdyvzp4pp4ylggcttc44j63bdhxy | RawLeaf | /eorþscyld.pdf[1536864:1793008] -bafkreifxe2irzoxtzk3ltpunioehodovu4pwzdtjr2lqtmkg7hyh3o5r2e | RawLeaf | /eorþscyld.pdf[1793008:2049152] -bafkreihr2m4zfx4qvpwkp2fnihn7gbjq6gyr2gv5nbibt2nuajnp2k6pvq | RawLeaf | /eorþscyld.pdf[2049152:2305296] -bafkreibhyumtv62kh3d4rvwpxkdlr52uur5thgmbtyuyxas3c424od7cta | RawLeaf | /eorþscyld.pdf[2305296:2561440] -bafkreiatyvemaol2uxceo4suktpmusl53dqgg5bcqi2rkfhlxnnvpigkci | RawLeaf | /eorþscyld.pdf[2561440:2817584] -bafkreias6ygzx2pnowi3hxj5kmed2rm74hgigwijiandz7vamvrohtkpje | RawLeaf | /eorþscyld.pdf[2817584:3073728] -bafkreihf5g2wmsblx67664w2k3m7hj2bg2wnpumlnty7ssw4gj46oj3bxa | RawLeaf | /eorþscyld.pdf[3073728:3329872] -bafkreib7gr6yi6lhl2p5izuxrxpknf5tc5lhkf5tdhzaxw5g66jxqichra | RawLeaf | /eorþscyld.pdf[3329872:3586016] -bafkreigmnnfehab7c4tblwqgydbaoc76y34rkjfetayb6ytql2eywqnq5y | RawLeaf | /eorþscyld.pdf[3586016:3842160] -bafkreiax5uhfktinmfo3ovm7geaf2266u5ya2slq4bob6eeginffbewszu | RawLeaf | /eorþscyld.pdf[3842160:4098304] -bafkreic5chuwrtofd6ymxawcaxulacfewzfi3zroakmkwjkfmadrxrmln4 | RawLeaf | /eorþscyld.pdf[4098304:4354448] -bafkreich7pcb4lnheypmqb4sikk5m6cuqqbxbjzuk767cx47djxac45o64 | RawLeaf | /eorþscyld.pdf[4354448:4558478] +bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558477] (4558478 B) +bafkreidz265issniggrhdjbwp3c5o4fnmg7xsa6hbn3kmyo7bsafc3jjsy | RawLeaf | ↳ /eorþscyld.pdf[0:256143] (256144 B) +bafkreiepissrnjsgcanpe6hzen5kv66glcr5x5vw5vhferkjscxdj2uv3y | RawLeaf | /eorþscyld.pdf[256144:512287] (256144 B) +bafkreifohr4rlzjpskbdcntrjzolglq6e5uf7n4mgdiw7wl2yhovhx46ve | RawLeaf | /eorþscyld.pdf[512288:768431] (256144 B) +bafkreihiulihaonenbbtrbptmcmzcg2hvk6dypdxvmposz7ansd6h5mkla | RawLeaf | /eorþscyld.pdf[768432:1024575] (256144 B) +bafkreidbhmgdt3ajzgoeywwxdbdcquone7buyenwhirl6af2z3gmftiys4 | RawLeaf | /eorþscyld.pdf[1024576:1280719] (256144 B) +bafkreib6iawulrjvzet7dxbqdqneqa7kpy2kacvq7fawjvmlgveoakuzzy | RawLeaf | /eorþscyld.pdf[1280720:1536863] (256144 B) +bafkreicbbl7whtfqsoz6tu663qicdisdyvzp4pp4ylggcttc44j63bdhxy | RawLeaf | /eorþscyld.pdf[1536864:1793007] (256144 B) +bafkreifxe2irzoxtzk3ltpunioehodovu4pwzdtjr2lqtmkg7hyh3o5r2e | RawLeaf | /eorþscyld.pdf[1793008:2049151] (256144 B) +bafkreihr2m4zfx4qvpwkp2fnihn7gbjq6gyr2gv5nbibt2nuajnp2k6pvq | RawLeaf | /eorþscyld.pdf[2049152:2305295] (256144 B) +bafkreibhyumtv62kh3d4rvwpxkdlr52uur5thgmbtyuyxas3c424od7cta | RawLeaf | /eorþscyld.pdf[2305296:2561439] (256144 B) +bafkreiatyvemaol2uxceo4suktpmusl53dqgg5bcqi2rkfhlxnnvpigkci | RawLeaf | /eorþscyld.pdf[2561440:2817583] (256144 B) +bafkreias6ygzx2pnowi3hxj5kmed2rm74hgigwijiandz7vamvrohtkpje | RawLeaf | /eorþscyld.pdf[2817584:3073727] (256144 B) +bafkreihf5g2wmsblx67664w2k3m7hj2bg2wnpumlnty7ssw4gj46oj3bxa | RawLeaf | /eorþscyld.pdf[3073728:3329871] (256144 B) +bafkreib7gr6yi6lhl2p5izuxrxpknf5tc5lhkf5tdhzaxw5g66jxqichra | RawLeaf | /eorþscyld.pdf[3329872:3586015] (256144 B) +bafkreigmnnfehab7c4tblwqgydbaoc76y34rkjfetayb6ytql2eywqnq5y | RawLeaf | /eorþscyld.pdf[3586016:3842159] (256144 B) +bafkreiax5uhfktinmfo3ovm7geaf2266u5ya2slq4bob6eeginffbewszu | RawLeaf | /eorþscyld.pdf[3842160:4098303] (256144 B) +bafkreic5chuwrtofd6ymxawcaxulacfewzfi3zroakmkwjkfmadrxrmln4 | RawLeaf | /eorþscyld.pdf[4098304:4354447] (256144 B) +bafkreich7pcb4lnheypmqb4sikk5m6cuqqbxbjzuk767cx47djxac45o64 | RawLeaf | /eorþscyld.pdf[4354448:4558477] (204030 B) ``` #### block @@ -188,7 +237,7 @@ bafkreich7pcb4lnheypmqb4sikk5m6cuqqbxbjzuk767cx47djxac45o64 | RawLeaf | ``` bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / bafybeicqlqnzd2tvs5vlo4c2dz72ryvouwmjwpu2mvxrocuj2u26joxeam | Directory | ↳ /Flibbertigibbet5 -bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558478] +bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558477] (4558478 B) ``` ### Sharded file in hamt in directory @@ -207,19 +256,19 @@ Same as above but the inner directory is a HAMT and we have an intermediate bloc bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / bafybeiahh6giyfzxpsp6b4y6j6r34xjgsb2si7n6dszwoevnjxdcuee5yq | HAMTShard | ↳ /O bafybeics7ohtet4wfuzszfchbttrgotqzpuruq4bencpdtc443vawbcbni | HAMTShard | ↳ -bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977731] -bafkreibyblakon7atsv2hwzyzrweeb5uriksuscfxjtlzj3l56xmrouwke | RawLeaf | ↳ /Othello.png[0:256144] -bafkreic7nx6e7zezkdz5lp2xcwarmgavr62cyrmtepyagvm2ivuo3jesqe | RawLeaf | /Othello.png[256144:512288] -bafkreicylwcsi7ovreggex5yc7jolt5zrwf74ida3sprzsw37oxpzkwvte | RawLeaf | /Othello.png[512288:768432] -bafkreiainpsn6gadltlhqzgakxvdbxrz5n2cstizn3afhsr2nwefus7pqm | RawLeaf | /Othello.png[768432:1024576] -bafkreihuwjz4av7scz3fdjejtrx3k6ycdjudjusuz6sjzbbsewpgslosxi | RawLeaf | /Othello.png[1024576:1280720] -bafkreiexmzld3vi72v665bbqn473mfc5smmtd6nnndl7dzcapzlxdwzhq4 | RawLeaf | /Othello.png[1280720:1536864] -bafkreignp6prdhvi4mxoef5ybl4c644eyeikd42w3z2atw5mgbzw5p4374 | RawLeaf | /Othello.png[1536864:1793008] -bafkreia6nxplcal74ycukjwfmwmaaraeg6qwclfvb6qja2iaayukzsedoa | RawLeaf | /Othello.png[1793008:2049152] -bafkreiae6iay4fqiethqcvva4qazdxb7szb4dh77dsxiskvewhm5sepwfm | RawLeaf | /Othello.png[2049152:2305296] -bafkreieprkc6i3qzg47k5y3zxejsoglyl5fwrk3fnslbmdmau7surwiocy | RawLeaf | /Othello.png[2305296:2561440] -bafkreidwsenghahefqcfw5ikxd6osi6a35koyki4gibw3sunlmqrroqzc4 | RawLeaf | /Othello.png[2561440:2817584] -bafkreicn24jr4rwjcptuzu677u2aeqyirf7weradcafcnkq5hzo7wickkq | RawLeaf | /Othello.png[2817584:2977731] +bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977730] (2977731 B) +bafkreibyblakon7atsv2hwzyzrweeb5uriksuscfxjtlzj3l56xmrouwke | RawLeaf | ↳ /Othello.png[0:256143] (256144 B) +bafkreic7nx6e7zezkdz5lp2xcwarmgavr62cyrmtepyagvm2ivuo3jesqe | RawLeaf | /Othello.png[256144:512287] (256144 B) +bafkreicylwcsi7ovreggex5yc7jolt5zrwf74ida3sprzsw37oxpzkwvte | RawLeaf | /Othello.png[512288:768431] (256144 B) +bafkreiainpsn6gadltlhqzgakxvdbxrz5n2cstizn3afhsr2nwefus7pqm | RawLeaf | /Othello.png[768432:1024575] (256144 B) +bafkreihuwjz4av7scz3fdjejtrx3k6ycdjudjusuz6sjzbbsewpgslosxi | RawLeaf | /Othello.png[1024576:1280719] (256144 B) +bafkreiexmzld3vi72v665bbqn473mfc5smmtd6nnndl7dzcapzlxdwzhq4 | RawLeaf | /Othello.png[1280720:1536863] (256144 B) +bafkreignp6prdhvi4mxoef5ybl4c644eyeikd42w3z2atw5mgbzw5p4374 | RawLeaf | /Othello.png[1536864:1793007] (256144 B) +bafkreia6nxplcal74ycukjwfmwmaaraeg6qwclfvb6qja2iaayukzsedoa | RawLeaf | /Othello.png[1793008:2049151] (256144 B) +bafkreiae6iay4fqiethqcvva4qazdxb7szb4dh77dsxiskvewhm5sepwfm | RawLeaf | /Othello.png[2049152:2305295] (256144 B) +bafkreieprkc6i3qzg47k5y3zxejsoglyl5fwrk3fnslbmdmau7surwiocy | RawLeaf | /Othello.png[2305296:2561439] (256144 B) +bafkreidwsenghahefqcfw5ikxd6osi6a35koyki4gibw3sunlmqrroqzc4 | RawLeaf | /Othello.png[2561440:2817583] (256144 B) +bafkreicn24jr4rwjcptuzu677u2aeqyirf7weradcafcnkq5hzo7wickkq | RawLeaf | /Othello.png[2817584:2977730] (160147 B) ``` #### entity @@ -234,19 +283,19 @@ bafkreicn24jr4rwjcptuzu677u2aeqyirf7weradcafcnkq5hzo7wickkq | RawLeaf | bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / bafybeiahh6giyfzxpsp6b4y6j6r34xjgsb2si7n6dszwoevnjxdcuee5yq | HAMTShard | ↳ /O bafybeics7ohtet4wfuzszfchbttrgotqzpuruq4bencpdtc443vawbcbni | HAMTShard | ↳ -bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977731] -bafkreibyblakon7atsv2hwzyzrweeb5uriksuscfxjtlzj3l56xmrouwke | RawLeaf | ↳ /Othello.png[0:256144] -bafkreic7nx6e7zezkdz5lp2xcwarmgavr62cyrmtepyagvm2ivuo3jesqe | RawLeaf | /Othello.png[256144:512288] -bafkreicylwcsi7ovreggex5yc7jolt5zrwf74ida3sprzsw37oxpzkwvte | RawLeaf | /Othello.png[512288:768432] -bafkreiainpsn6gadltlhqzgakxvdbxrz5n2cstizn3afhsr2nwefus7pqm | RawLeaf | /Othello.png[768432:1024576] -bafkreihuwjz4av7scz3fdjejtrx3k6ycdjudjusuz6sjzbbsewpgslosxi | RawLeaf | /Othello.png[1024576:1280720] -bafkreiexmzld3vi72v665bbqn473mfc5smmtd6nnndl7dzcapzlxdwzhq4 | RawLeaf | /Othello.png[1280720:1536864] -bafkreignp6prdhvi4mxoef5ybl4c644eyeikd42w3z2atw5mgbzw5p4374 | RawLeaf | /Othello.png[1536864:1793008] -bafkreia6nxplcal74ycukjwfmwmaaraeg6qwclfvb6qja2iaayukzsedoa | RawLeaf | /Othello.png[1793008:2049152] -bafkreiae6iay4fqiethqcvva4qazdxb7szb4dh77dsxiskvewhm5sepwfm | RawLeaf | /Othello.png[2049152:2305296] -bafkreieprkc6i3qzg47k5y3zxejsoglyl5fwrk3fnslbmdmau7surwiocy | RawLeaf | /Othello.png[2305296:2561440] -bafkreidwsenghahefqcfw5ikxd6osi6a35koyki4gibw3sunlmqrroqzc4 | RawLeaf | /Othello.png[2561440:2817584] -bafkreicn24jr4rwjcptuzu677u2aeqyirf7weradcafcnkq5hzo7wickkq | RawLeaf | /Othello.png[2817584:2977731] +bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977730] (2977731 B) +bafkreibyblakon7atsv2hwzyzrweeb5uriksuscfxjtlzj3l56xmrouwke | RawLeaf | ↳ /Othello.png[0:256143] (256144 B) +bafkreic7nx6e7zezkdz5lp2xcwarmgavr62cyrmtepyagvm2ivuo3jesqe | RawLeaf | /Othello.png[256144:512287] (256144 B) +bafkreicylwcsi7ovreggex5yc7jolt5zrwf74ida3sprzsw37oxpzkwvte | RawLeaf | /Othello.png[512288:768431] (256144 B) +bafkreiainpsn6gadltlhqzgakxvdbxrz5n2cstizn3afhsr2nwefus7pqm | RawLeaf | /Othello.png[768432:1024575] (256144 B) +bafkreihuwjz4av7scz3fdjejtrx3k6ycdjudjusuz6sjzbbsewpgslosxi | RawLeaf | /Othello.png[1024576:1280719] (256144 B) +bafkreiexmzld3vi72v665bbqn473mfc5smmtd6nnndl7dzcapzlxdwzhq4 | RawLeaf | /Othello.png[1280720:1536863] (256144 B) +bafkreignp6prdhvi4mxoef5ybl4c644eyeikd42w3z2atw5mgbzw5p4374 | RawLeaf | /Othello.png[1536864:1793007] (256144 B) +bafkreia6nxplcal74ycukjwfmwmaaraeg6qwclfvb6qja2iaayukzsedoa | RawLeaf | /Othello.png[1793008:2049151] (256144 B) +bafkreiae6iay4fqiethqcvva4qazdxb7szb4dh77dsxiskvewhm5sepwfm | RawLeaf | /Othello.png[2049152:2305295] (256144 B) +bafkreieprkc6i3qzg47k5y3zxejsoglyl5fwrk3fnslbmdmau7surwiocy | RawLeaf | /Othello.png[2305296:2561439] (256144 B) +bafkreidwsenghahefqcfw5ikxd6osi6a35koyki4gibw3sunlmqrroqzc4 | RawLeaf | /Othello.png[2561440:2817583] (256144 B) +bafkreicn24jr4rwjcptuzu677u2aeqyirf7weradcafcnkq5hzo7wickkq | RawLeaf | /Othello.png[2817584:2977730] (160147 B) ``` #### block @@ -261,7 +310,7 @@ bafkreicn24jr4rwjcptuzu677u2aeqyirf7weradcafcnkq5hzo7wickkq | RawLeaf | bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / bafybeiahh6giyfzxpsp6b4y6j6r34xjgsb2si7n6dszwoevnjxdcuee5yq | HAMTShard | ↳ /O bafybeics7ohtet4wfuzszfchbttrgotqzpuruq4bencpdtc443vawbcbni | HAMTShard | ↳ -bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977731] +bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977730] (2977731 B) ``` ### Sharded file in directory in hamt in directory @@ -281,10 +330,10 @@ bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | ↳ bafybeifjkcls323ddq3t5ov22yl7i6ks36a7jzv33pjy4stqkils6jzvqe | Directory | ↳ /Throttlebottom -bafybeify427noacqiu6sxaaunk5uw2xhelnkcwktkvh6woi6ahlipsz7em | File | ↳ /supercalifragilisticexpialidocious.txt[0:568521] -bafkreicdwgfhxwnzq7i34cuhnqqqxz6d6jtirupjy5kq3uaphopyw5e2ky | RawLeaf | ↳ /supercalifragilisticexpialidocious.txt[0:256144] -bafkreifmk6qfl6ucap7btfu35rjd37bh7uuefavq6nsuungxjl3or7bz2e | RawLeaf | /supercalifragilisticexpialidocious.txt[256144:512288] -bafkreifk2vcldftxe57ml2cxyfcwb34ukkhaopm46kv3as5vo26ht63fci | RawLeaf | /supercalifragilisticexpialidocious.txt[512288:568521] +bafybeify427noacqiu6sxaaunk5uw2xhelnkcwktkvh6woi6ahlipsz7em | File | ↳ /supercalifragilisticexpialidocious.txt[0:568520] (568521 B) +bafkreicdwgfhxwnzq7i34cuhnqqqxz6d6jtirupjy5kq3uaphopyw5e2ky | RawLeaf | ↳ /supercalifragilisticexpialidocious.txt[0:256143] (256144 B) +bafkreifmk6qfl6ucap7btfu35rjd37bh7uuefavq6nsuungxjl3or7bz2e | RawLeaf | /supercalifragilisticexpialidocious.txt[256144:512287] (256144 B) +bafkreifk2vcldftxe57ml2cxyfcwb34ukkhaopm46kv3as5vo26ht63fci | RawLeaf | /supercalifragilisticexpialidocious.txt[512288:568520] (56233 B) ``` #### entity @@ -317,6 +366,61 @@ bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | ↳ bafybeifjkcls323ddq3t5ov22yl7i6ks36a7jzv33pjy4stqkils6jzvqe | Directory | ↳ /Throttlebottom ``` +### Small file in a directory in a hamt in a directory + +Same result regardless of scope. + +#### all + +[testmark]:# (test/small_file_in_directory_in_hamt_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/fyrd/Cordelia.docx?dag-scope=all +``` + +[testmark]:# (test/small_file_in_directory_in_hamt_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +bafybeiggzqpxmfbve7sutrmcwrlwpx4tkno536d7qdgd6hnhxf74t7gzee | HAMTShard | ↳ +bafybeigfwz54atwgnklb72rfqqn2a3ywoontgf3b3kekxwoxalptx7gc3e | HAMTShard | ↳ +bafybeid25clhrmlypqcsl4ehgxczbidzu2sqvrtuoeckh32hmzz7qisk6m | Directory | ↳ /fyrd +bafkreigtatgpa6dpkm2vxaeglfz4t3j2kkcan6jjxdf4ytwehspjbpxe54 | RawLeaf | ↳ /Cordelia.docx[0:329] (330 B) +``` + +#### entity + +[testmark]:# (test/small_file_in_directory_in_hamt_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/fyrd/Cordelia.docx?dag-scope=entity +``` + +[testmark]:# (test/small_file_in_directory_in_hamt_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +bafybeiggzqpxmfbve7sutrmcwrlwpx4tkno536d7qdgd6hnhxf74t7gzee | HAMTShard | ↳ +bafybeigfwz54atwgnklb72rfqqn2a3ywoontgf3b3kekxwoxalptx7gc3e | HAMTShard | ↳ +bafybeid25clhrmlypqcsl4ehgxczbidzu2sqvrtuoeckh32hmzz7qisk6m | Directory | ↳ /fyrd +bafkreigtatgpa6dpkm2vxaeglfz4t3j2kkcan6jjxdf4ytwehspjbpxe54 | RawLeaf | ↳ /Cordelia.docx[0:329] (330 B) +``` + +#### block + +[testmark]:# (test/small_file_in_directory_in_hamt_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/fyrd/Cordelia.docx?dag-scope=block +``` + +[testmark]:# (test/small_file_in_directory_in_hamt_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +bafybeiggzqpxmfbve7sutrmcwrlwpx4tkno536d7qdgd6hnhxf74t7gzee | HAMTShard | ↳ +bafybeigfwz54atwgnklb72rfqqn2a3ywoontgf3b3kekxwoxalptx7gc3e | HAMTShard | ↳ +bafybeid25clhrmlypqcsl4ehgxczbidzu2sqvrtuoeckh32hmzz7qisk6m | Directory | ↳ /fyrd +bafkreigtatgpa6dpkm2vxaeglfz4t3j2kkcan6jjxdf4ytwehspjbpxe54 | RawLeaf | ↳ /Cordelia.docx[0:329] (330 B) +``` + ### Hamt in directory "All" is too large to be useful, so we'll just do "entity" and "block". @@ -362,3 +466,419 @@ bafybeidaz5aob6tdurpnuj2xeqsjkarqudtono6x2kgbzrzv46ugp3g4ju | HAMTShard | /Z bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious ``` + +### Directory in a directory + +#### all + +[testmark]:# (test/directory_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/by/procrastinatorily?dag-scope=all +``` + +[testmark]:# (test/directory_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeigdgcs3nulbgu6vtnyesfgsaau2grbxzbxxdtpgolsfhtrpjx6mw4 | Directory | ↳ /by +bafybeicsa4ltuq6m3sxzzrkfy6wetemiqvbdgnitkjwlcduswh3cvsccxy | Directory | ↳ /procrastinatorily +bafkreiaquanb6uzch2bkv7iltvwipmawjfihb7jgejqww63vgbqshderxq | RawLeaf | ↳ * (/F[0:727] (728 B)) +bafkreigtze43gghvasxpp3o6mklllhilbyzs63kzdtdlzpyaznk44vxnye | RawLeaf | * (/Gallivant.png[0:1339] (1340 B)) +bafkreifrkcg75acl2yu4jv65x5u7yzn2cnpezmx432p2v2lryzhwqyp63u | RawLeaf | * (/Oberon.xml[0:512] (513 B)) +bafkreiac4vqpljai2qjxx7mgs2equqcs4oq6lmmie2oiqhrkmwukuez3cm | RawLeaf | * (/Pandemonium[0:917] (918 B)) +bafkreiclovx7khthefexgfku4n7pndbqthj5xuiv6ymbd3cx2bve3qebby | RawLeaf | * (/gūþweard.docx[0:580] (581 B)) +bafkreiaqchjvbofqnpwoxwrtcvelnz5neg4y5buqchwgnxwoigjofmnol4 | RawLeaf | * (/juxtapositionally.png[0:2426] (2427 B)) +bafkreif3y7x7uailqhjjuketaa2p34gidoj7ynmnmviejtynydov75dkbq | RawLeaf | * (/sǣlācend[0:430] (431 B)) +bafkreias4xj2qbg36tyo4wirl2db2x2emh4lbfpwq7t3dexjos236a5wgu | RawLeaf | * (/t.png[0:57] (58 B)) +bafkreidzryf2cxeo5bfkxulhxlgr6axh5qfrsh4ns6jwfd2yutzpkcoaxu | RawLeaf | * (/ya.xml[0:417] (418 B)) +bafkreihys4i2ow3csfrhefnmwruwte2bv5ik223ocgbvq3ckwj7xdxzc2m | RawLeaf | * (/ætheling.docx[0:81] (82 B)) +``` + +#### entity + +[testmark]:# (test/directory_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/by/procrastinatorily?dag-scope=entity +``` + +[testmark]:# (test/directory_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeigdgcs3nulbgu6vtnyesfgsaau2grbxzbxxdtpgolsfhtrpjx6mw4 | Directory | ↳ /by +bafybeicsa4ltuq6m3sxzzrkfy6wetemiqvbdgnitkjwlcduswh3cvsccxy | Directory | ↳ /procrastinatorily +``` + +#### block + +[testmark]:# (test/directory_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/by/procrastinatorily?dag-scope=block +``` + +[testmark]:# (test/directory_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeigdgcs3nulbgu6vtnyesfgsaau2grbxzbxxdtpgolsfhtrpjx6mw4 | Directory | ↳ /by +bafybeicsa4ltuq6m3sxzzrkfy6wetemiqvbdgnitkjwlcduswh3cvsccxy | Directory | ↳ /procrastinatorily +``` + +### File in directory, byte ranges + +#### 0:* + +[testmark]:# (test/file_in_directory_byte_ranges/0:*/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:* +``` + +[testmark]:# (test/file_in_directory_byte_ranges/0:*/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) +bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) +bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) +bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) +bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) +bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) +bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) +``` + +#### 0:10 + +Only needs first block. + +[testmark]:# (test/file_in_directory_byte_ranges/0:10/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:10 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/0:10/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) +``` + +#### 0:256143 first block only + +Matches full byte range of first block only. + +[testmark]:# (test/file_in_directory_byte_ranges/0:256143/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:256143 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/0:256143/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) +``` + + +#### 0:256144 second block boundary + +Needs the first byte of the second block, so matches first two blocks. + +[testmark]:# (test/file_in_directory_byte_ranges/0:256144/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:256144 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/0:256144/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) +``` + +#### 0:1793007 second last block + +Needs all but the last block to match. + +[testmark]:# (test/file_in_directory_byte_ranges/0:1793007/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:1793007 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/0:1793007/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) +bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) +bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) +bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) +bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) +bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) +``` + +#### 0:1793008 last block boundary + +Should match all blocks because it includes the first byte of the last block. + +[testmark]:# (test/file_in_directory_byte_ranges/0:1793008/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:1793008 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/0:1793008/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) +bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) +bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) +bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) +bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) +bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) +bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) +``` + +#### 0:1889755 full file + +Precise byte boundaries start:finish. + +[testmark]:# (test/file_in_directory_byte_ranges/0:1889755/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:1889755 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/0:1889755/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) +bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) +bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) +bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) +bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) +bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) +bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) +``` + +#### 0:1889756 full file +1 + +Beyond the end of the file, so should be the same as 0:1889755. + +[testmark]:# (test/file_in_directory_byte_ranges/0:1889756/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:1889756 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/0:1889756/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) +bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) +bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) +bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) +bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) +bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) +bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) +``` + +#### 0:99999999 + +Beyond last byte, should match all blocks. + +[testmark]:# (test/file_in_directory_byte_ranges/0:99999999/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:99999999 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/0:99999999/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) +bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) +bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) +bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) +bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) +bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) +bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) +``` + +#### 1793008:* last block + +Starts at the first byte of the last block. + +[testmark]:# (test/file_in_directory_byte_ranges/1793008:*/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=1793008:* +``` + +[testmark]:# (test/file_in_directory_byte_ranges/1793008:*/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | ↳ /clippet.txt[1793008:1889755] (96748 B) +``` + +#### 1793007:* second last block boundary + +Starts at the last byte of the second last block. + +[testmark]:# (test/file_in_directory_byte_ranges/1793007:*/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=1793007:* +``` + +[testmark]:# (test/file_in_directory_byte_ranges/1793007:*/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | ↳ /clippet.txt[1536864:1793007] (256144 B) +bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) +``` + +#### 256144:* all but first block + +Starts at the first byte of the second block. + +[testmark]:# (test/file_in_directory_byte_ranges/256144:*/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=256144:* +``` + +[testmark]:# (test/file_in_directory_byte_ranges/256144:*/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | ↳ /clippet.txt[256144:512287] (256144 B) +bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) +bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) +bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) +bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) +bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) +bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) +``` + +#### 256143:* first block boundary + +Need all blocks to match this. + +[testmark]:# (test/file_in_directory_byte_ranges/256143:*/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=256143:* +``` + +[testmark]:# (test/file_in_directory_byte_ranges/256143:*/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) +bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) +bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) +bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) +bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) +bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) +bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) +``` + +#### 256143:256144 first and second boundary + +[testmark]:# (test/file_in_directory_byte_ranges/256143:256144/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=256143:256144 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/256143:256144/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) +``` + +#### 256144:256144 second only + +Matching a single byte at the start of the second block + +[testmark]:# (test/file_in_directory_byte_ranges/256144:256144/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=256144:256144 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/256144:256144/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | ↳ /clippet.txt[256144:512287] (256144 B) +``` + +#### 512287:512287 second only + +Matching a single byte at the end of the second block + +[testmark]:# (test/file_in_directory_byte_ranges/512287:512287/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=512287:512287 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/512287:512287/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | ↳ /clippet.txt[256144:512287] (256144 B) +``` + +#### 256143:1793008 full file, boundaries + +Matching this needs all blocks, including one byte from first and last + +[testmark]:# (test/file_in_directory_byte_ranges/256143:1793008/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=256143:1793008 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/256143:1793008/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) +bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) +bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) +bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) +bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) +bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) +bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) +``` + +#### 256144:1793007 inner blocks, boundaries + +Matching this only requires the middle blocks, excluding the first and last + +[testmark]:# (test/file_in_directory_byte_ranges/256144:1793007/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=256144:1793007 +``` + +[testmark]:# (test/file_in_directory_byte_ranges/256144:1793007/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) +bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | ↳ /clippet.txt[256144:512287] (256144 B) +bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) +bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) +bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) +bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) +bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) +``` + +## TODO: + +* Negative ranges, To and From +* Out-of-bounds conditions - return full entity? + diff --git a/pkg/storage/duplicateaddercar.go b/pkg/storage/duplicateaddercar.go index df11ef17..1167d255 100644 --- a/pkg/storage/duplicateaddercar.go +++ b/pkg/storage/duplicateaddercar.go @@ -53,7 +53,7 @@ func (da *DuplicateAdderCar) addDupes() { defer func() { da.streamCompletion <- err }() - sel := types.PathScopeSelector(da.path, da.scope) + sel := types.PathScopeSelector(da.path, da.scope, types.DefaultByteRange()) // we're going to do a verified car where we add dupes back in cfg := verifiedcar.Config{ diff --git a/pkg/types/request.go b/pkg/types/request.go index 20dcedb7..e8b19e33 100644 --- a/pkg/types/request.go +++ b/pkg/types/request.go @@ -3,6 +3,7 @@ package types import ( "errors" "fmt" + "math" "strconv" "strings" @@ -13,7 +14,9 @@ import ( "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" ipldstorage "github.com/ipld/go-ipld-prime/storage" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multicodec" ) @@ -72,6 +75,10 @@ type RetrievalRequest struct { // is not set, Scope and Path will be used to construct a selector. Scope DagScope + // Bytes is the optional byte range within the DAG to fetch. If not set + // the default byte range will fetch the entire file. + Bytes *ByteRange + // Duplicates is a flag that indicates whether duplicate blocks should be // stored into the LinkSystem where they occur in the traversal. Duplicates bool @@ -124,9 +131,21 @@ func NewRequestForPath(store ipldstorage.WritableStorage, cid cid.Cid, path stri }, nil } -func PathScopeSelector(path string, scope DagScope) ipld.Node { +// PathScopeSelector generates a selector for the given path, scope and byte +// range. Use DefaultByteRange() for the default byte range value if none is +// specified. +func PathScopeSelector(path string, scope DagScope, bytes ByteRange) ipld.Node { // Turn the path / scope into a selector - return unixfsnode.UnixFSPathSelectorBuilder(path, scope.TerminalSelectorSpec(), false) + terminal := scope.TerminalSelectorSpec() + if !bytes.IsDefault() { + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + to := bytes.To + if to != math.MaxInt64 { + to++ // selector is exclusive, so increment the end + } + terminal = ssb.ExploreInterpretAs("unixfs", ssb.MatcherSubset(bytes.From, to)) + } + return unixfsnode.UnixFSPathSelectorBuilder(path, terminal, false) } // GetSelector will safely return a selector for this request. If none has been @@ -135,7 +154,11 @@ func (r RetrievalRequest) GetSelector() ipld.Node { if r.Selector != nil { // custom selector return r.Selector } - return PathScopeSelector(r.Path, r.Scope) + br := DefaultByteRange() + if r.Bytes != nil { + br = *r.Bytes + } + return PathScopeSelector(r.Path, r.Scope, br) } // GetUrlPath returns a URL path and query string valid with the Trusted HTTP diff --git a/pkg/types/types.go b/pkg/types/types.go index fd83f8bc..7f299dd9 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -305,6 +305,10 @@ type ByteRange struct { To int64 } +func DefaultByteRange() ByteRange { + return ByteRange{From: 0, To: math.MaxInt64} +} + func (br ByteRange) String() string { to := strconv.FormatInt(br.To, 10) if br.To == math.MaxInt64 { @@ -318,7 +322,7 @@ func (br ByteRange) IsDefault() bool { } func ParseByteRange(s string) (ByteRange, error) { - br := ByteRange{From: 0, To: math.MaxInt64} + br := DefaultByteRange() if s == "" { return br, nil } diff --git a/pkg/verifiedcar/verifiedcar_test.go b/pkg/verifiedcar/verifiedcar_test.go index 890089de..fb2be541 100644 --- a/pkg/verifiedcar/verifiedcar_test.go +++ b/pkg/verifiedcar/verifiedcar_test.go @@ -4,9 +4,11 @@ import ( "bytes" "context" "errors" + "fmt" "io" "math/rand" "os" + "runtime/debug" "testing" "time" @@ -21,6 +23,8 @@ import ( unixfs "github.com/ipfs/go-unixfsnode/testutil" "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/storage" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" @@ -68,7 +72,7 @@ func TestUnixfs20mVariety(t *testing.T) { expectedBlocks[ii] = expectedBlock{blk, false} } - carStream := makeCarStream(t, ctx, []cid.Cid{tc.Root}, expectedBlocks, false, false, false) + carStream, errorCh := makeCarStream(t, ctx, []cid.Cid{tc.Root}, expectedBlocks, false, false, false, nil) lsys := cidlink.DefaultLinkSystem() var writeCounter int @@ -81,13 +85,23 @@ func TestUnixfs20mVariety(t *testing.T) { return nil }, nil } + lsys.StorageReadOpener = func(lc linking.LinkContext, l datamodel.Link) (io.Reader, error) { + debug.PrintStack() + return nil, fmt.Errorf("unexpected read of %s", l.String()) + } // Run the verifier over the CAR stream to see if we end up with // the same query. + br := types.DefaultByteRange() + if tc.ByteRange != nil { + br = *tc.ByteRange + } cfg := verifiedcar.Config{ Root: tc.Root, - Selector: types.PathScopeSelector(tc.Path, tc.Scope), + Selector: types.PathScopeSelector(tc.Path, tc.Scope, br), } + selBytes, _ := ipld.Encode(cfg.Selector, dagjson.Encode) + t.Logf("from=%d, to=%d, selector=%s", br.From, br.To, string(selBytes)) blockCount, byteCount, err := cfg.VerifyCar(ctx, carStream, lsys) req.NoError(err) @@ -95,6 +109,12 @@ func TestUnixfs20mVariety(t *testing.T) { req.Equal(sizeOf(expectedBlocks), byteCount) req.Equal(int(count(expectedBlocks)), writeCounter) + select { + case err := <-errorCh: + req.NoError(err) + default: + } + // Make sure we consumed the entire stream. byt, err := io.ReadAll(carStream) req.NoError(err) @@ -668,12 +688,18 @@ func TestVerifiedCar(t *testing.T) { }, nil } - carStream := makeCarStream(t, ctx, testCase.roots, testCase.blocks, testCase.carv2, testCase.expectErr != "", testCase.incomingHasDups, testCase.streamErr) + carStream, errorCh := makeCarStream(t, ctx, testCase.roots, testCase.blocks, testCase.carv2, testCase.expectErr != "", testCase.incomingHasDups, testCase.streamErr) blockCount, byteCount, err := testCase.cfg.VerifyCar(ctx, carStream, lsys) // read the rest of data io.ReadAll(carStream) + select { + case err := <-errorCh: + req.NoError(err) + default: + } + if testCase.expectErr != "" { req.ErrorContains(err, testCase.expectErr) req.Equal(uint64(0), blockCount) @@ -697,13 +723,12 @@ func makeCarStream( expectErrors bool, allowDuplicatePuts bool, streamError error, -) io.Reader { +) (io.Reader, chan error) { r, w := io.Pipe() + errorCh := make(chan error, 1) go func() { - req := require.New(t) - var carW io.Writer = w var v2f *os.File @@ -712,7 +737,10 @@ func makeCarStream( // can't create a streaming v2 var err error v2f, err = os.CreateTemp(t.TempDir(), "carv2") - req.NoError(err) + if err != nil { + errorCh <- err + return + } t.Cleanup(func() { v2f.Close() os.Remove(v2f.Name()) @@ -721,8 +749,8 @@ func makeCarStream( } carWriter, err := storage.NewWritable(carW, roots, car.WriteAsCarV1(!carv2), car.AllowDuplicatePuts(allowDuplicatePuts)) - req.NoError(err) if err != nil { + errorCh <- err return } for ii, block := range blocks { @@ -731,14 +759,18 @@ func makeCarStream( return } err := carWriter.Put(ctx, block.Cid().KeyString(), block.RawData()) - if !expectErrors { - req.NoError(err) + if !expectErrors && err != nil { + errorCh <- err + return } if ctx.Err() != nil { return } } - req.NoError(carWriter.Finalize()) + if err := carWriter.Finalize(); err != nil { + errorCh <- err + return + } if carv2 { v2f.Seek(0, io.SeekStart) @@ -747,7 +779,9 @@ func makeCarStream( io.Copy(w, v2f) } - req.NoError(w.Close()) + if err := w.Close(); err != nil { + errorCh <- err + } }() go func() { @@ -757,7 +791,7 @@ func makeCarStream( } }() - return r + return r, errorCh } type expectedBlock struct { From b820f66fd4fddb6c7b386039bc2953e62c1b37af Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 28 Jun 2023 17:14:05 +1000 Subject: [PATCH 05/15] fix: make work on windows --- pkg/internal/fixtures/unixfs_20m_variety.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/internal/fixtures/unixfs_20m_variety.go b/pkg/internal/fixtures/unixfs_20m_variety.go index 8937842a..ca078da6 100644 --- a/pkg/internal/fixtures/unixfs_20m_variety.go +++ b/pkg/internal/fixtures/unixfs_20m_variety.go @@ -21,12 +21,13 @@ func filepath(typ string) (string, error) { if err != nil { return "", err } + // convert wd to absolute path, normalizing to / separators + wd = strings.ReplaceAll(wd, "\\", "/") rootInd := strings.LastIndex(wd, "/lassie/pkg/") if rootInd == -1 { return "", fmt.Errorf("could not find root of lassie package") } filename := wd[:rootInd] + "/lassie/pkg/" + file + typ - fmt.Println("Using", filename) return filename, nil } From 0567dbd42ebbb4b1aa026e294b274580e5cd8c18 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Thu, 29 Jun 2023 19:50:43 +1000 Subject: [PATCH 06/15] feat: support _or_ matching when entity-bytes query reaches non-bytes --- go.mod | 3 +- go.sum | 7 ++-- pkg/internal/fixtures/testcases.go | 2 +- pkg/storage/duplicateaddercar.go | 2 +- pkg/types/request.go | 57 +++++++++++++++++++++-------- pkg/types/types.go | 34 ++++++++++------- pkg/verifiedcar/verifiedcar_test.go | 12 +++--- 7 files changed, 73 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index 2e9a86c2..943bbb68 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/ipfs/go-unixfsnode v1.7.2-0.20230622024455-de5cbf74cbe0 github.com/ipld/go-car/v2 v2.10.1 github.com/ipld/go-codec-dagpb v1.6.0 - github.com/ipld/go-ipld-prime v0.20.1-0.20230628064445-0955c324b41a + github.com/ipld/go-ipld-prime v0.20.1-0.20230629094729-829a31766846 github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff github.com/libp2p/go-libp2p v0.27.1 github.com/libp2p/go-libp2p-routing-helpers v0.7.0 @@ -84,7 +84,6 @@ require ( github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect - github.com/ipfs/go-ipfs-files v0.3.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect diff --git a/go.sum b/go.sum index 9caa707d..fcd84706 100644 --- a/go.sum +++ b/go.sum @@ -292,8 +292,7 @@ github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNo github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= -github.com/ipfs/go-ipfs-files v0.3.0 h1:fallckyc5PYjuMEitPNrjRfpwl7YFt69heCOUhsbGxQ= -github.com/ipfs/go-ipfs-files v0.3.0/go.mod h1:xAUtYMwB+iu/dtf6+muHNSFQCJG2dSiStR2P6sn9tIM= +github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6ZpusRg= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= @@ -337,8 +336,8 @@ github.com/ipld/go-car/v2 v2.10.1 h1:MRDqkONNW9WRhB79u+Z3U5b+NoN7lYA5B8n8qI3+BoI github.com/ipld/go-car/v2 v2.10.1/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= -github.com/ipld/go-ipld-prime v0.20.1-0.20230628064445-0955c324b41a h1:o5zc8GsB8u1UMp4Etmb8VBMJun1sEC5SLYye9iLTXr0= -github.com/ipld/go-ipld-prime v0.20.1-0.20230628064445-0955c324b41a/go.mod h1:PRQpXNcJypaPiiSdarsrJABPkYrBvafwDl0B9HjujZ8= +github.com/ipld/go-ipld-prime v0.20.1-0.20230629094729-829a31766846 h1:0gIWxUu4gU/Do25cwk82TRA621GjoZRSyVY472LaBrQ= +github.com/ipld/go-ipld-prime v0.20.1-0.20230629094729-829a31766846/go.mod h1:PRQpXNcJypaPiiSdarsrJABPkYrBvafwDl0B9HjujZ8= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff h1:xbKrIvnpQkbF8iHPk/HGcegsypCDpcXWHhzBCLyCWf8= github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff/go.mod h1:paYP9U4N3/vOzGCuN9kU972vtvw9JUcQjOKyiCFGwRk= diff --git a/pkg/internal/fixtures/testcases.go b/pkg/internal/fixtures/testcases.go index 689fedd0..02e64fe3 100644 --- a/pkg/internal/fixtures/testcases.go +++ b/pkg/internal/fixtures/testcases.go @@ -27,7 +27,7 @@ func (tc TestCase) AsQuery() string { pp = "/" + pp } br := "" - if tc.ByteRange != nil && !tc.ByteRange.IsDefault() { + if !tc.ByteRange.IsDefault() { br = fmt.Sprintf("&entity-bytes=%s", tc.ByteRange.String()) } dup := "" diff --git a/pkg/storage/duplicateaddercar.go b/pkg/storage/duplicateaddercar.go index 1167d255..29454c90 100644 --- a/pkg/storage/duplicateaddercar.go +++ b/pkg/storage/duplicateaddercar.go @@ -53,7 +53,7 @@ func (da *DuplicateAdderCar) addDupes() { defer func() { da.streamCompletion <- err }() - sel := types.PathScopeSelector(da.path, da.scope, types.DefaultByteRange()) + sel := types.PathScopeSelector(da.path, da.scope, nil) // we're going to do a verified car where we add dupes back in cfg := verifiedcar.Config{ diff --git a/pkg/types/request.go b/pkg/types/request.go index e8b19e33..1f931240 100644 --- a/pkg/types/request.go +++ b/pkg/types/request.go @@ -16,6 +16,7 @@ import ( cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" ipldstorage "github.com/ipld/go-ipld-prime/storage" + "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multicodec" @@ -134,16 +135,38 @@ func NewRequestForPath(store ipldstorage.WritableStorage, cid cid.Cid, path stri // PathScopeSelector generates a selector for the given path, scope and byte // range. Use DefaultByteRange() for the default byte range value if none is // specified. -func PathScopeSelector(path string, scope DagScope, bytes ByteRange) ipld.Node { +func PathScopeSelector(path string, scope DagScope, bytes *ByteRange) ipld.Node { // Turn the path / scope into a selector terminal := scope.TerminalSelectorSpec() if !bytes.IsDefault() { - ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) - to := bytes.To - if to != math.MaxInt64 { - to++ // selector is exclusive, so increment the end + var to int64 = math.MaxInt64 + if bytes.To != nil && *bytes.To > 0 { + to = *bytes.To + 1 // selector is exclusive, so increment the end + } + // TODO: negative ranges are not currently supported, we fall-back to matching entire file + if bytes.From >= 0 && to >= 0 { + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + // if we reach a terminal and it's not a file, then we need to fall-back to the default + // selector for the given scope. We do this with a union of the original terminal. + if scope == DagScopeEntity { + // entity is a special case which we can't just union with our matcher because it + // has its own matcher in it which we need to replace with the subset matcher. + terminal = ssb.ExploreInterpretAs("unixfs", + ssb.ExploreUnion( + ssb.MatcherSubset(bytes.From, to), + ssb.ExploreRecursive( + selector.RecursionLimitDepth(1), + ssb.ExploreAll(ssb.ExploreRecursiveEdge()), + ), + ), + ) + } else { + terminal = ssb.ExploreUnion( + ssb.ExploreInterpretAs("unixfs", ssb.MatcherSubset(bytes.From, to)), + terminal, + ) + } } - terminal = ssb.ExploreInterpretAs("unixfs", ssb.MatcherSubset(bytes.From, to)) } return unixfsnode.UnixFSPathSelectorBuilder(path, terminal, false) } @@ -154,11 +177,7 @@ func (r RetrievalRequest) GetSelector() ipld.Node { if r.Selector != nil { // custom selector return r.Selector } - br := DefaultByteRange() - if r.Bytes != nil { - br = *r.Bytes - } - return PathScopeSelector(r.Path, r.Scope, br) + return PathScopeSelector(r.Path, r.Scope, r.Bytes) } // GetUrlPath returns a URL path and query string valid with the Trusted HTTP @@ -206,23 +225,31 @@ func (r RetrievalRequest) GetSupportedProtocols(allSupportedProtocols []multicod } func (r RetrievalRequest) Etag() string { - // https://github.com/ipfs/boxo/pull/303/commits/f61f95481041406df46a1781b1daab34b6605650#r1213918777 + // similar, but extended form of: + // https://github.com/ipfs/boxo/blob/a91e44dbdbd4c36a5b25a1b9df6ee237aa4442d2/gateway/handler_car.go#L167-L184 sb := strings.Builder{} sb.WriteString("/ipfs/") sb.WriteString(r.Cid.String()) if r.Path != "" { - sb.WriteString("/") + sb.WriteRune('/') sb.WriteString(datamodel.ParsePath(r.Path).String()) } if r.Scope != DagScopeAll { - sb.WriteString(".") + sb.WriteRune('.') sb.WriteString(string(r.Scope)) } + if !r.Bytes.IsDefault() { + sb.WriteRune('.') + sb.WriteString(strconv.FormatInt(r.Bytes.From, 10)) + if r.Bytes.To != nil { + sb.WriteRune('.') + sb.WriteString(strconv.FormatInt(*r.Bytes.To, 10)) + } + } if r.Duplicates { sb.WriteString(".dups") } sb.WriteString(".dfs") - // range bytes would go here: `.from.to` suffix := strconv.FormatUint(xxhash.Sum64([]byte(sb.String())), 32) return `"` + r.Cid.String() + ".car." + suffix + `"` } diff --git a/pkg/types/types.go b/pkg/types/types.go index 7f299dd9..2fbba541 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "math" "net/url" "strconv" "strings" @@ -300,29 +299,35 @@ func (ds DagScope) AcceptHeader() string { return "application/vnd.ipld.car;version=1;order=dfs;dups=y" } +// ByteRange represents a range of bytes in a file. The default value is 0 to +// the end of the file, [0:*]. +// The range is inclusive at both ends, so the case of From==To selects a single +// byte. +// Where the end is * or beyond the end of the file, the end of the file is +// selected. type ByteRange struct { From int64 - To int64 + To *int64 } -func DefaultByteRange() ByteRange { - return ByteRange{From: 0, To: math.MaxInt64} +// IsDefault is roughly equivalent to the range matching [0:*] +func (br *ByteRange) IsDefault() bool { + return br == nil || br.From == 0 && br.To == nil } -func (br ByteRange) String() string { - to := strconv.FormatInt(br.To, 10) - if br.To == math.MaxInt64 { - to = "*" +func (br *ByteRange) String() string { + if br.IsDefault() { + return "0:*" + } + to := "*" // default to end of file + if br.To != nil { + to = strconv.FormatInt(*br.To, 10) } return fmt.Sprintf("%d:%s", br.From, to) } -func (br ByteRange) IsDefault() bool { - return br.From == 0 && br.To == math.MaxInt64 -} - func ParseByteRange(s string) (ByteRange, error) { - br := DefaultByteRange() + br := ByteRange{} if s == "" { return br, nil } @@ -339,10 +344,11 @@ func ParseByteRange(s string) (ByteRange, error) { return br, fmt.Errorf("invalid byte range: %s", s) } if parts[1] != "*" { - br.To, err = strconv.ParseInt(parts[1], 10, 64) + to, err := strconv.ParseInt(parts[1], 10, 64) if err != nil { return br, err } + br.To = &to } return br, nil } diff --git a/pkg/verifiedcar/verifiedcar_test.go b/pkg/verifiedcar/verifiedcar_test.go index fb2be541..a3671edb 100644 --- a/pkg/verifiedcar/verifiedcar_test.go +++ b/pkg/verifiedcar/verifiedcar_test.go @@ -92,16 +92,14 @@ func TestUnixfs20mVariety(t *testing.T) { // Run the verifier over the CAR stream to see if we end up with // the same query. - br := types.DefaultByteRange() - if tc.ByteRange != nil { - br = *tc.ByteRange - } cfg := verifiedcar.Config{ Root: tc.Root, - Selector: types.PathScopeSelector(tc.Path, tc.Scope, br), + Selector: types.PathScopeSelector(tc.Path, tc.Scope, tc.ByteRange), + } + { + selBytes, _ := ipld.Encode(cfg.Selector, dagjson.Encode) + t.Logf("selector=%s, entity-bytes=%s", string(selBytes), tc.ByteRange.String()) } - selBytes, _ := ipld.Encode(cfg.Selector, dagjson.Encode) - t.Logf("from=%d, to=%d, selector=%s", br.From, br.To, string(selBytes)) blockCount, byteCount, err := cfg.VerifyCar(ctx, carStream, lsys) req.NoError(err) From 3e3660ab04f7e14cb80a9272a85b394db34776b4 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 30 Jun 2023 13:25:38 +1000 Subject: [PATCH 07/15] feat: support entity-bytes via daemon, add unixfs fixture itests --- pkg/internal/itest/direct_fetch_test.go | 2 +- pkg/internal/itest/testpeer/backedstore.go | 126 +++++++++++++++ pkg/internal/itest/testpeer/generator.go | 143 +----------------- pkg/internal/itest/testpeer/peerhttpserver.go | 140 +++++++++++++++++ pkg/internal/itest/trustless_fetch_test.go | 139 +++++++++++++++++ pkg/server/http/ipfs.go | 27 +++- pkg/server/http/util.go | 14 ++ pkg/storage/duplicateaddercar.go | 15 +- pkg/storage/duplicateaddercar_test.go | 3 +- pkg/types/request.go | 16 +- pkg/types/types.go | 4 +- 11 files changed, 481 insertions(+), 148 deletions(-) create mode 100644 pkg/internal/itest/testpeer/backedstore.go create mode 100644 pkg/internal/itest/trustless_fetch_test.go diff --git a/pkg/internal/itest/direct_fetch_test.go b/pkg/internal/itest/direct_fetch_test.go index c15d4551..a03fa60e 100644 --- a/pkg/internal/itest/direct_fetch_test.go +++ b/pkg/internal/itest/direct_fetch_test.go @@ -116,7 +116,7 @@ func TestDirectFetch(t *testing.T) { }() outCar, err := storage.NewReadableWritable(outFile, []cid.Cid{srcData1.Root}, carv2.WriteAsCarV1(true)) req.NoError(err) - request, err := types.NewRequestForPath(outCar, srcData1.Root, "", types.DagScopeAll) + request, err := types.NewRequestForPath(outCar, srcData1.Root, "", types.DagScopeAll, nil) req.NoError(err) _, err = lassie.Fetch(ctx, request, func(types.RetrievalEvent) {}) req.NoError(err) diff --git a/pkg/internal/itest/testpeer/backedstore.go b/pkg/internal/itest/testpeer/backedstore.go new file mode 100644 index 00000000..e9491530 --- /dev/null +++ b/pkg/internal/itest/testpeer/backedstore.go @@ -0,0 +1,126 @@ +package testpeer + +import ( + "bytes" + "context" + "errors" + "io" + + "github.com/ipfs/boxo/blockstore" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +var _ blockstore.Blockstore = (*BackedStore)(nil) +var _ blockstore.Blockstore = (*linkSystemBlockstore)(nil) + +type BackedStore struct { + Backing blockstore.Blockstore +} + +func (bs *BackedStore) DeleteBlock(ctx context.Context, c cid.Cid) error { + return bs.Backing.DeleteBlock(ctx, c) +} + +func (bs *BackedStore) Has(ctx context.Context, c cid.Cid) (bool, error) { + return bs.Backing.Has(ctx, c) +} + +func (bs *BackedStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { + return bs.Backing.Get(ctx, c) +} + +func (bs *BackedStore) GetSize(ctx context.Context, c cid.Cid) (int, error) { + return bs.Backing.GetSize(ctx, c) +} + +func (bs *BackedStore) Put(ctx context.Context, blk blocks.Block) error { + return bs.Backing.Put(ctx, blk) +} + +func (bs *BackedStore) PutMany(ctx context.Context, blks []blocks.Block) error { + return bs.Backing.PutMany(ctx, blks) +} + +func (bs *BackedStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return bs.Backing.AllKeysChan(ctx) +} + +func (bs *BackedStore) HashOnRead(enabled bool) { + bs.Backing.HashOnRead(enabled) +} + +func (bs *BackedStore) UseLinkSystem(lsys linking.LinkSystem) { + bs.Backing = &linkSystemBlockstore{lsys} +} + +type linkSystemBlockstore struct { + lsys linking.LinkSystem +} + +func (lsbs *linkSystemBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error { + return errors.New("not supported") +} + +func (lsbs *linkSystemBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) { + _, err := lsbs.lsys.StorageReadOpener(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: c}) + if err != nil { + return false, err + } + return true, nil +} + +func (lsbs *linkSystemBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { + rdr, err := lsbs.lsys.StorageReadOpener(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: c}) + if err != nil { + return nil, err + } + var buf bytes.Buffer + _, err = io.Copy(&buf, rdr) + if err != nil { + return nil, err + } + return blocks.NewBlockWithCid(buf.Bytes(), c) +} + +func (lsbs *linkSystemBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) { + rdr, err := lsbs.lsys.StorageReadOpener(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: c}) + if err != nil { + return 0, err + } + i, err := io.Copy(io.Discard, rdr) + if err != nil { + return 0, err + } + return int(i), nil +} + +func (lsbs *linkSystemBlockstore) Put(ctx context.Context, blk blocks.Block) error { + w, wc, err := lsbs.lsys.StorageWriteOpener(linking.LinkContext{Ctx: ctx}) + if err != nil { + return err + } + if _, err = io.Copy(w, bytes.NewReader(blk.RawData())); err != nil { + return err + } + return wc(cidlink.Link{Cid: blk.Cid()}) +} + +func (lsbs *linkSystemBlockstore) PutMany(ctx context.Context, blks []blocks.Block) error { + for _, blk := range blks { + if err := lsbs.Put(ctx, blk); err != nil { + return err + } + } + return nil +} + +func (lsbs *linkSystemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return nil, errors.New("not supported") +} + +func (lsbs *linkSystemBlockstore) HashOnRead(enabled bool) { + lsbs.lsys.TrustedStorage = !enabled +} diff --git a/pkg/internal/itest/testpeer/generator.go b/pkg/internal/itest/testpeer/generator.go index ae1d9648..bb683cf8 100644 --- a/pkg/internal/itest/testpeer/generator.go +++ b/pkg/internal/itest/testpeer/generator.go @@ -1,13 +1,11 @@ package testpeer import ( - "bytes" "context" "errors" "fmt" "io" "net" - "net/http" "strings" "testing" "time" @@ -16,7 +14,6 @@ import ( dtimpl "github.com/filecoin-project/go-data-transfer/v2/impl" dtnet "github.com/filecoin-project/go-data-transfer/v2/network" gstransport "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" - "github.com/filecoin-project/lassie/pkg/types" bsnet "github.com/ipfs/boxo/bitswap/network" "github.com/ipfs/boxo/bitswap/server" "github.com/ipfs/go-cid" @@ -29,17 +26,9 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" delay "github.com/ipfs/go-ipfs-delay" "github.com/ipfs/go-log/v2" - "github.com/ipfs/go-unixfsnode" - "github.com/ipld/go-car/v2" - "github.com/ipld/go-car/v2/storage" - dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipld/go-ipld-prime/traversal" - "github.com/ipld/go-ipld-prime/traversal/selector" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" tnet "github.com/libp2p/go-libp2p-testing/net" p2ptestutil "github.com/libp2p/go-libp2p-testing/netutil" @@ -166,7 +155,7 @@ type TestPeer struct { BitswapNetwork bsnet.BitSwapNetwork DatatransferServer datatransfer.Manager HttpServer *TestPeerHttpServer - blockstore blockstore.Blockstore + blockstore *BackedStore Host host.Host blockstoreDelay delay.D LinkSystem *linking.LinkSystem @@ -175,7 +164,7 @@ type TestPeer struct { } // Blockstore returns the block store for this test instance -func (i *TestPeer) Blockstore() blockstore.Blockstore { +func (i *TestPeer) Blockstore() *BackedStore { return i.blockstore } @@ -287,7 +276,8 @@ func newTestPeer(ctx context.Context, mn mocknet.Mocknet, p tnet.Identity) (Test panic(err.Error()) } - dstore := ds_sync.MutexWrap(ds.NewMapDatastore()) + baseStore := ds.NewMapDatastore() + dstore := ds_sync.MutexWrap(baseStore) dstoreDelayed := delayed.New(dstore, bsdelay) bstore, err := blockstore.CachedBlockstore(ctx, @@ -296,11 +286,12 @@ func newTestPeer(ctx context.Context, mn mocknet.Mocknet, p tnet.Identity) (Test if err != nil { return TestPeer{}, nil, err } - lsys := storeutil.LinkSystemForBlockstore(bstore) + backedStore := &BackedStore{Backing: bstore} + lsys := storeutil.LinkSystemForBlockstore(backedStore) tp := TestPeer{ Host: client, ID: p.ID(), - blockstore: bstore, + blockstore: backedStore, blockstoreDelay: bsdelay, LinkSystem: &lsys, Cids: make(map[cid.Cid]struct{}), @@ -342,126 +333,6 @@ func StartAndWaitForReady(ctx context.Context, manager datatransfer.Manager) err } } -func MockIpfsHandler(ctx context.Context, lsys linking.LinkSystem) func(http.ResponseWriter, *http.Request) { - return func(res http.ResponseWriter, req *http.Request) { - urlPath := strings.Split(req.URL.Path, "/")[1:] - - // validate CID path parameter - cidStr := urlPath[1] - rootCid, err := cid.Parse(cidStr) - if err != nil { - http.Error(res, fmt.Sprintf("Failed to parse CID path parameter: %s", cidStr), http.StatusBadRequest) - return - } - - // Grab unixfs path if it exists - unixfsPath := "" - if len(urlPath) > 2 { - unixfsPath = "/" + strings.Join(urlPath[2:], "/") - } - - acceptTypes := strings.Split(req.Header.Get("Accept"), ",") - includeDupes := false - for _, acceptType := range acceptTypes { - typeParts := strings.Split(acceptType, ";") - if typeParts[0] == "application/vnd.ipld.car" { - for _, nextPart := range typeParts[1:] { - pair := strings.Split(nextPart, "=") - if len(pair) == 2 { - attr := strings.TrimSpace(pair[0]) - value := strings.TrimSpace(pair[1]) - if attr == "dups" && value == "y" { - includeDupes = true - } - } - } - } - } - - // We're always providing the dag-scope parameter, so add a failure case if we stop - // providing it in the future - if !req.URL.Query().Has("dag-scope") { - http.Error(res, "Missing dag-scope parameter", http.StatusBadRequest) - return - } - - // Parse car scope and use it to get selector - var dagScope types.DagScope - switch req.URL.Query().Get("dag-scope") { - case "all": - dagScope = types.DagScopeAll - case "entity": - dagScope = types.DagScopeEntity - case "block": - dagScope = types.DagScopeBlock - default: - http.Error(res, fmt.Sprintf("Invalid dag-scope parameter: %s", req.URL.Query().Get("dag-scope")), http.StatusBadRequest) - return - } - - selNode := unixfsnode.UnixFSPathSelectorBuilder(unixfsPath, dagScope.TerminalSelectorSpec(), false) - sel, err := selector.CompileSelector(selNode) - if err != nil { - http.Error(res, fmt.Sprintf("Failed to compile selector from dag-scope: %v", err), http.StatusInternalServerError) - return - } - - // Write to response writer - carWriter, err := storage.NewWritable(res, []cid.Cid{rootCid}, car.WriteAsCarV1(true), car.AllowDuplicatePuts(includeDupes)) - if err != nil { - http.Error(res, fmt.Sprintf("Failed to create car writer: %v", err), http.StatusInternalServerError) - return - } - - // Extend the StorageReadOpener func to write to the carWriter - originalSRO := lsys.StorageReadOpener - lsys.StorageReadOpener = func(lc linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { - r, err := originalSRO(lc, lnk) - if err != nil { - return nil, err - } - byts, err := io.ReadAll(r) - if err != nil { - return nil, err - } - err = carWriter.Put(ctx, lnk.(cidlink.Link).Cid.KeyString(), byts) - if err != nil { - return nil, err - } - - return bytes.NewReader(byts), nil - } - - protoChooser := dagpb.AddSupportToChooser(basicnode.Chooser) - lnk := cidlink.Link{Cid: rootCid} - lnkCtx := linking.LinkContext{} - proto, err := protoChooser(lnk, lnkCtx) - if err != nil { - http.Error(res, fmt.Sprintf("Failed to choose prototype node: %s", cidStr), http.StatusBadRequest) - return - } - - rootNode, err := lsys.Load(lnkCtx, lnk, proto) - if err != nil { - http.Error(res, fmt.Sprintf("Failed to load root cid into link system: %v", err), http.StatusInternalServerError) - return - } - - cfg := &traversal.Config{ - Ctx: ctx, - LinkSystem: lsys, - LinkTargetNodePrototypeChooser: protoChooser, - } - progress := traversal.Progress{Cfg: cfg} - - err = progress.WalkMatching(rootNode, sel, unixfsnode.BytesConsumingMatcher) - if err != nil { - // if we loaded the first block, we can't write headers any more - return - } - } -} - // RandTestPeerIdentity is a wrapper around // github.com/libp2p/go-libp2p-testing/netutil/RandTestBogusIdentity that // ensures the returned identity has an available port. The identity generated diff --git a/pkg/internal/itest/testpeer/peerhttpserver.go b/pkg/internal/itest/testpeer/peerhttpserver.go index 54588c2e..a439d408 100644 --- a/pkg/internal/itest/testpeer/peerhttpserver.go +++ b/pkg/internal/itest/testpeer/peerhttpserver.go @@ -1,11 +1,26 @@ package testpeer import ( + "bytes" "context" "fmt" + "io" "net" "net/http" + "strings" + "github.com/filecoin-project/lassie/pkg/types" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-unixfsnode" + "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/storage" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector" servertiming "github.com/mitchellh/go-server-timing" ) @@ -65,3 +80,128 @@ func (s *TestPeerHttpServer) Close() error { s.cancel() return s.server.Shutdown(context.Background()) } + +func MockIpfsHandler(ctx context.Context, lsys linking.LinkSystem) func(http.ResponseWriter, *http.Request) { + return func(res http.ResponseWriter, req *http.Request) { + urlPath := strings.Split(req.URL.Path, "/")[1:] + + // validate CID path parameter + cidStr := urlPath[1] + rootCid, err := cid.Parse(cidStr) + if err != nil { + http.Error(res, fmt.Sprintf("Failed to parse CID path parameter: %s", cidStr), http.StatusBadRequest) + return + } + + // Grab unixfs path if it exists + unixfsPath := "" + if len(urlPath) > 2 { + unixfsPath = "/" + strings.Join(urlPath[2:], "/") + } + + acceptTypes := strings.Split(req.Header.Get("Accept"), ",") + includeDupes := false + for _, acceptType := range acceptTypes { + typeParts := strings.Split(acceptType, ";") + if typeParts[0] == "application/vnd.ipld.car" { + for _, nextPart := range typeParts[1:] { + pair := strings.Split(nextPart, "=") + if len(pair) == 2 { + attr := strings.TrimSpace(pair[0]) + value := strings.TrimSpace(pair[1]) + if attr == "dups" && value == "y" { + includeDupes = true + } + } + } + } + } + + // We're always providing the dag-scope parameter, so add a failure case if we stop + // providing it in the future + if !req.URL.Query().Has("dag-scope") { + http.Error(res, "Missing dag-scope parameter", http.StatusBadRequest) + return + } + + // Parse car scope and use it to get selector + var dagScope types.DagScope + switch req.URL.Query().Get("dag-scope") { + case "all": + dagScope = types.DagScopeAll + case "entity": + dagScope = types.DagScopeEntity + case "block": + dagScope = types.DagScopeBlock + default: + http.Error(res, fmt.Sprintf("Invalid dag-scope parameter: %s", req.URL.Query().Get("dag-scope")), http.StatusBadRequest) + return + } + var byteRange *types.ByteRange + if req.URL.Query().Get("entity-bytes") != "" { + br, err := types.ParseByteRange(req.URL.Query().Get("entity-bytes")) + if err != nil { + http.Error(res, fmt.Sprintf("Invalid entity-bytes parameter: %s", req.URL.Query().Get("entity-bytes")), http.StatusBadRequest) + return + } + byteRange = &br + } + + sel, err := selector.CompileSelector(types.PathScopeSelector(unixfsPath, dagScope, byteRange)) + if err != nil { + http.Error(res, fmt.Sprintf("Failed to compile selector from dag-scope: %v", err), http.StatusInternalServerError) + return + } + + // Write to response writer + carWriter, err := storage.NewWritable(res, []cid.Cid{rootCid}, car.WriteAsCarV1(true), car.AllowDuplicatePuts(includeDupes)) + if err != nil { + http.Error(res, fmt.Sprintf("Failed to create car writer: %v", err), http.StatusInternalServerError) + return + } + + // Extend the StorageReadOpener func to write to the carWriter + originalSRO := lsys.StorageReadOpener + lsys.StorageReadOpener = func(lc linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { + r, err := originalSRO(lc, lnk) + if err != nil { + return nil, err + } + byts, err := io.ReadAll(r) + if err != nil { + return nil, err + } + err = carWriter.Put(ctx, lnk.(cidlink.Link).Cid.KeyString(), byts) + if err != nil { + return nil, err + } + + return bytes.NewReader(byts), nil + } + + protoChooser := dagpb.AddSupportToChooser(basicnode.Chooser) + lnk := cidlink.Link{Cid: rootCid} + lnkCtx := linking.LinkContext{} + proto, err := protoChooser(lnk, lnkCtx) + if err != nil { + http.Error(res, fmt.Sprintf("Failed to choose prototype node: %s", cidStr), http.StatusBadRequest) + return + } + + rootNode, err := lsys.Load(lnkCtx, lnk, proto) + if err != nil { + http.Error(res, fmt.Sprintf("Failed to load root cid into link system: %v", err), http.StatusInternalServerError) + return + } + + cfg := &traversal.Config{ + Ctx: ctx, + LinkSystem: lsys, + LinkTargetNodePrototypeChooser: protoChooser, + } + progress := traversal.Progress{Cfg: cfg} + + _ = progress.WalkMatching(rootNode, sel, unixfsnode.BytesConsumingMatcher) + // if we loaded the first block, we can't write headers any more so don't bother + } +} diff --git a/pkg/internal/itest/trustless_fetch_test.go b/pkg/internal/itest/trustless_fetch_test.go new file mode 100644 index 00000000..67502794 --- /dev/null +++ b/pkg/internal/itest/trustless_fetch_test.go @@ -0,0 +1,139 @@ +/* //go:build !race */ + +package itest + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "sync" + "testing" + "time" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/lassie/pkg/internal/fixtures" + "github.com/filecoin-project/lassie/pkg/internal/itest/mocknet" + "github.com/filecoin-project/lassie/pkg/lassie" + httpserver "github.com/filecoin-project/lassie/pkg/server/http" + "github.com/google/uuid" + "github.com/ipfs/go-unixfsnode" + "github.com/ipld/go-car/v2" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/stretchr/testify/require" +) + +func TestTrustlessUnixfsFetch(t *testing.T) { + req := require.New(t) + + testCases, err := fixtures.Unixfs20mVarietyCases() + req.NoError(err) + storage, closer, err := fixtures.Unixfs20mVarietyReadableStorage() + req.NoError(err) + defer closer.Close() + + lsys := cidlink.DefaultLinkSystem() + lsys.TrustedStorage = true + unixfsnode.AddUnixFSReificationToLinkSystem(&lsys) + lsys.SetReadStorage(storage) + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + req := require.New(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + t.Logf("query=%s, blocks=%d", tc.AsQuery(), len(tc.ExpectedCids)) + + mrn := mocknet.NewMockRetrievalNet(ctx, t) + mrn.AddHttpPeers(1) + require.NoError(t, mrn.MN.LinkAll()) + mrn.Remotes[0].Blockstore().UseLinkSystem(lsys) + mrn.Remotes[0].Cids[tc.Root] = struct{}{} + var finishedChan chan []datatransfer.Event + // for graphsync: finishedChan := mocknet.SetupRetrieval(t, mrn.Remotes[0]) + + lassie, err := lassie.NewLassie( + ctx, + lassie.WithProviderTimeout(20*time.Second), + lassie.WithHost(mrn.Self), + lassie.WithFinder(mrn.Finder), + ) + req.NoError(err) + cfg := httpserver.HttpServerConfig{Address: "127.0.0.1", Port: 0, TempDir: t.TempDir()} + httpServer, err := httpserver.NewHttpServer(ctx, lassie, cfg) + req.NoError(err) + serverError := make(chan error, 1) + go func() { + serverError <- httpServer.Start() + }() + responseChan := make(chan *http.Response, 1) + go func() { + // Make a request for our CID and read the complete CAR bytes + addr := fmt.Sprintf("http://%s%s", httpServer.Addr(), tc.AsQuery()) + getReq, err := http.NewRequest("GET", addr, nil) + req.NoError(err) + getReq.Header.Add("Accept", "application/vnd.ipld.car") + t.Log("Fetching", getReq.URL.String()) + resp, err := http.DefaultClient.Do(getReq) + req.NoError(err) + responseChan <- resp + }() + var resp *http.Response + select { + case resp = <-responseChan: + case <-ctx.Done(): + req.FailNow("Did not receive responses") + } + if finishedChan != nil { + // for graphsync + var wg sync.WaitGroup + wg.Add(1) + go func() { + mocknet.WaitForFinish(ctx, t, finishedChan, 1*time.Second) + wg.Done() + }() + wg.Wait() + } + if resp.StatusCode != http.StatusOK { + body, err := io.ReadAll(resp.Body) + req.NoError(err) + req.Failf("200 response code not received", "got code: %d, body: %s", resp.StatusCode, string(body)) + } + req.Equal(fmt.Sprintf(`attachment; filename="%s.car"`, tc.Root.String()), resp.Header.Get("Content-Disposition")) + req.Equal("none", resp.Header.Get("Accept-Ranges")) + req.Equal("public, max-age=29030400, immutable", resp.Header.Get("Cache-Control")) + req.Equal("application/vnd.ipld.car; version=1", resp.Header.Get("Content-Type")) + req.Equal("nosniff", resp.Header.Get("X-Content-Type-Options")) + etagStart := fmt.Sprintf(`"%s.car.`, tc.Root.String()) + etagGot := resp.Header.Get("ETag") + req.True(strings.HasPrefix(etagGot, etagStart), "ETag should start with [%s], got [%s]", etagStart, etagGot) + req.Equal(`"`, etagGot[len(etagGot)-1:], "ETag should end with a quote") + req.Equal(fmt.Sprintf("/ipfs/%s%s", tc.Root.String(), tc.Path), resp.Header.Get("X-Ipfs-Path")) + requestId := resp.Header.Get("X-Trace-Id") + require.NotEmpty(t, requestId) + _, err = uuid.Parse(requestId) + req.NoError(err) + + rdr, err := car.NewBlockReader(resp.Body) + req.NoError(err) + req.Len(rdr.Roots, 1) + req.Equal(tc.Root.String(), rdr.Roots[0].String()) + for ii := 0; ; ii++ { + blk, err := rdr.Next() + if err == io.EOF { + if ii != len(tc.ExpectedCids) { + req.FailNowf("unexpected EOF", "expected %d blocks, got %d", len(tc.ExpectedCids), ii) + } + break + } + req.NoError(err) + if ii >= len(tc.ExpectedCids) { + req.FailNowf("unexpected block", "got block %d, expected %d", ii, len(tc.ExpectedCids)) + } + req.Equal(tc.ExpectedCids[ii].String(), blk.Cid().String(), "unexpected block #%d", ii) + } + }) + } +} diff --git a/pkg/server/http/ipfs.go b/pkg/server/http/ipfs.go index f805c6a1..e5e8738d 100644 --- a/pkg/server/http/ipfs.go +++ b/pkg/server/http/ipfs.go @@ -82,6 +82,12 @@ func ipfsHandler(lassie *lassie.Lassie, cfg HttpServerConfig) func(http.Response return } + byteRange, err := ParseByteRange(req) + if err != nil { + errorResponse(res, statusLogger, http.StatusBadRequest, err) + return + } + protocols, err := parseProtocols(req) if err != nil { errorResponse(res, statusLogger, http.StatusBadRequest, err) @@ -123,7 +129,7 @@ func ipfsHandler(lassie *lassie.Lassie, cfg HttpServerConfig) func(http.Response tempStore := storage.NewDeferredStorageCar(cfg.TempDir) var carWriter storage.DeferredWriter if includeDupes { - carWriter = storage.NewDuplicateAdderCarForStream(req.Context(), rootCid, path.String(), dagScope, tempStore, res) + carWriter = storage.NewDuplicateAdderCarForStream(req.Context(), rootCid, path.String(), dagScope, byteRange, tempStore, res) } else { carWriter = storage.NewDeferredCarWriterForStream(rootCid, res) } @@ -135,7 +141,7 @@ func ipfsHandler(lassie *lassie.Lassie, cfg HttpServerConfig) func(http.Response }() var store types.ReadableWritableStorage = carStore - request, err := types.NewRequestForPath(store, rootCid, path.String(), dagScope) + request, err := types.NewRequestForPath(store, rootCid, path.String(), dagScope, byteRange) if err != nil { errorResponse(res, statusLogger, http.StatusInternalServerError, fmt.Errorf("failed to create request: %w", err)) return @@ -185,7 +191,22 @@ func ipfsHandler(lassie *lassie.Lassie, cfg HttpServerConfig) func(http.Response } // servertiming metrics - logger.Debugw("fetching CID", "retrievalId", retrievalId, "CID", rootCid.String(), "path", path.String(), "dagScope", dagScope) + logger.Debugw("fetching CID", + "retrievalId", + retrievalId, + "CID", + rootCid.String(), + "path", + path.String(), + "dagScope", + dagScope, + "byteRange", + byteRange, + "includeDupes", + includeDupes, + "blockLimit", + blockLimit, + ) stats, err := lassie.Fetch(req.Context(), request, servertimingsSubscriber(req)) // force all blocks to flush diff --git a/pkg/server/http/util.go b/pkg/server/http/util.go index f5c7e9b5..7c5101c7 100644 --- a/pkg/server/http/util.go +++ b/pkg/server/http/util.go @@ -23,6 +23,20 @@ func ParseScope(req *http.Request) (types.DagScope, error) { return types.DagScopeAll, nil } +// ParseByteRange returns the entity-bytes query parameter if one is set in the +// query string or nil if one is not set. An error is returned if an +// entity-bytes query string is not a valid byte range. +func ParseByteRange(req *http.Request) (*types.ByteRange, error) { + if req.URL.Query().Has("entity-bytes") { + br, err := types.ParseByteRange(req.URL.Query().Get("entity-bytes")) + if err != nil { + return nil, err + } + return &br, nil + } + return nil, nil +} + // ParseFilename returns the filename query parameter or an error if the filename // extension is not ".car". Lassie only supports returning CAR data. // See https://specs.ipfs.tech/http-gateways/path-gateway/#filename-request-query-parameter diff --git a/pkg/storage/duplicateaddercar.go b/pkg/storage/duplicateaddercar.go index 29454c90..f5b6a164 100644 --- a/pkg/storage/duplicateaddercar.go +++ b/pkg/storage/duplicateaddercar.go @@ -24,13 +24,23 @@ type DuplicateAdderCar struct { root cid.Cid path string scope types.DagScope + bytes *types.ByteRange store *DeferredStorageCar blockStream *blockStream streamCompletion chan error streamCompletionLk sync.Mutex } -func NewDuplicateAdderCarForStream(ctx context.Context, root cid.Cid, path string, scope types.DagScope, store *DeferredStorageCar, outStream io.Writer) *DuplicateAdderCar { +func NewDuplicateAdderCarForStream( + ctx context.Context, + root cid.Cid, + path string, + scope types.DagScope, + bytes *types.ByteRange, + store *DeferredStorageCar, + outStream io.Writer, +) *DuplicateAdderCar { + blockStream := &blockStream{ctx: ctx, seen: make(map[cid.Cid]struct{})} blockStream.blockBuffer = list.New() blockStream.cond = sync.NewCond(&blockStream.mu) @@ -43,6 +53,7 @@ func NewDuplicateAdderCarForStream(ctx context.Context, root cid.Cid, path strin root: root, path: path, scope: scope, + bytes: bytes, store: store, blockStream: blockStream, } @@ -53,7 +64,7 @@ func (da *DuplicateAdderCar) addDupes() { defer func() { da.streamCompletion <- err }() - sel := types.PathScopeSelector(da.path, da.scope, nil) + sel := types.PathScopeSelector(da.path, da.scope, da.bytes) // we're going to do a verified car where we add dupes back in cfg := verifiedcar.Config{ diff --git a/pkg/storage/duplicateaddercar_test.go b/pkg/storage/duplicateaddercar_test.go index 3061ef16..38e57627 100644 --- a/pkg/storage/duplicateaddercar_test.go +++ b/pkg/storage/duplicateaddercar_test.go @@ -19,7 +19,6 @@ import ( ) func TestDuplicateAdderCar(t *testing.T) { - setupStore := &testutil.CorrectedMemStore{ParentStore: &memstore.Store{ Bag: make(map[string][]byte), }} @@ -34,7 +33,7 @@ func TestDuplicateAdderCar(t *testing.T) { store := storage.NewDeferredStorageCar("") ctx := context.Background() - carWriter := storage.NewDuplicateAdderCarForStream(ctx, unixfsFileWithDups.Root, "", types.DagScopeAll, store, buf) + carWriter := storage.NewDuplicateAdderCarForStream(ctx, unixfsFileWithDups.Root, "", types.DagScopeAll, nil, store, buf) cachingTempStore := storage.NewCachingTempStore(carWriter.BlockWriteOpener(), store) // write the root block, containing sharding metadata diff --git a/pkg/types/request.go b/pkg/types/request.go index 1f931240..682d4f3e 100644 --- a/pkg/types/request.go +++ b/pkg/types/request.go @@ -109,7 +109,14 @@ type RetrievalRequest struct { // and writing and it is explicitly set to be trusted (i.e. it will not // check CIDs match bytes). If the storage is not truested, // request.LinkSystem.TrustedStore should be set to false after this call. -func NewRequestForPath(store ipldstorage.WritableStorage, cid cid.Cid, path string, dagScope DagScope) (RetrievalRequest, error) { +func NewRequestForPath( + store ipldstorage.WritableStorage, + cid cid.Cid, + path string, + dagScope DagScope, + byteRange *ByteRange, +) (RetrievalRequest, error) { + retrievalId, err := NewRetrievalID() if err != nil { return RetrievalRequest{}, err @@ -128,6 +135,7 @@ func NewRequestForPath(store ipldstorage.WritableStorage, cid cid.Cid, path stri Cid: cid, Path: path, Scope: dagScope, + Bytes: byteRange, LinkSystem: linkSystem, }, nil } @@ -197,11 +205,15 @@ func (r RetrievalRequest) GetUrlPath() (string, error) { if legacyScope == string(DagScopeEntity) { legacyScope = "file" } + byteRange := "" + if !r.Bytes.IsDefault() { + byteRange = "&entity-bytes=" + r.Bytes.String() + } path := r.Path if path != "" { path = "/" + path } - return fmt.Sprintf("%s?dag-scope=%s&car-scope=%s", path, scope, legacyScope), nil + return fmt.Sprintf("%s?dag-scope=%s&car-scope=%s%s", path, scope, legacyScope, byteRange), nil } // GetSupportedProtocols will safely return the supported protocols for a specific request. diff --git a/pkg/types/types.go b/pkg/types/types.go index 2fbba541..17ca949d 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -333,7 +333,7 @@ func ParseByteRange(s string) (ByteRange, error) { } parts := strings.Split(s, ":") if len(parts) != 2 { - return br, fmt.Errorf("invalid byte range: %s", s) + return br, fmt.Errorf("invalid entity-bytes: %s", s) } var err error br.From, err = strconv.ParseInt(parts[0], 10, 64) @@ -341,7 +341,7 @@ func ParseByteRange(s string) (ByteRange, error) { return br, err } if br.From < 0 { - return br, fmt.Errorf("invalid byte range: %s", s) + return br, fmt.Errorf("invalid entity-bytes: %s", s) } if parts[1] != "*" { to, err := strconv.ParseInt(parts[1], 10, 64) From 5d8d43c624eca895307a90c148b78eb2699ff430 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 30 Jun 2023 14:39:46 +1000 Subject: [PATCH 08/15] feat: add entity-bytes to fetch cli --- cmd/lassie/fetch.go | 24 ++++++++++++- cmd/lassie/fetch_test.go | 76 ++++++++++++++++++++++++++++++---------- 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/cmd/lassie/fetch.go b/cmd/lassie/fetch.go index 5b71caa4..b29f4103 100644 --- a/cmd/lassie/fetch.go +++ b/cmd/lassie/fetch.go @@ -46,6 +46,19 @@ var fetchFlags = []cli.Flag{ return nil }, }, + &cli.StringFlag{ + Name: "entity-bytes", + Usage: "describes the byte range to consider when selecting the blocks from a sharded file." + + " Valid values should be of the form from:to, where from and to are byte offsets and to may be '*'", + DefaultText: "defaults to the entire file, 0:*", + Action: func(cctx *cli.Context, v string) error { + if _, err := types.ParseByteRange(v); err != nil { + return fmt.Errorf("invalid entity-bytes parameter, must be of the form from:to," + + " where from and to are byte offsets and to may be '*'") + } + return nil + }, + }, FlagIPNIEndpoint, FlagEventRecorderAuth, FlagEventRecorderInstanceId, @@ -78,6 +91,7 @@ func fetchAction(cctx *cli.Context) error { dataWriter := cctx.App.Writer dagScope := cctx.String("dag-scope") + entityBytes := cctx.String("entity-bytes") tempDir := cctx.String("tempdir") progress := cctx.Bool("progress") @@ -111,6 +125,7 @@ func fetchAction(cctx *cli.Context) error { rootCid, path, dagScope, + entityBytes, tempDir, progress, outfile, @@ -188,6 +203,7 @@ type fetchRunFunc func( rootCid cid.Cid, path string, dagScope string, + entityBytes string, tempDir string, progress bool, outfile string, @@ -207,6 +223,7 @@ func defaultFetchRun( rootCid cid.Cid, path string, dagScope string, + entityBytes string, tempDir string, progress bool, outfile string, @@ -258,7 +275,12 @@ func defaultFetchRun( } }, false) - request, err := types.NewRequestForPath(carStore, rootCid, path, types.DagScope(dagScope)) + bytes, _ := types.ParseByteRange(entityBytes) + var br *types.ByteRange + if !bytes.IsDefault() { + br = &bytes + } + request, err := types.NewRequestForPath(carStore, rootCid, path, types.DagScope(dagScope), br) if err != nil { return err } diff --git a/cmd/lassie/fetch_test.go b/cmd/lassie/fetch_test.go index 2ca6f5aa..15ff0f85 100644 --- a/cmd/lassie/fetch_test.go +++ b/cmd/lassie/fetch_test.go @@ -28,11 +28,12 @@ func TestFetchCommandFlags(t *testing.T) { { name: "with default args", args: []string{"fetch", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"}, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { // fetch specific params require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", rootCid.String()) require.Equal(t, "", path) require.Equal(t, string(types.DagScopeAll), dagScope) + require.Empty(t, entityBytes) require.Equal(t, false, progress) require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4.car", outfile) @@ -70,7 +71,7 @@ func TestFetchCommandFlags(t *testing.T) { "fetch", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4/birb.mp4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, "/birb.mp4", path) return nil }, @@ -83,7 +84,7 @@ func TestFetchCommandFlags(t *testing.T) { "entity", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, string(types.DagScopeEntity), dagScope) return nil }, @@ -96,11 +97,50 @@ func TestFetchCommandFlags(t *testing.T) { "block", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, string(types.DagScopeBlock), dagScope) return nil }, }, + { + name: "with entity-bytes 0:*", + args: []string{ + "fetch", + "--entity-bytes", + "0:*", + "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", + }, + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { + require.Equal(t, "0:*", entityBytes) + return nil + }, + }, + { + name: "with entity-bytes 0:10", + args: []string{ + "fetch", + "--entity-bytes", + "0:10", + "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", + }, + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { + require.Equal(t, "0:10", entityBytes) + return nil + }, + }, + { + name: "with entity-bytes 1000:20000", + args: []string{ + "fetch", + "--entity-bytes", + "1000:20000", + "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", + }, + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { + require.Equal(t, "1000:20000", entityBytes) + return nil + }, + }, { name: "with progress", args: []string{ @@ -108,7 +148,7 @@ func TestFetchCommandFlags(t *testing.T) { "--progress", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, true, progress) return nil }, @@ -121,7 +161,7 @@ func TestFetchCommandFlags(t *testing.T) { "myfile", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, "myfile", outfile) return nil }, @@ -134,7 +174,7 @@ func TestFetchCommandFlags(t *testing.T) { "/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.IsType(t, &retriever.DirectCandidateFinder{}, lCfg.Finder, "finder should be a DirectCandidateFinder when providers are specified") return nil }, @@ -147,7 +187,7 @@ func TestFetchCommandFlags(t *testing.T) { "https://cid.contact", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.IsType(t, &indexerlookup.IndexerCandidateFinder{}, lCfg.Finder, "finder should be an IndexerCandidateFinder when providing an ipni endpoint") return nil }, @@ -170,7 +210,7 @@ func TestFetchCommandFlags(t *testing.T) { "/mytmpdir", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, "/mytmpdir", tempDir) return nil }, @@ -183,7 +223,7 @@ func TestFetchCommandFlags(t *testing.T) { "30s", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, 30*time.Second, lCfg.ProviderTimeout) return nil }, @@ -196,7 +236,7 @@ func TestFetchCommandFlags(t *testing.T) { "30s", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, 30*time.Second, lCfg.GlobalTimeout) return nil }, @@ -209,7 +249,7 @@ func TestFetchCommandFlags(t *testing.T) { "bitswap,graphsync", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, []multicodec.Code{multicodec.TransportBitswap, multicodec.TransportGraphsyncFilecoinv1}, lCfg.Protocols) return nil }, @@ -222,7 +262,7 @@ func TestFetchCommandFlags(t *testing.T) { "12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4,12D3KooWPNbkEgjdBNeaCGpsgCrPRETe4uBZf1ShFXStobdN18ys", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { p1, err := peer.Decode("12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4") require.NoError(t, err) p2, err := peer.Decode("12D3KooWPNbkEgjdBNeaCGpsgCrPRETe4uBZf1ShFXStobdN18ys") @@ -241,7 +281,7 @@ func TestFetchCommandFlags(t *testing.T) { "10", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, 10, lCfg.BitswapConcurrency) return nil }, @@ -254,7 +294,7 @@ func TestFetchCommandFlags(t *testing.T) { "https://myeventrecorder.com/v1/retrieval-events", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, "https://myeventrecorder.com/v1/retrieval-events", erCfg.EndpointURL) return nil }, @@ -267,7 +307,7 @@ func TestFetchCommandFlags(t *testing.T) { "secret", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, "secret", erCfg.EndpointAuthorization) return nil }, @@ -280,7 +320,7 @@ func TestFetchCommandFlags(t *testing.T) { "myinstanceid", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", }, - assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { + assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { require.Equal(t, "myinstanceid", erCfg.InstanceID) return nil }, @@ -312,6 +352,6 @@ func TestFetchCommandFlags(t *testing.T) { } } -func noopRun(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error { +func noopRun(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error { return nil } From 9af54dec11f2c103f5e4fec13b57eb9c6e7858b0 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 30 Jun 2023 15:15:08 +1000 Subject: [PATCH 09/15] chore: tests for etag w/ byte range --- pkg/types/request_test.go | 49 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/pkg/types/request_test.go b/pkg/types/request_test.go index 174cf7ae..554e6b3a 100644 --- a/pkg/types/request_test.go +++ b/pkg/types/request_test.go @@ -22,6 +22,7 @@ func TestEtag(t *testing.T) { cid cid.Cid path string scope types.DagScope + bytes *types.ByteRange dups bool expected string }{ @@ -115,14 +116,56 @@ func TestEtag(t *testing.T) { scope: types.DagScopeAll, expected: `"bafyrgqhai26anf3i7pips7q22coa4sz2fr4gk4q4sqdtymvvjyginfzaqewveaeqdh524nsktaq43j65v22xxrybrtertmcfxufdam3da3hbk.car.9lumqv26cg30t"`, }, + { + cid: cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), + scope: types.DagScopeAll, + bytes: &types.ByteRange{From: 0}, // default, not included + expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.58mf8vcmd2eo8"`, + }, + { + cid: cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), + scope: types.DagScopeAll, + bytes: &types.ByteRange{From: 10}, + expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.560ditjelh0u2"`, + }, + { + cid: cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), + scope: types.DagScopeAll, + bytes: &types.ByteRange{From: 0, To: ptr(200)}, + expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.faqf14andvfmb"`, + }, + { + cid: cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), + scope: types.DagScopeAll, + bytes: &types.ByteRange{From: 100, To: ptr(200)}, + expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.bvebrb14stt94"`, + }, + { + cid: cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), + scope: types.DagScopeEntity, + bytes: &types.ByteRange{From: 100, To: ptr(200)}, + expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.bq3u6t9t877t3"`, + }, + { + cid: cid.MustParse("QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), + scope: types.DagScopeEntity, + dups: true, + bytes: &types.ByteRange{From: 100, To: ptr(200)}, + expected: `"QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK.car.fhf498an52uqb"`, + }, } for _, tc := range testCases { - t.Run(fmt.Sprintf("%s:%s:%s:%v", tc.cid.String(), tc.path, tc.scope, tc.dups), func(t *testing.T) { + br := "" + if tc.bytes != nil { + br = ":" + tc.bytes.String() + } + t.Run(fmt.Sprintf("%s:%s:%s:%v%s", tc.cid.String(), tc.path, tc.scope, tc.dups, br), func(t *testing.T) { rr := types.RetrievalRequest{ Cid: tc.cid, Path: tc.path, Scope: tc.scope, + Bytes: tc.bytes, Duplicates: tc.dups, } actual := rr.Etag() @@ -132,3 +175,7 @@ func TestEtag(t *testing.T) { }) } } + +func ptr(i int64) *int64 { + return &i +} From 8e3ae79b2295028aada65a0e7fa15b2526d598b0 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 30 Jun 2023 15:23:49 +1000 Subject: [PATCH 10/15] feat: add bitswap and graphsync tests to trustless fixture suite --- pkg/internal/itest/trustless_fetch_test.go | 184 +++++++++++---------- 1 file changed, 97 insertions(+), 87 deletions(-) diff --git a/pkg/internal/itest/trustless_fetch_test.go b/pkg/internal/itest/trustless_fetch_test.go index 67502794..dd873ef4 100644 --- a/pkg/internal/itest/trustless_fetch_test.go +++ b/pkg/internal/itest/trustless_fetch_test.go @@ -1,5 +1,3 @@ -/* //go:build !race */ - package itest import ( @@ -39,101 +37,113 @@ func TestTrustlessUnixfsFetch(t *testing.T) { lsys.SetReadStorage(storage) for _, tc := range testCases { - t.Run(tc.Name, func(t *testing.T) { - req := require.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() + for _, proto := range []string{"http", "graphsync", "bitswap"} { + t.Run(tc.Name+"/"+proto, func(t *testing.T) { + req := require.New(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + t.Logf("query=%s, blocks=%d", tc.AsQuery(), len(tc.ExpectedCids)) - t.Logf("query=%s, blocks=%d", tc.AsQuery(), len(tc.ExpectedCids)) + mrn := mocknet.NewMockRetrievalNet(ctx, t) + switch proto { + case "http": + mrn.AddHttpPeers(1) + case "graphsync": + mrn.AddGraphsyncPeers(1) + case "bitswap": + mrn.AddBitswapPeers(1) + } + require.NoError(t, mrn.MN.LinkAll()) + var finishedChan chan []datatransfer.Event + if proto == "graphsync" { + finishedChan = mocknet.SetupRetrieval(t, mrn.Remotes[0]) + } - mrn := mocknet.NewMockRetrievalNet(ctx, t) - mrn.AddHttpPeers(1) - require.NoError(t, mrn.MN.LinkAll()) - mrn.Remotes[0].Blockstore().UseLinkSystem(lsys) - mrn.Remotes[0].Cids[tc.Root] = struct{}{} - var finishedChan chan []datatransfer.Event - // for graphsync: finishedChan := mocknet.SetupRetrieval(t, mrn.Remotes[0]) + mrn.Remotes[0].Blockstore().UseLinkSystem(lsys) + mrn.Remotes[0].Cids[tc.Root] = struct{}{} - lassie, err := lassie.NewLassie( - ctx, - lassie.WithProviderTimeout(20*time.Second), - lassie.WithHost(mrn.Self), - lassie.WithFinder(mrn.Finder), - ) - req.NoError(err) - cfg := httpserver.HttpServerConfig{Address: "127.0.0.1", Port: 0, TempDir: t.TempDir()} - httpServer, err := httpserver.NewHttpServer(ctx, lassie, cfg) - req.NoError(err) - serverError := make(chan error, 1) - go func() { - serverError <- httpServer.Start() - }() - responseChan := make(chan *http.Response, 1) - go func() { - // Make a request for our CID and read the complete CAR bytes - addr := fmt.Sprintf("http://%s%s", httpServer.Addr(), tc.AsQuery()) - getReq, err := http.NewRequest("GET", addr, nil) + lassie, err := lassie.NewLassie( + ctx, + lassie.WithProviderTimeout(20*time.Second), + lassie.WithHost(mrn.Self), + lassie.WithFinder(mrn.Finder), + ) req.NoError(err) - getReq.Header.Add("Accept", "application/vnd.ipld.car") - t.Log("Fetching", getReq.URL.String()) - resp, err := http.DefaultClient.Do(getReq) + cfg := httpserver.HttpServerConfig{Address: "127.0.0.1", Port: 0, TempDir: t.TempDir()} + httpServer, err := httpserver.NewHttpServer(ctx, lassie, cfg) req.NoError(err) - responseChan <- resp - }() - var resp *http.Response - select { - case resp = <-responseChan: - case <-ctx.Done(): - req.FailNow("Did not receive responses") - } - if finishedChan != nil { - // for graphsync - var wg sync.WaitGroup - wg.Add(1) + serverError := make(chan error, 1) go func() { - mocknet.WaitForFinish(ctx, t, finishedChan, 1*time.Second) - wg.Done() + serverError <- httpServer.Start() }() - wg.Wait() - } - if resp.StatusCode != http.StatusOK { - body, err := io.ReadAll(resp.Body) + responseChan := make(chan *http.Response, 1) + go func() { + // Make a request for our CID and read the complete CAR bytes + addr := fmt.Sprintf("http://%s%s", httpServer.Addr(), tc.AsQuery()) + getReq, err := http.NewRequest("GET", addr, nil) + req.NoError(err) + getReq.Header.Add("Accept", "application/vnd.ipld.car") + t.Log("Fetching", getReq.URL.String()) + resp, err := http.DefaultClient.Do(getReq) + req.NoError(err) + responseChan <- resp + }() + var resp *http.Response + select { + case resp = <-responseChan: + case <-ctx.Done(): + req.FailNow("Did not receive responses") + } + if finishedChan != nil { + // for graphsync + var wg sync.WaitGroup + wg.Add(1) + go func() { + mocknet.WaitForFinish(ctx, t, finishedChan, 1*time.Second) + wg.Done() + }() + wg.Wait() + } + if resp.StatusCode != http.StatusOK { + body, err := io.ReadAll(resp.Body) + req.NoError(err) + req.Failf("200 response code not received", "got code: %d, body: %s", resp.StatusCode, string(body)) + } + req.Equal(fmt.Sprintf(`attachment; filename="%s.car"`, tc.Root.String()), resp.Header.Get("Content-Disposition")) + req.Equal("none", resp.Header.Get("Accept-Ranges")) + req.Equal("public, max-age=29030400, immutable", resp.Header.Get("Cache-Control")) + req.Equal("application/vnd.ipld.car; version=1", resp.Header.Get("Content-Type")) + req.Equal("nosniff", resp.Header.Get("X-Content-Type-Options")) + etagStart := fmt.Sprintf(`"%s.car.`, tc.Root.String()) + etagGot := resp.Header.Get("ETag") + req.True(strings.HasPrefix(etagGot, etagStart), "ETag should start with [%s], got [%s]", etagStart, etagGot) + req.Equal(`"`, etagGot[len(etagGot)-1:], "ETag should end with a quote") + req.Equal(fmt.Sprintf("/ipfs/%s%s", tc.Root.String(), tc.Path), resp.Header.Get("X-Ipfs-Path")) + requestId := resp.Header.Get("X-Trace-Id") + require.NotEmpty(t, requestId) + _, err = uuid.Parse(requestId) req.NoError(err) - req.Failf("200 response code not received", "got code: %d, body: %s", resp.StatusCode, string(body)) - } - req.Equal(fmt.Sprintf(`attachment; filename="%s.car"`, tc.Root.String()), resp.Header.Get("Content-Disposition")) - req.Equal("none", resp.Header.Get("Accept-Ranges")) - req.Equal("public, max-age=29030400, immutable", resp.Header.Get("Cache-Control")) - req.Equal("application/vnd.ipld.car; version=1", resp.Header.Get("Content-Type")) - req.Equal("nosniff", resp.Header.Get("X-Content-Type-Options")) - etagStart := fmt.Sprintf(`"%s.car.`, tc.Root.String()) - etagGot := resp.Header.Get("ETag") - req.True(strings.HasPrefix(etagGot, etagStart), "ETag should start with [%s], got [%s]", etagStart, etagGot) - req.Equal(`"`, etagGot[len(etagGot)-1:], "ETag should end with a quote") - req.Equal(fmt.Sprintf("/ipfs/%s%s", tc.Root.String(), tc.Path), resp.Header.Get("X-Ipfs-Path")) - requestId := resp.Header.Get("X-Trace-Id") - require.NotEmpty(t, requestId) - _, err = uuid.Parse(requestId) - req.NoError(err) - rdr, err := car.NewBlockReader(resp.Body) - req.NoError(err) - req.Len(rdr.Roots, 1) - req.Equal(tc.Root.String(), rdr.Roots[0].String()) - for ii := 0; ; ii++ { - blk, err := rdr.Next() - if err == io.EOF { - if ii != len(tc.ExpectedCids) { - req.FailNowf("unexpected EOF", "expected %d blocks, got %d", len(tc.ExpectedCids), ii) - } - break - } + rdr, err := car.NewBlockReader(resp.Body) req.NoError(err) - if ii >= len(tc.ExpectedCids) { - req.FailNowf("unexpected block", "got block %d, expected %d", ii, len(tc.ExpectedCids)) + req.Len(rdr.Roots, 1) + req.Equal(tc.Root.String(), rdr.Roots[0].String()) + for ii := 0; ; ii++ { + blk, err := rdr.Next() + if err == io.EOF { + if ii != len(tc.ExpectedCids) { + req.FailNowf("unexpected EOF", "expected %d blocks, got %d", len(tc.ExpectedCids), ii) + } + break + } + req.NoError(err) + if ii >= len(tc.ExpectedCids) { + req.FailNowf("unexpected block", "got block %d, expected %d", ii, len(tc.ExpectedCids)) + } + req.Equal(tc.ExpectedCids[ii].String(), blk.Cid().String(), "unexpected block #%d", ii) } - req.Equal(tc.ExpectedCids[ii].String(), blk.Cid().String(), "unexpected block #%d", ii) - } - }) + }) + } } } From 0ef1667cb660f1a6268608f771b582862adab213 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 3 Jul 2023 12:29:05 +1000 Subject: [PATCH 11/15] fix: address feedback --- cmd/lassie/fetch.go | 6 ++-- pkg/internal/itest/testpeer/backedstore.go | 36 ++-------------------- pkg/internal/itest/testpeer/generator.go | 2 +- 3 files changed, 6 insertions(+), 38 deletions(-) diff --git a/cmd/lassie/fetch.go b/cmd/lassie/fetch.go index b29f4103..51184a77 100644 --- a/cmd/lassie/fetch.go +++ b/cmd/lassie/fetch.go @@ -275,10 +275,10 @@ func defaultFetchRun( } }, false) - bytes, _ := types.ParseByteRange(entityBytes) + byteRange, _ := types.ParseByteRange(entityBytes) var br *types.ByteRange - if !bytes.IsDefault() { - br = &bytes + if !byteRange.IsDefault() { + br = &byteRange } request, err := types.NewRequestForPath(carStore, rootCid, path, types.DagScope(dagScope), br) if err != nil { diff --git a/pkg/internal/itest/testpeer/backedstore.go b/pkg/internal/itest/testpeer/backedstore.go index e9491530..1b62863d 100644 --- a/pkg/internal/itest/testpeer/backedstore.go +++ b/pkg/internal/itest/testpeer/backedstore.go @@ -17,43 +17,11 @@ var _ blockstore.Blockstore = (*BackedStore)(nil) var _ blockstore.Blockstore = (*linkSystemBlockstore)(nil) type BackedStore struct { - Backing blockstore.Blockstore -} - -func (bs *BackedStore) DeleteBlock(ctx context.Context, c cid.Cid) error { - return bs.Backing.DeleteBlock(ctx, c) -} - -func (bs *BackedStore) Has(ctx context.Context, c cid.Cid) (bool, error) { - return bs.Backing.Has(ctx, c) -} - -func (bs *BackedStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { - return bs.Backing.Get(ctx, c) -} - -func (bs *BackedStore) GetSize(ctx context.Context, c cid.Cid) (int, error) { - return bs.Backing.GetSize(ctx, c) -} - -func (bs *BackedStore) Put(ctx context.Context, blk blocks.Block) error { - return bs.Backing.Put(ctx, blk) -} - -func (bs *BackedStore) PutMany(ctx context.Context, blks []blocks.Block) error { - return bs.Backing.PutMany(ctx, blks) -} - -func (bs *BackedStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { - return bs.Backing.AllKeysChan(ctx) -} - -func (bs *BackedStore) HashOnRead(enabled bool) { - bs.Backing.HashOnRead(enabled) + blockstore.Blockstore } func (bs *BackedStore) UseLinkSystem(lsys linking.LinkSystem) { - bs.Backing = &linkSystemBlockstore{lsys} + bs.Blockstore = &linkSystemBlockstore{lsys} } type linkSystemBlockstore struct { diff --git a/pkg/internal/itest/testpeer/generator.go b/pkg/internal/itest/testpeer/generator.go index bb683cf8..5a74bd88 100644 --- a/pkg/internal/itest/testpeer/generator.go +++ b/pkg/internal/itest/testpeer/generator.go @@ -286,7 +286,7 @@ func newTestPeer(ctx context.Context, mn mocknet.Mocknet, p tnet.Identity) (Test if err != nil { return TestPeer{}, nil, err } - backedStore := &BackedStore{Backing: bstore} + backedStore := &BackedStore{bstore} lsys := storeutil.LinkSystemForBlockstore(backedStore) tp := TestPeer{ Host: client, From 79f2092d418927ec534dd3a38d8b2dfab4ee07f8 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 5 Jul 2023 09:35:56 +1000 Subject: [PATCH 12/15] chore: extract test fixture data to ipld/ipld/specs repo --- go.mod | 3 +- go.sum | 2 + pkg/internal/fixtures/testcases.go | 94 --- pkg/internal/fixtures/unixfs_20m_variety.go | 76 -- pkg/internal/itest/trustless_fetch_test.go | 6 +- pkg/internal/testdata/unixfs_20m_variety.md | 884 -------------------- pkg/types/types.go | 3 + pkg/verifiedcar/verifiedcar_test.go | 18 +- 8 files changed, 23 insertions(+), 1063 deletions(-) delete mode 100644 pkg/internal/fixtures/testcases.go delete mode 100644 pkg/internal/fixtures/unixfs_20m_variety.go delete mode 100644 pkg/internal/testdata/unixfs_20m_variety.md diff --git a/go.mod b/go.mod index 943bbb68..fe6917a1 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/ipld/go-car/v2 v2.10.1 github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-prime v0.20.1-0.20230629094729-829a31766846 + github.com/ipld/ipld/specs v0.0.0-20230705075038-29da2e853cdb github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff github.com/libp2p/go-libp2p v0.27.1 github.com/libp2p/go-libp2p-routing-helpers v0.7.0 @@ -36,7 +37,6 @@ require ( github.com/multiformats/go-multihash v0.2.3 github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.24.4 - github.com/warpfork/go-testmark v0.12.1 go.opentelemetry.io/otel v1.14.0 go.opentelemetry.io/otel/trace v1.14.0 go.uber.org/multierr v1.11.0 @@ -145,6 +145,7 @@ require ( github.com/raulk/go-watchdog v1.3.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/warpfork/go-testmark v0.12.1 // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect diff --git a/go.sum b/go.sum index fcd84706..28778b3c 100644 --- a/go.sum +++ b/go.sum @@ -339,6 +339,8 @@ github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYt github.com/ipld/go-ipld-prime v0.20.1-0.20230629094729-829a31766846 h1:0gIWxUu4gU/Do25cwk82TRA621GjoZRSyVY472LaBrQ= github.com/ipld/go-ipld-prime v0.20.1-0.20230629094729-829a31766846/go.mod h1:PRQpXNcJypaPiiSdarsrJABPkYrBvafwDl0B9HjujZ8= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= +github.com/ipld/ipld/specs v0.0.0-20230705075038-29da2e853cdb h1:SDDUglxqma8Zl5BwGb8VPnkXXR48t7pcPnh9x8Mfp9I= +github.com/ipld/ipld/specs v0.0.0-20230705075038-29da2e853cdb/go.mod h1:eB2ZYKBGUJFuxTCO8YC0jJu41iquw6AvHhpP7N2yw8k= github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff h1:xbKrIvnpQkbF8iHPk/HGcegsypCDpcXWHhzBCLyCWf8= github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff/go.mod h1:paYP9U4N3/vOzGCuN9kU972vtvw9JUcQjOKyiCFGwRk= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= diff --git a/pkg/internal/fixtures/testcases.go b/pkg/internal/fixtures/testcases.go deleted file mode 100644 index 02e64fe3..00000000 --- a/pkg/internal/fixtures/testcases.go +++ /dev/null @@ -1,94 +0,0 @@ -package fixtures - -import ( - "errors" - "fmt" - "net/url" - "strings" - - "github.com/filecoin-project/lassie/pkg/types" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" -) - -type TestCase struct { - Name string - Root cid.Cid - Path string - Scope types.DagScope - Duplicates bool - ByteRange *types.ByteRange - ExpectedCids []cid.Cid -} - -func (tc TestCase) AsQuery() string { - pp := ipld.ParsePath(tc.Path).String() - if pp != "" { - pp = "/" + pp - } - br := "" - if !tc.ByteRange.IsDefault() { - br = fmt.Sprintf("&entity-bytes=%s", tc.ByteRange.String()) - } - dup := "" - if tc.Duplicates { - dup = "&dups=y" - } - return fmt.Sprintf("/ipfs/%s%s?dag-scope=%s%s%s", tc.Root, pp, tc.Scope, br, dup) -} - -func ParseCase(name, spec, exec string) (TestCase, error) { - lines := strings.Split(exec, "\n") - for len(lines) > 0 && strings.TrimSpace(lines[0]) == "" { - lines = lines[1:] - } - for len(lines) > 0 && strings.TrimSpace(lines[len(lines)-1]) == "" { - lines = lines[:len(lines)-1] - } - specParts := strings.Split(strings.TrimSpace(spec), "?") - if len(specParts) != 2 { - return TestCase{}, errors.New("invalid spec") - } - spec = specParts[0] - query, err := url.ParseQuery(specParts[1]) - if err != nil { - return TestCase{}, err - } - specParts = strings.Split(spec, "/") - if specParts[0] != "" && specParts[1] != "ipfs" { - return TestCase{}, errors.New("invalid spec") - } - root, err := cid.Parse(specParts[2]) - if err != nil { - return TestCase{}, err - } - path := "/" + ipld.ParsePath(strings.Join(specParts[3:], "/")).String() - scope, err := types.ParseDagScope(query.Get("dag-scope")) // required - if err != nil { - return TestCase{}, err - } - duplicates := query.Get("dups") == "y" - var byteRange *types.ByteRange - if query.Get("entity-bytes") != "" { - if br, err := types.ParseByteRange(query.Get("entity-bytes")); err != nil { - return TestCase{}, err - } else { - byteRange = &br - } - } - expectedCids := make([]cid.Cid, 0, len(lines)) - for _, line := range lines { - la := strings.Split(line, "|") - c := cid.MustParse(strings.TrimSpace(la[0])) - expectedCids = append(expectedCids, c) - } - return TestCase{ - Name: name, - Root: root, - Path: path, - Scope: scope, - Duplicates: duplicates, - ByteRange: byteRange, - ExpectedCids: expectedCids, - }, nil -} diff --git a/pkg/internal/fixtures/unixfs_20m_variety.go b/pkg/internal/fixtures/unixfs_20m_variety.go deleted file mode 100644 index ca078da6..00000000 --- a/pkg/internal/fixtures/unixfs_20m_variety.go +++ /dev/null @@ -1,76 +0,0 @@ -package fixtures - -import ( - "fmt" - "io" - "os" - "strings" - - "github.com/ipfs/go-cid" - carstorage "github.com/ipld/go-car/v2/storage" - "github.com/ipld/go-ipld-prime/storage" - "github.com/warpfork/go-testmark" -) - -var Unixfs20mVarietyRoot = cid.MustParse("bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq") - -const file = "internal/testdata/unixfs_20m_variety." - -func filepath(typ string) (string, error) { - wd, err := os.Getwd() - if err != nil { - return "", err - } - // convert wd to absolute path, normalizing to / separators - wd = strings.ReplaceAll(wd, "\\", "/") - rootInd := strings.LastIndex(wd, "/lassie/pkg/") - if rootInd == -1 { - return "", fmt.Errorf("could not find root of lassie package") - } - filename := wd[:rootInd] + "/lassie/pkg/" + file + typ - return filename, nil -} - -func Unixfs20mVarietyReadableStorage() (storage.ReadableStorage, io.Closer, error) { - file, err := filepath("car") - if err != nil { - return nil, nil, err - } - carFile, err := os.Open(file) - if err != nil { - return nil, nil, err - } - reader, err := carstorage.OpenReadable(carFile) - if err != nil { - carFile.Close() - return nil, nil, err - } - return reader, carFile, nil -} - -func Unixfs20mVarietyCases() ([]TestCase, error) { - file, err := filepath("md") - if err != nil { - return nil, err - } - doc, err := testmark.ReadFile(file) - if err != nil { - return nil, err - } - doc.BuildDirIndex() - testCases := make([]TestCase, 0) - for _, test := range doc.DirEnt.Children["test"].ChildrenList { - for _, scope := range test.ChildrenList { - tc, err := ParseCase(test.Name+"/"+scope.Name, dstr(scope, "query"), dstr(scope, "execution")) - if err != nil { - return nil, err - } - testCases = append(testCases, tc) - } - } - return testCases, nil -} - -func dstr(dir *testmark.DirEnt, ch string) string { - return string(dir.Children[ch].Hunk.Body) -} diff --git a/pkg/internal/itest/trustless_fetch_test.go b/pkg/internal/itest/trustless_fetch_test.go index dd873ef4..aa534a22 100644 --- a/pkg/internal/itest/trustless_fetch_test.go +++ b/pkg/internal/itest/trustless_fetch_test.go @@ -11,7 +11,6 @@ import ( "time" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/lassie/pkg/internal/fixtures" "github.com/filecoin-project/lassie/pkg/internal/itest/mocknet" "github.com/filecoin-project/lassie/pkg/lassie" httpserver "github.com/filecoin-project/lassie/pkg/server/http" @@ -19,15 +18,16 @@ import ( "github.com/ipfs/go-unixfsnode" "github.com/ipld/go-car/v2" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + trustlesspathing "github.com/ipld/ipld/specs/pkg-go/trustless-pathing" "github.com/stretchr/testify/require" ) func TestTrustlessUnixfsFetch(t *testing.T) { req := require.New(t) - testCases, err := fixtures.Unixfs20mVarietyCases() + testCases, err := trustlesspathing.Unixfs20mVarietyCases() req.NoError(err) - storage, closer, err := fixtures.Unixfs20mVarietyReadableStorage() + storage, closer, err := trustlesspathing.Unixfs20mVarietyReadableStorage() req.NoError(err) defer closer.Close() diff --git a/pkg/internal/testdata/unixfs_20m_variety.md b/pkg/internal/testdata/unixfs_20m_variety.md deleted file mode 100644 index aad131ed..00000000 --- a/pkg/internal/testdata/unixfs_20m_variety.md +++ /dev/null @@ -1,884 +0,0 @@ -# unixfs_20m_variety (test fixture) - - * 20 MB of files with a variety of UnixFS features across 1,103 blocks - * unixfs_20m_variety.car is a CARv1 in strict dfs order with a single root. - -## Root CID -[testmark]:# (root) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq -``` - -## Test Cases - -### Small file in directory - -Same result regardless of scope. - -#### all - -[testmark]:# (test/small_file_in_directory/all/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/pi?dag-scope=all -``` - -[testmark]:# (test/small_file_in_directory/all/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1700] (1701 B) -``` - -#### entity - -[testmark]:# (test/small_file_in_directory/entity/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/pi?dag-scope=entity -``` - -[testmark]:# (test/small_file_in_directory/entity/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1700] (1701 B) -``` - -#### block - -[testmark]:# (test/small_file_in_directory/block/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/pi?dag-scope=block -``` - -[testmark]:# (test/small_file_in_directory/block/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1700] (1701 B) -``` - -### Small file in directory in directory in directory - -Same result regardless of scope. - -#### all - -[testmark]:# (test/small_file_in_directory_in_directory_in_directory/all/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/by/ğħōšţ/Whatchamacallit.json?dag-scope=all -``` - -[testmark]:# (test/small_file_in_directory_in_directory_in_directory/all/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeigdgcs3nulbgu6vtnyesfgsaau2grbxzbxxdtpgolsfhtrpjx6mw4 | Directory | ↳ /by -bafybeidbjnqmuizharp3piroes43frsqip37qafovv4cjok667hbdy7vhq | Directory | ↳ /ğħōšţ -bafkreicgd36kzadnbwjhat2eyvcbvv63a36l5oufslvryixcymnd342oei | RawLeaf | ↳ /Whatchamacallit.json[0:1204] (1205 B) -``` - -#### entity - -[testmark]:# (test/small_file_in_directory_in_directory_in_directory/entity/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/by/ğħōšţ/Whatchamacallit.json?dag-scope=entity -``` - -[testmark]:# (test/small_file_in_directory_in_directory_in_directory/entity/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeigdgcs3nulbgu6vtnyesfgsaau2grbxzbxxdtpgolsfhtrpjx6mw4 | Directory | ↳ /by -bafybeidbjnqmuizharp3piroes43frsqip37qafovv4cjok667hbdy7vhq | Directory | ↳ /ğħōšţ -bafkreicgd36kzadnbwjhat2eyvcbvv63a36l5oufslvryixcymnd342oei | RawLeaf | ↳ /Whatchamacallit.json[0:1204] (1205 B) -``` - -#### block - -[testmark]:# (test/small_file_in_directory_in_directory_in_directory/block/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/by/ğħōšţ/Whatchamacallit.json?dag-scope=entity -``` - -[testmark]:# (test/small_file_in_directory_in_directory_in_directory/block/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeigdgcs3nulbgu6vtnyesfgsaau2grbxzbxxdtpgolsfhtrpjx6mw4 | Directory | ↳ /by -bafybeidbjnqmuizharp3piroes43frsqip37qafovv4cjok667hbdy7vhq | Directory | ↳ /ğħōšţ -bafkreicgd36kzadnbwjhat2eyvcbvv63a36l5oufslvryixcymnd342oei | RawLeaf | ↳ /Whatchamacallit.json[0:1204] (1205 B) -``` - -### Sharded file in directory - -All and entity are the same but block should just get the root File block - -#### all - -[testmark]:# (test/sharded_file_in_directory/all/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/želva.xml?dag-scope=all -``` - -[testmark]:# (test/sharded_file_in_directory/all/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352049] (1352050 B) -bafkreigtwsisgpg6x752y5md2z2r4jhuhohh46y4x3mvxrbkcubo7mqlgi | RawLeaf | ↳ /želva.xml[0:256143] (256144 B) -bafkreigahqispwg55yvqwobavwlheongcyhk63eufsaqutqgjiwfwesfau | RawLeaf | /želva.xml[256144:512287] (256144 B) -bafkreic4kgh44v2ung3wspd7y6wigcxake45ztpfx3c5ibfwmqe2kt7uay | RawLeaf | /želva.xml[512288:768431] (256144 B) -bafkreibdjsvoyftwgcywb3xylnfod2wififs2tj7pww4zkvaba7z5aigtm | RawLeaf | /želva.xml[768432:1024575] (256144 B) -bafkreidtv6frnbb4o4yobyu3xbtd5onzne67dhgngpd3vwrbdcneapy6fa | RawLeaf | /želva.xml[1024576:1280719] (256144 B) -bafkreidy2ntimx4u22b5uy6tjh7du5ex5lhgu7comxwcjfkzfbdsdunbou | RawLeaf | /želva.xml[1280720:1352049] (71330 B) -``` - -#### entity - -[testmark]:# (test/sharded_file_in_directory/entity/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/želva.xml?dag-scope=entity -``` - -[testmark]:# (test/sharded_file_in_directory/entity/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352049] (1352050 B) -bafkreigtwsisgpg6x752y5md2z2r4jhuhohh46y4x3mvxrbkcubo7mqlgi | RawLeaf | ↳ /želva.xml[0:256143] (256144 B) -bafkreigahqispwg55yvqwobavwlheongcyhk63eufsaqutqgjiwfwesfau | RawLeaf | /želva.xml[256144:512287] (256144 B) -bafkreic4kgh44v2ung3wspd7y6wigcxake45ztpfx3c5ibfwmqe2kt7uay | RawLeaf | /želva.xml[512288:768431] (256144 B) -bafkreibdjsvoyftwgcywb3xylnfod2wififs2tj7pww4zkvaba7z5aigtm | RawLeaf | /želva.xml[768432:1024575] (256144 B) -bafkreidtv6frnbb4o4yobyu3xbtd5onzne67dhgngpd3vwrbdcneapy6fa | RawLeaf | /želva.xml[1024576:1280719] (256144 B) -bafkreidy2ntimx4u22b5uy6tjh7du5ex5lhgu7comxwcjfkzfbdsdunbou | RawLeaf | /želva.xml[1280720:1352049] (71330 B) -``` - -#### block - -[testmark]:# (test/sharded_file_in_directory/block/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/želva.xml?dag-scope=block -``` - -[testmark]:# (test/sharded_file_in_directory/block/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352049] (1352050 B) -``` - -### Sharded file in directory in directory - -Aame as above but one extra level of nesting. - -#### all - -[testmark]:# (test/sharded_file_in_directory_in_directory/all/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Flibbertigibbet5/eorþscyld.pdf?dag-scope=all -``` - -[testmark]:# (test/sharded_file_in_directory_in_directory/all/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeicqlqnzd2tvs5vlo4c2dz72ryvouwmjwpu2mvxrocuj2u26joxeam | Directory | ↳ /Flibbertigibbet5 -bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558477] (4558478 B) -bafkreidz265issniggrhdjbwp3c5o4fnmg7xsa6hbn3kmyo7bsafc3jjsy | RawLeaf | ↳ /eorþscyld.pdf[0:256143] (256144 B) -bafkreiepissrnjsgcanpe6hzen5kv66glcr5x5vw5vhferkjscxdj2uv3y | RawLeaf | /eorþscyld.pdf[256144:512287] (256144 B) -bafkreifohr4rlzjpskbdcntrjzolglq6e5uf7n4mgdiw7wl2yhovhx46ve | RawLeaf | /eorþscyld.pdf[512288:768431] (256144 B) -bafkreihiulihaonenbbtrbptmcmzcg2hvk6dypdxvmposz7ansd6h5mkla | RawLeaf | /eorþscyld.pdf[768432:1024575] (256144 B) -bafkreidbhmgdt3ajzgoeywwxdbdcquone7buyenwhirl6af2z3gmftiys4 | RawLeaf | /eorþscyld.pdf[1024576:1280719] (256144 B) -bafkreib6iawulrjvzet7dxbqdqneqa7kpy2kacvq7fawjvmlgveoakuzzy | RawLeaf | /eorþscyld.pdf[1280720:1536863] (256144 B) -bafkreicbbl7whtfqsoz6tu663qicdisdyvzp4pp4ylggcttc44j63bdhxy | RawLeaf | /eorþscyld.pdf[1536864:1793007] (256144 B) -bafkreifxe2irzoxtzk3ltpunioehodovu4pwzdtjr2lqtmkg7hyh3o5r2e | RawLeaf | /eorþscyld.pdf[1793008:2049151] (256144 B) -bafkreihr2m4zfx4qvpwkp2fnihn7gbjq6gyr2gv5nbibt2nuajnp2k6pvq | RawLeaf | /eorþscyld.pdf[2049152:2305295] (256144 B) -bafkreibhyumtv62kh3d4rvwpxkdlr52uur5thgmbtyuyxas3c424od7cta | RawLeaf | /eorþscyld.pdf[2305296:2561439] (256144 B) -bafkreiatyvemaol2uxceo4suktpmusl53dqgg5bcqi2rkfhlxnnvpigkci | RawLeaf | /eorþscyld.pdf[2561440:2817583] (256144 B) -bafkreias6ygzx2pnowi3hxj5kmed2rm74hgigwijiandz7vamvrohtkpje | RawLeaf | /eorþscyld.pdf[2817584:3073727] (256144 B) -bafkreihf5g2wmsblx67664w2k3m7hj2bg2wnpumlnty7ssw4gj46oj3bxa | RawLeaf | /eorþscyld.pdf[3073728:3329871] (256144 B) -bafkreib7gr6yi6lhl2p5izuxrxpknf5tc5lhkf5tdhzaxw5g66jxqichra | RawLeaf | /eorþscyld.pdf[3329872:3586015] (256144 B) -bafkreigmnnfehab7c4tblwqgydbaoc76y34rkjfetayb6ytql2eywqnq5y | RawLeaf | /eorþscyld.pdf[3586016:3842159] (256144 B) -bafkreiax5uhfktinmfo3ovm7geaf2266u5ya2slq4bob6eeginffbewszu | RawLeaf | /eorþscyld.pdf[3842160:4098303] (256144 B) -bafkreic5chuwrtofd6ymxawcaxulacfewzfi3zroakmkwjkfmadrxrmln4 | RawLeaf | /eorþscyld.pdf[4098304:4354447] (256144 B) -bafkreich7pcb4lnheypmqb4sikk5m6cuqqbxbjzuk767cx47djxac45o64 | RawLeaf | /eorþscyld.pdf[4354448:4558477] (204030 B) -``` - -#### entity - -[testmark]:# (test/sharded_file_in_directory_in_directory/entity/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Flibbertigibbet5/eorþscyld.pdf?dag-scope=entity -``` - -[testmark]:# (test/sharded_file_in_directory_in_directory/entity/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeicqlqnzd2tvs5vlo4c2dz72ryvouwmjwpu2mvxrocuj2u26joxeam | Directory | ↳ /Flibbertigibbet5 -bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558477] (4558478 B) -bafkreidz265issniggrhdjbwp3c5o4fnmg7xsa6hbn3kmyo7bsafc3jjsy | RawLeaf | ↳ /eorþscyld.pdf[0:256143] (256144 B) -bafkreiepissrnjsgcanpe6hzen5kv66glcr5x5vw5vhferkjscxdj2uv3y | RawLeaf | /eorþscyld.pdf[256144:512287] (256144 B) -bafkreifohr4rlzjpskbdcntrjzolglq6e5uf7n4mgdiw7wl2yhovhx46ve | RawLeaf | /eorþscyld.pdf[512288:768431] (256144 B) -bafkreihiulihaonenbbtrbptmcmzcg2hvk6dypdxvmposz7ansd6h5mkla | RawLeaf | /eorþscyld.pdf[768432:1024575] (256144 B) -bafkreidbhmgdt3ajzgoeywwxdbdcquone7buyenwhirl6af2z3gmftiys4 | RawLeaf | /eorþscyld.pdf[1024576:1280719] (256144 B) -bafkreib6iawulrjvzet7dxbqdqneqa7kpy2kacvq7fawjvmlgveoakuzzy | RawLeaf | /eorþscyld.pdf[1280720:1536863] (256144 B) -bafkreicbbl7whtfqsoz6tu663qicdisdyvzp4pp4ylggcttc44j63bdhxy | RawLeaf | /eorþscyld.pdf[1536864:1793007] (256144 B) -bafkreifxe2irzoxtzk3ltpunioehodovu4pwzdtjr2lqtmkg7hyh3o5r2e | RawLeaf | /eorþscyld.pdf[1793008:2049151] (256144 B) -bafkreihr2m4zfx4qvpwkp2fnihn7gbjq6gyr2gv5nbibt2nuajnp2k6pvq | RawLeaf | /eorþscyld.pdf[2049152:2305295] (256144 B) -bafkreibhyumtv62kh3d4rvwpxkdlr52uur5thgmbtyuyxas3c424od7cta | RawLeaf | /eorþscyld.pdf[2305296:2561439] (256144 B) -bafkreiatyvemaol2uxceo4suktpmusl53dqgg5bcqi2rkfhlxnnvpigkci | RawLeaf | /eorþscyld.pdf[2561440:2817583] (256144 B) -bafkreias6ygzx2pnowi3hxj5kmed2rm74hgigwijiandz7vamvrohtkpje | RawLeaf | /eorþscyld.pdf[2817584:3073727] (256144 B) -bafkreihf5g2wmsblx67664w2k3m7hj2bg2wnpumlnty7ssw4gj46oj3bxa | RawLeaf | /eorþscyld.pdf[3073728:3329871] (256144 B) -bafkreib7gr6yi6lhl2p5izuxrxpknf5tc5lhkf5tdhzaxw5g66jxqichra | RawLeaf | /eorþscyld.pdf[3329872:3586015] (256144 B) -bafkreigmnnfehab7c4tblwqgydbaoc76y34rkjfetayb6ytql2eywqnq5y | RawLeaf | /eorþscyld.pdf[3586016:3842159] (256144 B) -bafkreiax5uhfktinmfo3ovm7geaf2266u5ya2slq4bob6eeginffbewszu | RawLeaf | /eorþscyld.pdf[3842160:4098303] (256144 B) -bafkreic5chuwrtofd6ymxawcaxulacfewzfi3zroakmkwjkfmadrxrmln4 | RawLeaf | /eorþscyld.pdf[4098304:4354447] (256144 B) -bafkreich7pcb4lnheypmqb4sikk5m6cuqqbxbjzuk767cx47djxac45o64 | RawLeaf | /eorþscyld.pdf[4354448:4558477] (204030 B) -``` - -#### block - -[testmark]:# (test/sharded_file_in_directory_in_directory/block/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Flibbertigibbet5/eorþscyld.pdf?dag-scope=block -``` - -[testmark]:# (test/sharded_file_in_directory_in_directory/block/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeicqlqnzd2tvs5vlo4c2dz72ryvouwmjwpu2mvxrocuj2u26joxeam | Directory | ↳ /Flibbertigibbet5 -bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558477] (4558478 B) -``` - -### Sharded file in hamt in directory - -Same as above but the inner directory is a HAMT and we have an intermediate block. - -#### all - -[testmark]:# (test/sharded_file_in_hamt_in_directory/all/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/O/Othello.png?dag-scope=all -``` - -[testmark]:# (test/sharded_file_in_hamt_in_directory/all/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiahh6giyfzxpsp6b4y6j6r34xjgsb2si7n6dszwoevnjxdcuee5yq | HAMTShard | ↳ /O -bafybeics7ohtet4wfuzszfchbttrgotqzpuruq4bencpdtc443vawbcbni | HAMTShard | ↳ -bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977730] (2977731 B) -bafkreibyblakon7atsv2hwzyzrweeb5uriksuscfxjtlzj3l56xmrouwke | RawLeaf | ↳ /Othello.png[0:256143] (256144 B) -bafkreic7nx6e7zezkdz5lp2xcwarmgavr62cyrmtepyagvm2ivuo3jesqe | RawLeaf | /Othello.png[256144:512287] (256144 B) -bafkreicylwcsi7ovreggex5yc7jolt5zrwf74ida3sprzsw37oxpzkwvte | RawLeaf | /Othello.png[512288:768431] (256144 B) -bafkreiainpsn6gadltlhqzgakxvdbxrz5n2cstizn3afhsr2nwefus7pqm | RawLeaf | /Othello.png[768432:1024575] (256144 B) -bafkreihuwjz4av7scz3fdjejtrx3k6ycdjudjusuz6sjzbbsewpgslosxi | RawLeaf | /Othello.png[1024576:1280719] (256144 B) -bafkreiexmzld3vi72v665bbqn473mfc5smmtd6nnndl7dzcapzlxdwzhq4 | RawLeaf | /Othello.png[1280720:1536863] (256144 B) -bafkreignp6prdhvi4mxoef5ybl4c644eyeikd42w3z2atw5mgbzw5p4374 | RawLeaf | /Othello.png[1536864:1793007] (256144 B) -bafkreia6nxplcal74ycukjwfmwmaaraeg6qwclfvb6qja2iaayukzsedoa | RawLeaf | /Othello.png[1793008:2049151] (256144 B) -bafkreiae6iay4fqiethqcvva4qazdxb7szb4dh77dsxiskvewhm5sepwfm | RawLeaf | /Othello.png[2049152:2305295] (256144 B) -bafkreieprkc6i3qzg47k5y3zxejsoglyl5fwrk3fnslbmdmau7surwiocy | RawLeaf | /Othello.png[2305296:2561439] (256144 B) -bafkreidwsenghahefqcfw5ikxd6osi6a35koyki4gibw3sunlmqrroqzc4 | RawLeaf | /Othello.png[2561440:2817583] (256144 B) -bafkreicn24jr4rwjcptuzu677u2aeqyirf7weradcafcnkq5hzo7wickkq | RawLeaf | /Othello.png[2817584:2977730] (160147 B) -``` - -#### entity - -[testmark]:# (test/sharded_file_in_hamt_in_directory/entity/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/O/Othello.png?dag-scope=entity -``` - -[testmark]:# (test/sharded_file_in_hamt_in_directory/entity/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiahh6giyfzxpsp6b4y6j6r34xjgsb2si7n6dszwoevnjxdcuee5yq | HAMTShard | ↳ /O -bafybeics7ohtet4wfuzszfchbttrgotqzpuruq4bencpdtc443vawbcbni | HAMTShard | ↳ -bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977730] (2977731 B) -bafkreibyblakon7atsv2hwzyzrweeb5uriksuscfxjtlzj3l56xmrouwke | RawLeaf | ↳ /Othello.png[0:256143] (256144 B) -bafkreic7nx6e7zezkdz5lp2xcwarmgavr62cyrmtepyagvm2ivuo3jesqe | RawLeaf | /Othello.png[256144:512287] (256144 B) -bafkreicylwcsi7ovreggex5yc7jolt5zrwf74ida3sprzsw37oxpzkwvte | RawLeaf | /Othello.png[512288:768431] (256144 B) -bafkreiainpsn6gadltlhqzgakxvdbxrz5n2cstizn3afhsr2nwefus7pqm | RawLeaf | /Othello.png[768432:1024575] (256144 B) -bafkreihuwjz4av7scz3fdjejtrx3k6ycdjudjusuz6sjzbbsewpgslosxi | RawLeaf | /Othello.png[1024576:1280719] (256144 B) -bafkreiexmzld3vi72v665bbqn473mfc5smmtd6nnndl7dzcapzlxdwzhq4 | RawLeaf | /Othello.png[1280720:1536863] (256144 B) -bafkreignp6prdhvi4mxoef5ybl4c644eyeikd42w3z2atw5mgbzw5p4374 | RawLeaf | /Othello.png[1536864:1793007] (256144 B) -bafkreia6nxplcal74ycukjwfmwmaaraeg6qwclfvb6qja2iaayukzsedoa | RawLeaf | /Othello.png[1793008:2049151] (256144 B) -bafkreiae6iay4fqiethqcvva4qazdxb7szb4dh77dsxiskvewhm5sepwfm | RawLeaf | /Othello.png[2049152:2305295] (256144 B) -bafkreieprkc6i3qzg47k5y3zxejsoglyl5fwrk3fnslbmdmau7surwiocy | RawLeaf | /Othello.png[2305296:2561439] (256144 B) -bafkreidwsenghahefqcfw5ikxd6osi6a35koyki4gibw3sunlmqrroqzc4 | RawLeaf | /Othello.png[2561440:2817583] (256144 B) -bafkreicn24jr4rwjcptuzu677u2aeqyirf7weradcafcnkq5hzo7wickkq | RawLeaf | /Othello.png[2817584:2977730] (160147 B) -``` - -#### block - -[testmark]:# (test/sharded_file_in_hamt_in_directory/block/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/O/Othello.png?dag-scope=block -``` - -[testmark]:# (test/sharded_file_in_hamt_in_directory/block/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiahh6giyfzxpsp6b4y6j6r34xjgsb2si7n6dszwoevnjxdcuee5yq | HAMTShard | ↳ /O -bafybeics7ohtet4wfuzszfchbttrgotqzpuruq4bencpdtc443vawbcbni | HAMTShard | ↳ -bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977730] (2977731 B) -``` - -### Sharded file in directory in hamt in directory - -Same as above but with an extra directory layer under the HAMT. - -#### all - -[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/all/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/Throttlebottom/supercalifragilisticexpialidocious.txt?dag-scope=all -``` - -[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/all/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious -bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | ↳ -bafybeifjkcls323ddq3t5ov22yl7i6ks36a7jzv33pjy4stqkils6jzvqe | Directory | ↳ /Throttlebottom -bafybeify427noacqiu6sxaaunk5uw2xhelnkcwktkvh6woi6ahlipsz7em | File | ↳ /supercalifragilisticexpialidocious.txt[0:568520] (568521 B) -bafkreicdwgfhxwnzq7i34cuhnqqqxz6d6jtirupjy5kq3uaphopyw5e2ky | RawLeaf | ↳ /supercalifragilisticexpialidocious.txt[0:256143] (256144 B) -bafkreifmk6qfl6ucap7btfu35rjd37bh7uuefavq6nsuungxjl3or7bz2e | RawLeaf | /supercalifragilisticexpialidocious.txt[256144:512287] (256144 B) -bafkreifk2vcldftxe57ml2cxyfcwb34ukkhaopm46kv3as5vo26ht63fci | RawLeaf | /supercalifragilisticexpialidocious.txt[512288:568520] (56233 B) -``` - -#### entity - -[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/entity/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/Throttlebottom?dag-scope=entity -``` - -[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/entity/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious -bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | ↳ -bafybeifjkcls323ddq3t5ov22yl7i6ks36a7jzv33pjy4stqkils6jzvqe | Directory | ↳ /Throttlebottom -``` - -#### block - -[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/block/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/Throttlebottom?dag-scope=block -``` - -[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/block/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious -bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | ↳ -bafybeifjkcls323ddq3t5ov22yl7i6ks36a7jzv33pjy4stqkils6jzvqe | Directory | ↳ /Throttlebottom -``` - -### Small file in a directory in a hamt in a directory - -Same result regardless of scope. - -#### all - -[testmark]:# (test/small_file_in_directory_in_hamt_in_directory/all/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/fyrd/Cordelia.docx?dag-scope=all -``` - -[testmark]:# (test/small_file_in_directory_in_hamt_in_directory/all/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious -bafybeiggzqpxmfbve7sutrmcwrlwpx4tkno536d7qdgd6hnhxf74t7gzee | HAMTShard | ↳ -bafybeigfwz54atwgnklb72rfqqn2a3ywoontgf3b3kekxwoxalptx7gc3e | HAMTShard | ↳ -bafybeid25clhrmlypqcsl4ehgxczbidzu2sqvrtuoeckh32hmzz7qisk6m | Directory | ↳ /fyrd -bafkreigtatgpa6dpkm2vxaeglfz4t3j2kkcan6jjxdf4ytwehspjbpxe54 | RawLeaf | ↳ /Cordelia.docx[0:329] (330 B) -``` - -#### entity - -[testmark]:# (test/small_file_in_directory_in_hamt_in_directory/entity/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/fyrd/Cordelia.docx?dag-scope=entity -``` - -[testmark]:# (test/small_file_in_directory_in_hamt_in_directory/entity/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious -bafybeiggzqpxmfbve7sutrmcwrlwpx4tkno536d7qdgd6hnhxf74t7gzee | HAMTShard | ↳ -bafybeigfwz54atwgnklb72rfqqn2a3ywoontgf3b3kekxwoxalptx7gc3e | HAMTShard | ↳ -bafybeid25clhrmlypqcsl4ehgxczbidzu2sqvrtuoeckh32hmzz7qisk6m | Directory | ↳ /fyrd -bafkreigtatgpa6dpkm2vxaeglfz4t3j2kkcan6jjxdf4ytwehspjbpxe54 | RawLeaf | ↳ /Cordelia.docx[0:329] (330 B) -``` - -#### block - -[testmark]:# (test/small_file_in_directory_in_hamt_in_directory/block/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/fyrd/Cordelia.docx?dag-scope=block -``` - -[testmark]:# (test/small_file_in_directory_in_hamt_in_directory/block/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious -bafybeiggzqpxmfbve7sutrmcwrlwpx4tkno536d7qdgd6hnhxf74t7gzee | HAMTShard | ↳ -bafybeigfwz54atwgnklb72rfqqn2a3ywoontgf3b3kekxwoxalptx7gc3e | HAMTShard | ↳ -bafybeid25clhrmlypqcsl4ehgxczbidzu2sqvrtuoeckh32hmzz7qisk6m | Directory | ↳ /fyrd -bafkreigtatgpa6dpkm2vxaeglfz4t3j2kkcan6jjxdf4ytwehspjbpxe54 | RawLeaf | ↳ /Cordelia.docx[0:329] (330 B) -``` - -### Hamt in directory - -"All" is too large to be useful, so we'll just do "entity" and "block". - -#### entity - -[testmark]:# (test/hamt_in_directory/entity/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious?dag-scope=entity -``` - -[testmark]:# (test/hamt_in_directory/entity/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious -bafybeiggzqpxmfbve7sutrmcwrlwpx4tkno536d7qdgd6hnhxf74t7gzee | HAMTShard | ↳ /Zigzagumptious -bafybeigfwz54atwgnklb72rfqqn2a3ywoontgf3b3kekxwoxalptx7gc3e | HAMTShard | ↳ /Zigzagumptious -bafybeifr3w3eyq2oi6hxwbfoblor7s5pfvu3m4xyyzovxlk4g5xgslsqui | HAMTShard | /Zigzagumptious -bafybeigivdbrxmcixaqfap3rrs7ti5ycr7vj4xfbkjougq67k7ac5oz55a | HAMTShard | /Zigzagumptious -bafybeid3cny2p7fwt4znzm3wjncsesdk2rdze5fm5rzxjpbethy4657j6m | HAMTShard | /Zigzagumptious -bafybeidivwpxv37o6p4ton3xcpmg64pwk4zqlkuge7isafgsgjsvb6ce2u | HAMTShard | /Zigzagumptious -bafybeib5ptplpmtmoyif4lxt5erw5diivp6so4s57lvgfigs53d7zpc4uu | HAMTShard | /Zigzagumptious -bafybeiafmairlfkvgqgticndcs4uomkll4h5xotvnnsw2j7cx3rjwrbh7i | HAMTShard | ↳ /Zigzagumptious -bafybeifjo6xz3onvav6cnpjjuroa7vbzegoxgg5jn6smfvgfcpzwzdaipa | HAMTShard | /Zigzagumptious -bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | /Zigzagumptious -bafybeiar4b5hfriv2pmmbj7npemto6bev7sqkn4jpstwqtaudexpetpg4m | HAMTShard | /Zigzagumptious -bafybeifcekpohkm7qaczfwrrzcdz6j3ykwntawdnp7b7fjfech75vxpwf4 | HAMTShard | ↳ /Zigzagumptious -bafybeihnejxifhvlunhcswhghhrbues366h7o4xqp3ziyeicrdt5473za4 | HAMTShard | /Zigzagumptious -bafybeihlijmc2xyfmomw4fx53mlz3gpt6svgfu3vfokm4wmyd4dagfup24 | HAMTShard | /Zigzagumptious -bafybeihjbh7kptdfaoanrkzhdpx2jwjacg5ncdlcbbwqlum6vwqfga5fii | HAMTShard | /Zigzagumptious -bafybeidaz5aob6tdurpnuj2xeqsjkarqudtono6x2kgbzrzv46ugp3g4ju | HAMTShard | /Zigzagumptious -``` - -#### block - -[testmark]:# (test/hamt_in_directory/block/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious?dag-scope=block -``` - -[testmark]:# (test/hamt_in_directory/block/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious -``` - -### Directory in a directory - -#### all - -[testmark]:# (test/directory_in_directory/all/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/by/procrastinatorily?dag-scope=all -``` - -[testmark]:# (test/directory_in_directory/all/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeigdgcs3nulbgu6vtnyesfgsaau2grbxzbxxdtpgolsfhtrpjx6mw4 | Directory | ↳ /by -bafybeicsa4ltuq6m3sxzzrkfy6wetemiqvbdgnitkjwlcduswh3cvsccxy | Directory | ↳ /procrastinatorily -bafkreiaquanb6uzch2bkv7iltvwipmawjfihb7jgejqww63vgbqshderxq | RawLeaf | ↳ * (/F[0:727] (728 B)) -bafkreigtze43gghvasxpp3o6mklllhilbyzs63kzdtdlzpyaznk44vxnye | RawLeaf | * (/Gallivant.png[0:1339] (1340 B)) -bafkreifrkcg75acl2yu4jv65x5u7yzn2cnpezmx432p2v2lryzhwqyp63u | RawLeaf | * (/Oberon.xml[0:512] (513 B)) -bafkreiac4vqpljai2qjxx7mgs2equqcs4oq6lmmie2oiqhrkmwukuez3cm | RawLeaf | * (/Pandemonium[0:917] (918 B)) -bafkreiclovx7khthefexgfku4n7pndbqthj5xuiv6ymbd3cx2bve3qebby | RawLeaf | * (/gūþweard.docx[0:580] (581 B)) -bafkreiaqchjvbofqnpwoxwrtcvelnz5neg4y5buqchwgnxwoigjofmnol4 | RawLeaf | * (/juxtapositionally.png[0:2426] (2427 B)) -bafkreif3y7x7uailqhjjuketaa2p34gidoj7ynmnmviejtynydov75dkbq | RawLeaf | * (/sǣlācend[0:430] (431 B)) -bafkreias4xj2qbg36tyo4wirl2db2x2emh4lbfpwq7t3dexjos236a5wgu | RawLeaf | * (/t.png[0:57] (58 B)) -bafkreidzryf2cxeo5bfkxulhxlgr6axh5qfrsh4ns6jwfd2yutzpkcoaxu | RawLeaf | * (/ya.xml[0:417] (418 B)) -bafkreihys4i2ow3csfrhefnmwruwte2bv5ik223ocgbvq3ckwj7xdxzc2m | RawLeaf | * (/ætheling.docx[0:81] (82 B)) -``` - -#### entity - -[testmark]:# (test/directory_in_directory/entity/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/by/procrastinatorily?dag-scope=entity -``` - -[testmark]:# (test/directory_in_directory/entity/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeigdgcs3nulbgu6vtnyesfgsaau2grbxzbxxdtpgolsfhtrpjx6mw4 | Directory | ↳ /by -bafybeicsa4ltuq6m3sxzzrkfy6wetemiqvbdgnitkjwlcduswh3cvsccxy | Directory | ↳ /procrastinatorily -``` - -#### block - -[testmark]:# (test/directory_in_directory/block/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/by/procrastinatorily?dag-scope=block -``` - -[testmark]:# (test/directory_in_directory/block/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeigdgcs3nulbgu6vtnyesfgsaau2grbxzbxxdtpgolsfhtrpjx6mw4 | Directory | ↳ /by -bafybeicsa4ltuq6m3sxzzrkfy6wetemiqvbdgnitkjwlcduswh3cvsccxy | Directory | ↳ /procrastinatorily -``` - -### File in directory, byte ranges - -#### 0:* - -[testmark]:# (test/file_in_directory_byte_ranges/0:*/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:* -``` - -[testmark]:# (test/file_in_directory_byte_ranges/0:*/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) -bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) -bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) -bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) -bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) -bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) -bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) -``` - -#### 0:10 - -Only needs first block. - -[testmark]:# (test/file_in_directory_byte_ranges/0:10/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:10 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/0:10/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) -``` - -#### 0:256143 first block only - -Matches full byte range of first block only. - -[testmark]:# (test/file_in_directory_byte_ranges/0:256143/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:256143 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/0:256143/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) -``` - - -#### 0:256144 second block boundary - -Needs the first byte of the second block, so matches first two blocks. - -[testmark]:# (test/file_in_directory_byte_ranges/0:256144/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:256144 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/0:256144/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) -``` - -#### 0:1793007 second last block - -Needs all but the last block to match. - -[testmark]:# (test/file_in_directory_byte_ranges/0:1793007/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:1793007 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/0:1793007/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) -bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) -bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) -bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) -bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) -bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) -``` - -#### 0:1793008 last block boundary - -Should match all blocks because it includes the first byte of the last block. - -[testmark]:# (test/file_in_directory_byte_ranges/0:1793008/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:1793008 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/0:1793008/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) -bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) -bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) -bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) -bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) -bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) -bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) -``` - -#### 0:1889755 full file - -Precise byte boundaries start:finish. - -[testmark]:# (test/file_in_directory_byte_ranges/0:1889755/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:1889755 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/0:1889755/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) -bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) -bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) -bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) -bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) -bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) -bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) -``` - -#### 0:1889756 full file +1 - -Beyond the end of the file, so should be the same as 0:1889755. - -[testmark]:# (test/file_in_directory_byte_ranges/0:1889756/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:1889756 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/0:1889756/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) -bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) -bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) -bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) -bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) -bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) -bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) -``` - -#### 0:99999999 - -Beyond last byte, should match all blocks. - -[testmark]:# (test/file_in_directory_byte_ranges/0:99999999/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=0:99999999 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/0:99999999/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) -bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) -bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) -bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) -bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) -bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) -bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) -``` - -#### 1793008:* last block - -Starts at the first byte of the last block. - -[testmark]:# (test/file_in_directory_byte_ranges/1793008:*/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=1793008:* -``` - -[testmark]:# (test/file_in_directory_byte_ranges/1793008:*/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | ↳ /clippet.txt[1793008:1889755] (96748 B) -``` - -#### 1793007:* second last block boundary - -Starts at the last byte of the second last block. - -[testmark]:# (test/file_in_directory_byte_ranges/1793007:*/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=1793007:* -``` - -[testmark]:# (test/file_in_directory_byte_ranges/1793007:*/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | ↳ /clippet.txt[1536864:1793007] (256144 B) -bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) -``` - -#### 256144:* all but first block - -Starts at the first byte of the second block. - -[testmark]:# (test/file_in_directory_byte_ranges/256144:*/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=256144:* -``` - -[testmark]:# (test/file_in_directory_byte_ranges/256144:*/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | ↳ /clippet.txt[256144:512287] (256144 B) -bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) -bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) -bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) -bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) -bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) -bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) -``` - -#### 256143:* first block boundary - -Need all blocks to match this. - -[testmark]:# (test/file_in_directory_byte_ranges/256143:*/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=256143:* -``` - -[testmark]:# (test/file_in_directory_byte_ranges/256143:*/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) -bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) -bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) -bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) -bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) -bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) -bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) -``` - -#### 256143:256144 first and second boundary - -[testmark]:# (test/file_in_directory_byte_ranges/256143:256144/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=256143:256144 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/256143:256144/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) -``` - -#### 256144:256144 second only - -Matching a single byte at the start of the second block - -[testmark]:# (test/file_in_directory_byte_ranges/256144:256144/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=256144:256144 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/256144:256144/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | ↳ /clippet.txt[256144:512287] (256144 B) -``` - -#### 512287:512287 second only - -Matching a single byte at the end of the second block - -[testmark]:# (test/file_in_directory_byte_ranges/512287:512287/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=512287:512287 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/512287:512287/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | ↳ /clippet.txt[256144:512287] (256144 B) -``` - -#### 256143:1793008 full file, boundaries - -Matching this needs all blocks, including one byte from first and last - -[testmark]:# (test/file_in_directory_byte_ranges/256143:1793008/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=256143:1793008 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/256143:1793008/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreib56a35cuh7mavlzlradaysqqpqx6lr2sn3veahyb2yb3y7dq5aei | RawLeaf | ↳ /clippet.txt[0:256143] (256144 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | /clippet.txt[256144:512287] (256144 B) -bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) -bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) -bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) -bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) -bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) -bafkreif6cdswphju3fd5onyiuoswxpstlskdydqh4s3rvgwvdjmx7zhpcq | RawLeaf | /clippet.txt[1793008:1889755] (96748 B) -``` - -#### 256144:1793007 inner blocks, boundaries - -Matching this only requires the middle blocks, excluding the first and last - -[testmark]:# (test/file_in_directory_byte_ranges/256144:1793007/query) -``` -/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/clippet.txt?dag-scope=entity&entity-bytes=256144:1793007 -``` - -[testmark]:# (test/file_in_directory_byte_ranges/256144:1793007/execution) -``` -bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / -bafybeiawxstujkmnv2doejkzfydtotgqreycncd7ro4ixxddeoahofwjkm | File | ↳ /clippet.txt[0:1889755] (1889756 B) -bafkreicjp7lobawh3bob4gjq63fmsz44sbsayf2xkwhf3jhme25y2tyzky | RawLeaf | ↳ /clippet.txt[256144:512287] (256144 B) -bafkreigcn244ayqteak6lignqrxdomc456362t4uysw624ccy7yk3ysfey | RawLeaf | /clippet.txt[512288:768431] (256144 B) -bafkreihbp4ur2mdgfqntlyvew7njve5pdjgtphr74ipmbebu7fofcn4fh4 | RawLeaf | /clippet.txt[768432:1024575] (256144 B) -bafkreibggbasrddspulesc5zum23cx3fqdknvzi6dttlrjsiga7qanakdu | RawLeaf | /clippet.txt[1024576:1280719] (256144 B) -bafkreibzp426cl6b3udplywqrmfpdm6witikihya6n5rfh724lwnn53oe4 | RawLeaf | /clippet.txt[1280720:1536863] (256144 B) -bafkreic47cecyo4ko65fdmpfhaonqz2kk73sd6zacnvsdjofee62u7d3oe | RawLeaf | /clippet.txt[1536864:1793007] (256144 B) -``` - -## TODO: - -* Negative ranges, To and From -* Out-of-bounds conditions - return full entity? - diff --git a/pkg/types/types.go b/pkg/types/types.go index 17ca949d..55b6057d 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -348,6 +348,9 @@ func ParseByteRange(s string) (ByteRange, error) { if err != nil { return br, err } + if to < 0 { + return br, fmt.Errorf("invalid entity-bytes: %s", s) + } br.To = &to } return br, nil diff --git a/pkg/verifiedcar/verifiedcar_test.go b/pkg/verifiedcar/verifiedcar_test.go index a3671edb..52d52656 100644 --- a/pkg/verifiedcar/verifiedcar_test.go +++ b/pkg/verifiedcar/verifiedcar_test.go @@ -12,7 +12,6 @@ import ( "testing" "time" - "github.com/filecoin-project/lassie/pkg/internal/fixtures" "github.com/filecoin-project/lassie/pkg/internal/testutil" "github.com/filecoin-project/lassie/pkg/types" "github.com/filecoin-project/lassie/pkg/verifiedcar" @@ -34,15 +33,16 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" + trustlesspathing "github.com/ipld/ipld/specs/pkg-go/trustless-pathing" "github.com/stretchr/testify/require" ) func TestUnixfs20mVariety(t *testing.T) { req := require.New(t) - testCases, err := fixtures.Unixfs20mVarietyCases() + testCases, err := trustlesspathing.Unixfs20mVarietyCases() req.NoError(err) - storage, closer, err := fixtures.Unixfs20mVarietyReadableStorage() + storage, closer, err := trustlesspathing.Unixfs20mVarietyReadableStorage() req.NoError(err) defer closer.Close() @@ -92,13 +92,21 @@ func TestUnixfs20mVariety(t *testing.T) { // Run the verifier over the CAR stream to see if we end up with // the same query. + scope, err := types.ParseDagScope(tc.Scope) + req.NoError(err) + var byteRange *types.ByteRange + if tc.ByteRange != "" { + br, err := types.ParseByteRange(tc.ByteRange) + req.NoError(err) + byteRange = &br + } cfg := verifiedcar.Config{ Root: tc.Root, - Selector: types.PathScopeSelector(tc.Path, tc.Scope, tc.ByteRange), + Selector: types.PathScopeSelector(tc.Path, scope, byteRange), } { selBytes, _ := ipld.Encode(cfg.Selector, dagjson.Encode) - t.Logf("selector=%s, entity-bytes=%s", string(selBytes), tc.ByteRange.String()) + t.Logf("selector=%s, entity-bytes=%s", string(selBytes), tc.ByteRange) } blockCount, byteCount, err := cfg.VerifyCar(ctx, carStream, lsys) From 711168abffe58d288d8d5bbdc3e231b84007fc7c Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 5 Jul 2023 18:30:13 +1000 Subject: [PATCH 13/15] fix: restore old car-scope parsing --- pkg/server/http/util.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/server/http/util.go b/pkg/server/http/util.go index 7c5101c7..537b1823 100644 --- a/pkg/server/http/util.go +++ b/pkg/server/http/util.go @@ -18,7 +18,16 @@ func ParseScope(req *http.Request) (types.DagScope, error) { } // check for legacy param name -- to do -- delete once we confirm this isn't used any more if req.URL.Query().Has("car-scope") { - return types.ParseDagScope(req.URL.Query().Get("car-scope")) + switch req.URL.Query().Get("car-scope") { + case "all": + return types.DagScopeAll, nil + case "file": + return types.DagScopeEntity, nil + case "block": + return types.DagScopeBlock, nil + default: + return types.DagScopeAll, errors.New("invalid car-scope parameter") + } } return types.DagScopeAll, nil } From 2f21ba964715c9fc68ec654dcc8d8bfc3b5021b2 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 7 Jul 2023 19:10:43 +1000 Subject: [PATCH 14/15] fix: update dependencies --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/types/types.go | 2 +- pkg/verifiedcar/verifiedcar_test.go | 8 +++----- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index fe6917a1..d21561ad 100644 --- a/go.mod +++ b/go.mod @@ -22,10 +22,10 @@ require ( github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipfs/go-unixfsnode v1.7.2-0.20230622024455-de5cbf74cbe0 + github.com/ipfs/go-unixfsnode v1.7.2-0.20230707090412-166be19558d9 github.com/ipld/go-car/v2 v2.10.1 github.com/ipld/go-codec-dagpb v1.6.0 - github.com/ipld/go-ipld-prime v0.20.1-0.20230629094729-829a31766846 + github.com/ipld/go-ipld-prime v0.20.1-0.20230707090759-349deb22a1fd github.com/ipld/ipld/specs v0.0.0-20230705075038-29da2e853cdb github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff github.com/libp2p/go-libp2p v0.27.1 diff --git a/go.sum b/go.sum index 28778b3c..410de784 100644 --- a/go.sum +++ b/go.sum @@ -329,15 +329,15 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= -github.com/ipfs/go-unixfsnode v1.7.2-0.20230622024455-de5cbf74cbe0 h1:XTKKTDz0rbJ53ff/7borHTvN64zJK74wZDvrYfdN0Fg= -github.com/ipfs/go-unixfsnode v1.7.2-0.20230622024455-de5cbf74cbe0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/go-unixfsnode v1.7.2-0.20230707090412-166be19558d9 h1:GXlyCGBZ235qjUd4HIP0D/qsBQxx2KWvugopoIuqeCk= +github.com/ipfs/go-unixfsnode v1.7.2-0.20230707090412-166be19558d9/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipld/go-car/v2 v2.10.1 h1:MRDqkONNW9WRhB79u+Z3U5b+NoN7lYA5B8n8qI3+BoI= github.com/ipld/go-car/v2 v2.10.1/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= -github.com/ipld/go-ipld-prime v0.20.1-0.20230629094729-829a31766846 h1:0gIWxUu4gU/Do25cwk82TRA621GjoZRSyVY472LaBrQ= -github.com/ipld/go-ipld-prime v0.20.1-0.20230629094729-829a31766846/go.mod h1:PRQpXNcJypaPiiSdarsrJABPkYrBvafwDl0B9HjujZ8= +github.com/ipld/go-ipld-prime v0.20.1-0.20230707090759-349deb22a1fd h1:/MI1u/0eL1gW3lFxMOs1sX/+bSCQgjQbK+yTJkvXMrc= +github.com/ipld/go-ipld-prime v0.20.1-0.20230707090759-349deb22a1fd/go.mod h1:PRQpXNcJypaPiiSdarsrJABPkYrBvafwDl0B9HjujZ8= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipld/ipld/specs v0.0.0-20230705075038-29da2e853cdb h1:SDDUglxqma8Zl5BwGb8VPnkXXR48t7pcPnh9x8Mfp9I= github.com/ipld/ipld/specs v0.0.0-20230705075038-29da2e853cdb/go.mod h1:eB2ZYKBGUJFuxTCO8YC0jJu41iquw6AvHhpP7N2yw8k= diff --git a/pkg/types/types.go b/pkg/types/types.go index 55b6057d..fb8dd3bd 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -273,7 +273,7 @@ func (ds DagScope) TerminalSelectorSpec() builder.SelectorSpec { case DagScopeAll: return unixfsnode.ExploreAllRecursivelySelector case DagScopeEntity: - return unixfsnode.MatchUnixFSPreloadSelector // file + return unixfsnode.MatchUnixFSEntitySelector case DagScopeBlock: return matcherSelector case DagScope(""): diff --git a/pkg/verifiedcar/verifiedcar_test.go b/pkg/verifiedcar/verifiedcar_test.go index 52d52656..a48660a6 100644 --- a/pkg/verifiedcar/verifiedcar_test.go +++ b/pkg/verifiedcar/verifiedcar_test.go @@ -8,7 +8,6 @@ import ( "io" "math/rand" "os" - "runtime/debug" "testing" "time" @@ -86,7 +85,6 @@ func TestUnixfs20mVariety(t *testing.T) { }, nil } lsys.StorageReadOpener = func(lc linking.LinkContext, l datamodel.Link) (io.Reader, error) { - debug.PrintStack() return nil, fmt.Errorf("unexpected read of %s", l.String()) } @@ -191,15 +189,15 @@ func TestVerifiedCar(t *testing.T) { }) unixfsShardedDirBlocks := testutil.ToBlocks(t, lsys, unixfsShardedDir.Root, allSelector) - unixfsPreloadSelector := unixfsnode.MatchUnixFSPreloadSelector.Node() + unixfsPreloadSelector := unixfsnode.MatchUnixFSEntitySelector.Node() unixfsPreloadDirBlocks := testutil.ToBlocks(t, lsys, unixfsDir.Root, unixfsPreloadSelector) unixfsPreloadShardedDirBlocks := testutil.ToBlocks(t, lsys, unixfsShardedDir.Root, unixfsPreloadSelector) - unixfsDirSubsetSelector := unixfsnode.UnixFSPathSelectorBuilder(unixfsDir.Children[1].Path, unixfsnode.MatchUnixFSPreloadSelector, false) + unixfsDirSubsetSelector := unixfsnode.UnixFSPathSelectorBuilder(unixfsDir.Children[1].Path, unixfsnode.MatchUnixFSEntitySelector, false) unixfsWrappedPathSelector := unixfsnode.UnixFSPathSelectorBuilder(wrapPath, unixfsnode.ExploreAllRecursivelySelector, false) - unixfsWrappedPreloadPathSelector := unixfsnode.UnixFSPathSelectorBuilder(wrapPath, unixfsnode.MatchUnixFSPreloadSelector, false) + unixfsWrappedPreloadPathSelector := unixfsnode.UnixFSPathSelectorBuilder(wrapPath, unixfsnode.MatchUnixFSEntitySelector, false) preloadSubst := ssb.ExploreInterpretAs("unixfs", ssb.ExploreRecursive( selector.RecursionLimitDepth(1), ssb.ExploreAll(ssb.ExploreRecursiveEdge()), From 4b3e0fbab3aef6ec38e66f3a7352d8d9061b1c10 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 7 Jul 2023 16:40:31 +0200 Subject: [PATCH 15/15] chore(mod): update deps to tagged --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d21561ad..dd5d30c4 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipfs/go-unixfsnode v1.7.2-0.20230707090412-166be19558d9 + github.com/ipfs/go-unixfsnode v1.7.2 github.com/ipld/go-car/v2 v2.10.1 github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-prime v0.20.1-0.20230707090759-349deb22a1fd diff --git a/go.sum b/go.sum index 410de784..0b808ac8 100644 --- a/go.sum +++ b/go.sum @@ -329,8 +329,8 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= -github.com/ipfs/go-unixfsnode v1.7.2-0.20230707090412-166be19558d9 h1:GXlyCGBZ235qjUd4HIP0D/qsBQxx2KWvugopoIuqeCk= -github.com/ipfs/go-unixfsnode v1.7.2-0.20230707090412-166be19558d9/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/go-unixfsnode v1.7.2 h1:460jNXtoBO7AJ5RnrNBVY9/ytZwUZOviDhcRxuLpAvA= +github.com/ipfs/go-unixfsnode v1.7.2/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipld/go-car/v2 v2.10.1 h1:MRDqkONNW9WRhB79u+Z3U5b+NoN7lYA5B8n8qI3+BoI= github.com/ipld/go-car/v2 v2.10.1/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg=