feat: download tars from @helia/verified-fetch

packages/verified-fetch/package.json

@@ -141,8 +141,8 @@
     "release": "aegir release"
   },
   "dependencies": {
-    "@helia/car": "^3.0.0",
     "@helia/block-brokers": "^2.0.1",
+    "@helia/car": "^3.0.0",
     "@helia/http": "^1.0.1",
     "@helia/interface": "^4.0.0",
     "@helia/ipns": "^6.0.0",
@@ -155,7 +155,11 @@
     "@libp2p/peer-id": "^4.0.5",
     "cborg": "^4.0.9",
     "hashlru": "^2.3.0",
+    "interface-blockstore": "^5.2.10",
     "ipfs-unixfs-exporter": "^13.5.0",
+    "it-map": "^3.0.5",
+    "it-pipe": "^3.0.1",
+    "it-tar": "^6.0.4",
     "it-to-browser-readablestream": "^2.0.6",
     "multiformats": "^13.1.0",
     "progress-events": "^1.0.0"
@@ -173,9 +177,12 @@
     "@types/sinon": "^17.0.3",
     "aegir": "^42.2.2",
     "blockstore-core": "^4.4.0",
+    "browser-readablestream-to-it": "^2.0.5",
     "datastore-core": "^9.2.8",
     "helia": "^4.0.1",
+    "ipfs-unixfs-importer": "^15.2.4",
     "ipns": "^9.0.0",
+    "it-all": "^3.0.4",
     "it-last": "^3.0.4",
     "it-to-buffer": "^4.0.5",
     "magic-bytes.js": "^1.8.0",

packages/verified-fetch/src/utils/get-tar-stream.ts

@@ -0,0 +1,68 @@
+import { CodeError } from '@libp2p/interface'
+import { exporter, recursive, type UnixFSEntry } from 'ipfs-unixfs-exporter'
+import map from 'it-map'
+import { pipe } from 'it-pipe'
+import { pack, type TarEntryHeader, type TarImportCandidate } from 'it-tar'
+import type { AbortOptions } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+
+const EXPORTABLE = ['file', 'raw', 'directory']
+
+function toHeader (file: UnixFSEntry): Partial<TarEntryHeader> & { name: string } {
+  let mode: number | undefined
+  let mtime: Date | undefined
+
+  if (file.type === 'file' || file.type === 'directory') {
+    mode = file.unixfs.mode
+    mtime = file.unixfs.mtime != null ? new Date(Number(file.unixfs.mtime.secs * 1000n)) : undefined
+  }
+
+  return {
+    name: file.path,
+    mode,
+    mtime,
+    size: Number(file.size),
+    type: file.type === 'directory' ? 'directory' : 'file'
+  }
+}
+
+function toTarImportCandidate (entry: UnixFSEntry): TarImportCandidate {
+  if (!EXPORTABLE.includes(entry.type)) {
+    throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS')
+  }
+
+  const candidate: TarImportCandidate = {
+    header: toHeader(entry)
+  }
+
+  if (entry.type === 'file' || entry.type === 'raw') {
+    candidate.body = entry.content()
+  }
+
+  return candidate
+}
+
+export async function * tarStream (ipfsPath: string, blockstore: Blockstore, options?: AbortOptions): AsyncGenerator<Uint8Array> {
+  const file = await exporter(ipfsPath, blockstore, options)
+
+  if (file.type === 'file' || file.type === 'raw') {
+    yield * pipe(
+      [toTarImportCandidate(file)],
+      pack()
+    )
+
+    return
+  }
+
+  if (file.type === 'directory') {
+    yield * pipe(
+      recursive(ipfsPath, blockstore, options),
+      (source) => map(source, (entry) => toTarImportCandidate(entry)),
+      pack()
+    )
+
+    return
+  }
+
+  throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS')
+}

packages/verified-fetch/src/utils/select-output-type.ts

@@ -55,7 +55,8 @@ const CID_TYPE_MAP: Record<number, string[]> = {
     'application/octet-stream',
     'application/vnd.ipld.raw',
     'application/vnd.ipfs.ipns-record',
-    'application/vnd.ipld.car'
+    'application/vnd.ipld.car',
+    'application/x-tar'
   ]
 }
 

packages/verified-fetch/src/verified-fetch.ts

@@ -14,6 +14,7 @@
 import { getContentDispositionFilename } from './utils/get-content-disposition-filename.js'
 import { getETag } from './utils/get-e-tag.js'
 import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
+import { tarStream } from './utils/get-tar-stream.js'
 import { parseResource } from './utils/parse-resource.js'
 import { notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js'
 import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js'
@@ -151,11 +152,16 @@
    * directory structure referenced by the `CID`.
    */
   private async handleTar ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
-    if (cid.code !== dagPbCode) {
-      return notAcceptableResponse('only dag-pb CIDs can be returned in TAR files')
+    if (cid.code !== dagPbCode && cid.code !== rawCode) {
+      return notAcceptableResponse('only UnixFS data can be returned in a TAR file')
     }
 
-    return notSupportedResponse('application/tar support is not implemented')
+    const stream = toBrowserReadableStream<Uint8Array>(tarStream(`/ipfs/${cid}/${path}`, this.helia.blockstore, options))
+
+    const response = okResponse(stream)
+    response.headers.set('content-type', 'application/x-tar')
+
+    return response
   }
 
   private async handleJson ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise<Response> {
@@ -397,6 +403,8 @@
     } else if (accept === 'application/x-tar') {
       // the user requested a TAR file
       reqFormat = 'tar'
+      query.download = true
+      query.filename = query.filename ?? `${cid.toString()}.tar`
       response = await this.handleTar({ cid, path, options })
     } else {
       // derive the handler from the CID type

packages/verified-fetch/test/tar.spec.ts

@@ -0,0 +1,149 @@
+import { unixfs } from '@helia/unixfs'
+import { stop } from '@libp2p/interface'
+import { expect } from 'aegir/chai'
+import browserReadableStreamToIt from 'browser-readablestream-to-it'
+import all from 'it-all'
+import last from 'it-last'
+import { pipe } from 'it-pipe'
+import { extract } from 'it-tar'
+import toBuffer from 'it-to-buffer'
+import { VerifiedFetch } from '../src/verified-fetch.js'
+import { createHelia } from './fixtures/create-offline-helia.js'
+import type { Helia } from '@helia/interface'
+import type { FileCandidate } from 'ipfs-unixfs-importer'
+
+describe('tar files', () => {
+  let helia: Helia
+  let verifiedFetch: VerifiedFetch
+
+  beforeEach(async () => {
+    helia = await createHelia()
+    verifiedFetch = new VerifiedFetch({
+      helia
+    })
+  })
+
+  afterEach(async () => {
+    await stop(helia, verifiedFetch)
+  })
+
+  it('should support fetching a TAR file', async () => {
+    const file = Uint8Array.from([0, 1, 2, 3, 4])
+    const fs = unixfs(helia)
+    const cid = await fs.addBytes(file)
+
+    const resp = await verifiedFetch.fetch(cid, {
+      headers: {
+        accept: 'application/x-tar'
+      }
+    })
+    expect(resp.status).to.equal(200)
+    expect(resp.headers.get('content-type')).to.equal('application/x-tar')
+    expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${cid.toString()}.tar"`)
+
+    if (resp.body == null) {
+      throw new Error('Download failed')
+    }
+
+    const entries = await pipe(
+      browserReadableStreamToIt(resp.body),
+      extract(),
+      async source => all(source)
+    )
+
+    expect(entries).to.have.lengthOf(1)
+    await expect(toBuffer(entries[0].body)).to.eventually.deep.equal(file)
+  })
+
+  it('should support fetching a TAR file containing a directory', async () => {
+    const directory: FileCandidate[] = [{
+      path: 'foo.txt',
+      content: Uint8Array.from([0, 1, 2, 3, 4])
+    }, {
+      path: 'bar.txt',
+      content: Uint8Array.from([5, 6, 7, 8, 9])
+    }, {
+      path: 'baz/qux.txt',
+      content: Uint8Array.from([1, 2, 3, 4, 5])
+    }]
+
+    const fs = unixfs(helia)
+    const importResult = await last(fs.addAll(directory, {
+      wrapWithDirectory: true
+    }))
+
+    if (importResult == null) {
+      throw new Error('Import failed')
+    }
+
+    const resp = await verifiedFetch.fetch(importResult.cid, {
+      headers: {
+        accept: 'application/x-tar'
+      }
+    })
+    expect(resp.status).to.equal(200)
+    expect(resp.headers.get('content-type')).to.equal('application/x-tar')
+    expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${importResult.cid.toString()}.tar"`)
+
+    if (resp.body == null) {
+      throw new Error('Download failed')
+    }
+
+    const entries = await pipe(
+      browserReadableStreamToIt(resp.body),
+      extract(),
+      async source => all(source)
+    )
+
+    expect(entries).to.have.lengthOf(5)
+    expect(entries[0]).to.have.nested.property('header.name', importResult.cid.toString())
+
+    expect(entries[1]).to.have.nested.property('header.name', `${importResult.cid}/${directory[1].path}`)
+    await expect(toBuffer(entries[1].body)).to.eventually.deep.equal(directory[1].content)
+
+    expect(entries[2]).to.have.nested.property('header.name', `${importResult.cid}/${directory[2].path?.split('/')[0]}`)
+
+    expect(entries[3]).to.have.nested.property('header.name', `${importResult.cid}/${directory[2].path}`)
+    await expect(toBuffer(entries[3].body)).to.eventually.deep.equal(directory[2].content)
+
+    expect(entries[4]).to.have.nested.property('header.name', `${importResult.cid}/${directory[0].path}`)
+    await expect(toBuffer(entries[4].body)).to.eventually.deep.equal(directory[0].content)
+  })
+
+  it('should support fetching a TAR file by format', async () => {
+    const file = Uint8Array.from([0, 1, 2, 3, 4])
+    const fs = unixfs(helia)
+    const cid = await fs.addBytes(file)
+
+    const resp = await verifiedFetch.fetch(`ipfs://${cid}?format=tar`)
+    expect(resp.status).to.equal(200)
+    expect(resp.headers.get('content-type')).to.equal('application/x-tar')
+    expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${cid.toString()}.tar"`)
+  })
+
+  it('should support specifying a filename for a TAR file', async () => {
+    const file = Uint8Array.from([0, 1, 2, 3, 4])
+    const fs = unixfs(helia)
+    const cid = await fs.addBytes(file)
+
+    const resp = await verifiedFetch.fetch(`ipfs://${cid}?filename=foo.bar`, {
+      headers: {
+        accept: 'application/x-tar'
+      }
+    })
+    expect(resp.status).to.equal(200)
+    expect(resp.headers.get('content-type')).to.equal('application/x-tar')
+    expect(resp.headers.get('content-disposition')).to.equal('attachment; filename="foo.bar"')
+  })
+
+  it('should support fetching a TAR file by format with a filename', async () => {
+    const file = Uint8Array.from([0, 1, 2, 3, 4])
+    const fs = unixfs(helia)
+    const cid = await fs.addBytes(file)
+
+    const resp = await verifiedFetch.fetch(`ipfs://${cid}?format=tar&filename=foo.bar`)
+    expect(resp.status).to.equal(200)
+    expect(resp.headers.get('content-type')).to.equal('application/x-tar')
+    expect(resp.headers.get('content-disposition')).to.equal('attachment; filename="foo.bar"')
+  })
+})

packages/verified-fetch/test/verified-fetch.spec.ts

@@ -80,7 +80,7 @@ describe('@helia/verifed-fetch', () => {
     })
 
     const formatsAndAcceptHeaders = [
-      ['tar', 'application/x-tar']
+      ['ipns-record', 'application/vnd.ipfs.ipns-record']
     ]
 
     for (const [format, acceptHeader] of formatsAndAcceptHeaders) {