chore: add more content type tests and default to application/octet-s…

…tream

packages/verified-fetch/.aegir.js

@@ -0,0 +1,8 @@
+/** @type {import('aegir').PartialOptions} */
+const options = {
+  build: {
+    bundlesizeMax: '132KB'
+  }
+}
+
+export default options

packages/verified-fetch/README.md

@@ -124,6 +124,30 @@ const resp = await fetch('ipfs://bafy...')
 const json = await resp.json()
 ```
 
+### Custom content-type parsing
+
+By default, `@helia/verified-fetch` sets the `Content-Type` header as `application/octet-stream` - this is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected without a detailed content type.
+
+If you require an accurate content-type you can provide a `contentTypeParser` function as an option to `createVerifiedFetch` to handle parsing the content type.
+
+The function you provide will be called with the first chunk of bytes from the file and should return a string or a promise of a string.
+
+## Example - Customizing content-type parsing
+
+```typescript
+import { createVerifiedFetch } from '@helia/verified-fetch'
+import { fileTypeFromBuffer } from '@sgtpooki/file-type'
+
+const fetch = await createVerifiedFetch({
+ gateways: ['https://trustless-gateway.link'],
+ routers: ['http://delegated-ipfs.dev'],
+ contentTypeParser: async (bytes) => {
+   // call to some magic-byte recognition library like magic-bytes, file-type, or your own custom byte recognition
+   return fileTypeFromBuffer(bytes)?.mime
+ }
+})
+```
+
 ## Comparison to fetch
 
 This module attempts to act as similarly to the `fetch()` API as possible.

packages/verified-fetch/package.json

@@ -163,10 +163,11 @@
   "devDependencies": {
     "@libp2p/logger": "^4.0.5",
     "@libp2p/peer-id-factory": "^4.0.5",
-    "@types/mime-types": "^2.1.4",
+    "@sgtpooki/file-type": "^1.0.1",
     "@types/sinon": "^17.0.3",
     "aegir": "^42.2.2",
     "helia": "^4.0.1",
+    "magic-bytes.js": "^1.8.0",
     "sinon": "^17.0.1",
     "sinon-ts": "^2.0.0"
   },

packages/verified-fetch/src/index.ts

@@ -114,7 +114,11 @@
  *
  * ### Custom content-type parsing
  *
- * By default, `@helia/verified-fetch` does not set the `Content-Type` header. This is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected. You can provide a `contentTypeParser` function to the `createVerifiedFetch` function to handle parsing the content type. The function you provide will be passed the first bytes we receive from the network.
+ * By default, `@helia/verified-fetch` sets the `Content-Type` header as `application/octet-stream` - this is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected without a detailed content type.
+ *
+ * If you require an accurate content-type you can provide a `contentTypeParser` function as an option to `createVerifiedFetch` to handle parsing the content type.
+ *
+ * The function you provide will be called with the first chunk of bytes from the file and should return a string or a promise of a string.
  *
  * @example Customizing content-type parsing
  *
@@ -127,7 +131,7 @@
  *  routers: ['http://delegated-ipfs.dev'],
  *  contentTypeParser: async (bytes) => {
  *    // call to some magic-byte recognition library like magic-bytes, file-type, or your own custom byte recognition
- *    return fileTypeFromBuffer(bytes)?.mime ?? 'application/octet-stream'
+ *    return fileTypeFromBuffer(bytes)?.mime
  *  }
  * })
  * ```
@@ -277,15 +281,18 @@ export interface VerifiedFetch {
 }
 
 /**
- * Instead of passing a Helia instance, you can pass a list of gateways and routers, and a HeliaHTTP instance will be created for you.
+ * Instead of passing a Helia instance, you can pass a list of gateways and
+ * routers, and a HeliaHTTP instance will be created for you.
  */
-export interface CreateVerifiedFetchWithOptions {
+export interface CreateVerifiedFetchOptions {
   gateways: string[]
   routers?: string[]
+
   /**
-   * A function to handle parsing content type from bytes. The function you provide will be passed the first set of
-   * bytes we receive from the network, and should return a string that will be used as the value for the `Content-Type`
-   * header in the response.
+   * A function to handle parsing content type from bytes. The function you
+   * provide will be passed the first set of bytes we receive from the network,
+   * and should return a string that will be used as the value for the
+   * `Content-Type` header in the response.
    */
   contentTypeParser?: ContentTypeParser
 }
@@ -308,16 +315,17 @@ export type VerifiedFetchProgressEvents =
 /**
  * Options for the `fetch` function returned by `createVerifiedFetch`.
  *
- * This method accepts all the same options as the `fetch` function in the browser, plus an `onProgress` option to
- * listen for progress events.
+ * This interface contains all the same fields as the [options object](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options)
+ * passed to `fetch` in browsers, plus an `onProgress` option to listen for
+ * progress events.
  */
 export interface VerifiedFetchInit extends RequestInit, ProgressOptions<BubbledProgressEvents | VerifiedFetchProgressEvents> {
 }
 
 /**
  * Create and return a Helia node
  */
-export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchWithOptions): Promise<VerifiedFetch> {
+export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchOptions): Promise<VerifiedFetch> {
   if (!isHelia(init)) {
     init = await createHeliaHTTP({
       blockBrokers: [

packages/verified-fetch/src/verified-fetch.ts

@@ -30,7 +30,7 @@ interface VerifiedFetchComponents {
 }
 
 export interface ContentTypeParser {
-  (bytes: Uint8Array): Promise<string>
+  (bytes: Uint8Array): Promise<string> | string
 }
 
 /**
@@ -204,13 +204,23 @@ export class VerifiedFetch {
   }
 
   private async setContentType (bytes: Uint8Array, response: Response): Promise<void> {
+    let contentType = 'application/octet-stream'
+
     if (this.contentTypeParser != null) {
       try {
-        response.headers.set('content-type', await this.contentTypeParser(bytes))
+        const res = this.contentTypeParser(bytes)
+
+        if (isPromise(res)) {
+          contentType = await res
+        } else {
+          contentType = res
+        }
       } catch (err) {
         this.log.error('Error parsing content type', err)
       }
     }
+
+    response.headers.set('content-type', contentType)
   }
 
   /**
@@ -336,3 +346,7 @@ export class VerifiedFetch {
     await this.helia.stop()
   }
 }
+
+function isPromise <T> (p?: any): p is Promise<T> {
+  return p?.then != null
+}

packages/verified-fetch/test/content-type-parser.spec.ts

@@ -0,0 +1,71 @@
+import { createHeliaHTTP } from '@helia/http'
+import { unixfs } from '@helia/unixfs'
+import { stop } from '@libp2p/interface'
+import { fileTypeFromBuffer } from '@sgtpooki/file-type'
+import { expect } from 'aegir/chai'
+import { filetypemime } from 'magic-bytes.js'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { VerifiedFetch } from '../src/verified-fetch.js'
+import type { Helia } from '@helia/interface'
+import type { CID } from 'multiformats/cid'
+
+describe('content-type-parser', () => {
+  let helia: Helia
+  let cid: CID
+  let verifiedFetch: VerifiedFetch
+
+  beforeEach(async () => {
+    helia = await createHeliaHTTP()
+    const fs = unixfs(helia)
+    cid = await fs.addByteStream((async function * () {
+      yield uint8ArrayFromString('H4sICIlTHVIACw', 'base64')
+    })())
+  })
+
+  afterEach(async () => {
+    await stop(verifiedFetch)
+  })
+
+  it('does not set content type if contentTypeParser is not passed', async () => {
+    verifiedFetch = new VerifiedFetch({
+      helia
+    })
+    const resp = await verifiedFetch.fetch(cid)
+    expect(resp.headers.get('content-type')).to.equal('application/octet-stream')
+  })
+
+  it('sets content type if contentTypeParser is passed', async () => {
+    verifiedFetch = new VerifiedFetch({
+      helia
+    }, {
+      contentTypeParser: () => 'text/plain'
+    })
+    const resp = await verifiedFetch.fetch(cid)
+    expect(resp.headers.get('content-type')).to.equal('text/plain')
+  })
+
+  it('supports @sgtpooki/file-type as a contentTypeParser', async () => {
+    verifiedFetch = new VerifiedFetch({
+      helia
+    }, {
+      contentTypeParser: async (bytes) => {
+        const type = await fileTypeFromBuffer(bytes)
+        return type?.mime
+      }
+    })
+    const resp = await verifiedFetch.fetch(cid)
+    expect(resp.headers.get('content-type')).to.equal('application/gzip')
+  })
+
+  it('supports magic-bytes.js as a contentTypeParser', async () => {
+    verifiedFetch = new VerifiedFetch({
+      helia
+    }, {
+      contentTypeParser: (bytes) => {
+        return filetypemime(bytes)?.[0]
+      }
+    })
+    const resp = await verifiedFetch.fetch(cid)
+    expect(resp.headers.get('content-type')).to.equal('application/gzip')
+  })
+})

packages/verified-fetch/test/verified-fetch.spec.ts

@@ -395,82 +395,4 @@ describe('@helia/verifed-fetch', () => {
       expect(data).to.equal('hello world')
     })
   })
-
-  describe('with contentTypeParser', () => {
-    let pathWalkerStub: SinonStub<Parameters<PathWalkerFn>, ReturnType<PathWalkerFn>>
-    let unixfsStub: ReturnType<typeof stubInterface<UnixFS>>
-    beforeEach(() => {
-      pathWalkerStub = sinon.stub<Parameters<PathWalkerFn>, ReturnType<PathWalkerFn>>()
-      unixfsStub = stubInterface<UnixFS>({
-        cat: sinon.stub(),
-        stat: sinon.stub()
-      })
-    })
-
-    it('does not set content type if contentTypeParser is not passed', async () => {
-      const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03])
-      const verifiedFetch = new VerifiedFetch({
-        helia: stubInterface<Helia>({
-          logger: defaultLogger()
-        }),
-        unixfs: unixfsStub
-      })
-      pathWalkerStub.returns(Promise.resolve({
-        ipfsRoots: [testCID],
-        terminalElement: {
-          cid: testCID,
-          size: BigInt(3),
-          depth: 1,
-          content: async function * () { yield finalRootFileContent },
-          name: 'foobar.txt',
-          path: '',
-          type: 'raw',
-          node: finalRootFileContent
-        }
-      }))
-      unixfsStub.cat.returns({
-        [Symbol.asyncIterator]: async function * () {
-          yield finalRootFileContent
-        }
-      })
-      const resp = await verifiedFetch.fetch(testCID)
-      expect(resp.headers.get('content-type')).to.equal(null)
-      await verifiedFetch.stop()
-    })
-
-    it('sets content type if contentTypeParser is passed', async () => {
-      const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03])
-      const contentTypeParser = sinon.stub().returns('text/plain')
-      pathWalkerStub.returns(Promise.resolve({
-        ipfsRoots: [testCID],
-        terminalElement: {
-          cid: testCID,
-          size: BigInt(3),
-          depth: 1,
-          content: async function * () { yield finalRootFileContent },
-          name: 'foobar.txt',
-          path: '',
-          type: 'raw',
-          node: finalRootFileContent
-        }
-      }))
-      unixfsStub.cat.returns({
-        [Symbol.asyncIterator]: async function * () {
-          yield finalRootFileContent
-        }
-      })
-      const verifiedFetch = new VerifiedFetch({
-        helia: stubInterface<Helia>({
-          logger: defaultLogger()
-        }),
-        unixfs: unixfsStub
-      }, {
-        contentTypeParser
-      })
-      const resp = await verifiedFetch.fetch(testCID)
-      expect(contentTypeParser.withArgs(finalRootFileContent).callCount).to.equal(1)
-      expect(resp.headers.get('content-type')).to.equal('text/plain')
-      await verifiedFetch.stop()
-    })
-  })
 })