From ab4ecd1ec182c83361d4c962436d937745f37828 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 8 Feb 2024 13:30:36 +0100 Subject: [PATCH] chore: add more content type tests and default to application/octet-stream --- packages/verified-fetch/.aegir.js | 8 ++ packages/verified-fetch/README.md | 24 ++++++ packages/verified-fetch/package.json | 3 +- packages/verified-fetch/src/index.ts | 28 ++++--- packages/verified-fetch/src/verified-fetch.ts | 18 ++++- .../test/content-type-parser.spec.ts | 71 +++++++++++++++++ .../test/verified-fetch.spec.ts | 78 ------------------- 7 files changed, 139 insertions(+), 91 deletions(-) create mode 100644 packages/verified-fetch/.aegir.js create mode 100644 packages/verified-fetch/test/content-type-parser.spec.ts diff --git a/packages/verified-fetch/.aegir.js b/packages/verified-fetch/.aegir.js new file mode 100644 index 000000000..89b563380 --- /dev/null +++ b/packages/verified-fetch/.aegir.js @@ -0,0 +1,8 @@ +/** @type {import('aegir').PartialOptions} */ +const options = { + build: { + bundlesizeMax: '132KB' + } +} + +export default options diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md index 8e8717465..0da153a69 100644 --- a/packages/verified-fetch/README.md +++ b/packages/verified-fetch/README.md @@ -124,6 +124,30 @@ const resp = await fetch('ipfs://bafy...') const json = await resp.json() ``` +### Custom content-type parsing + +By default, `@helia/verified-fetch` sets the `Content-Type` header as `application/octet-stream` - this is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected without a detailed content type. + +If you require an accurate content-type you can provide a `contentTypeParser` function as an option to `createVerifiedFetch` to handle parsing the content type. + +The function you provide will be called with the first chunk of bytes from the file and should return a string or a promise of a string. + +## Example - Customizing content-type parsing + +```typescript +import { createVerifiedFetch } from '@helia/verified-fetch' +import { fileTypeFromBuffer } from '@sgtpooki/file-type' + +const fetch = await createVerifiedFetch({ + gateways: ['https://trustless-gateway.link'], + routers: ['http://delegated-ipfs.dev'], + contentTypeParser: async (bytes) => { + // call to some magic-byte recognition library like magic-bytes, file-type, or your own custom byte recognition + return fileTypeFromBuffer(bytes)?.mime + } +}) +``` + ## Comparison to fetch This module attempts to act as similarly to the `fetch()` API as possible. diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 4ee1c6a7f..f4b55fb13 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -163,10 +163,11 @@ "devDependencies": { "@libp2p/logger": "^4.0.5", "@libp2p/peer-id-factory": "^4.0.5", - "@types/mime-types": "^2.1.4", + "@sgtpooki/file-type": "^1.0.1", "@types/sinon": "^17.0.3", "aegir": "^42.2.2", "helia": "^4.0.1", + "magic-bytes.js": "^1.8.0", "sinon": "^17.0.1", "sinon-ts": "^2.0.0" }, diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index a9de343f6..e2759d6e9 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -114,7 +114,11 @@ * * ### Custom content-type parsing * - * By default, `@helia/verified-fetch` does not set the `Content-Type` header. This is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected. You can provide a `contentTypeParser` function to the `createVerifiedFetch` function to handle parsing the content type. The function you provide will be passed the first bytes we receive from the network. + * By default, `@helia/verified-fetch` sets the `Content-Type` header as `application/octet-stream` - this is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected without a detailed content type. + * + * If you require an accurate content-type you can provide a `contentTypeParser` function as an option to `createVerifiedFetch` to handle parsing the content type. + * + * The function you provide will be called with the first chunk of bytes from the file and should return a string or a promise of a string. * * @example Customizing content-type parsing * @@ -127,7 +131,7 @@ * routers: ['http://delegated-ipfs.dev'], * contentTypeParser: async (bytes) => { * // call to some magic-byte recognition library like magic-bytes, file-type, or your own custom byte recognition - * return fileTypeFromBuffer(bytes)?.mime ?? 'application/octet-stream' + * return fileTypeFromBuffer(bytes)?.mime * } * }) * ``` @@ -277,15 +281,18 @@ export interface VerifiedFetch { } /** - * Instead of passing a Helia instance, you can pass a list of gateways and routers, and a HeliaHTTP instance will be created for you. + * Instead of passing a Helia instance, you can pass a list of gateways and + * routers, and a HeliaHTTP instance will be created for you. */ -export interface CreateVerifiedFetchWithOptions { +export interface CreateVerifiedFetchOptions { gateways: string[] routers?: string[] + /** - * A function to handle parsing content type from bytes. The function you provide will be passed the first set of - * bytes we receive from the network, and should return a string that will be used as the value for the `Content-Type` - * header in the response. + * A function to handle parsing content type from bytes. The function you + * provide will be passed the first set of bytes we receive from the network, + * and should return a string that will be used as the value for the + * `Content-Type` header in the response. */ contentTypeParser?: ContentTypeParser } @@ -308,8 +315,9 @@ export type VerifiedFetchProgressEvents = /** * Options for the `fetch` function returned by `createVerifiedFetch`. * - * This method accepts all the same options as the `fetch` function in the browser, plus an `onProgress` option to - * listen for progress events. + * This interface contains all the same fields as the [options object](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options) + * passed to `fetch` in browsers, plus an `onProgress` option to listen for + * progress events. */ export interface VerifiedFetchInit extends RequestInit, ProgressOptions { } @@ -317,7 +325,7 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions { +export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchOptions): Promise { if (!isHelia(init)) { init = await createHeliaHTTP({ blockBrokers: [ diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 28f9d4dfd..0dcbfd4b4 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -30,7 +30,7 @@ interface VerifiedFetchComponents { } export interface ContentTypeParser { - (bytes: Uint8Array): Promise + (bytes: Uint8Array): Promise | string } /** @@ -204,13 +204,23 @@ export class VerifiedFetch { } private async setContentType (bytes: Uint8Array, response: Response): Promise { + let contentType = 'application/octet-stream' + if (this.contentTypeParser != null) { try { - response.headers.set('content-type', await this.contentTypeParser(bytes)) + const res = this.contentTypeParser(bytes) + + if (isPromise(res)) { + contentType = await res + } else { + contentType = res + } } catch (err) { this.log.error('Error parsing content type', err) } } + + response.headers.set('content-type', contentType) } /** @@ -336,3 +346,7 @@ export class VerifiedFetch { await this.helia.stop() } } + +function isPromise (p?: any): p is Promise { + return p?.then != null +} diff --git a/packages/verified-fetch/test/content-type-parser.spec.ts b/packages/verified-fetch/test/content-type-parser.spec.ts new file mode 100644 index 000000000..7310e9e90 --- /dev/null +++ b/packages/verified-fetch/test/content-type-parser.spec.ts @@ -0,0 +1,71 @@ +import { createHeliaHTTP } from '@helia/http' +import { unixfs } from '@helia/unixfs' +import { stop } from '@libp2p/interface' +import { fileTypeFromBuffer } from '@sgtpooki/file-type' +import { expect } from 'aegir/chai' +import { filetypemime } from 'magic-bytes.js' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { VerifiedFetch } from '../src/verified-fetch.js' +import type { Helia } from '@helia/interface' +import type { CID } from 'multiformats/cid' + +describe('content-type-parser', () => { + let helia: Helia + let cid: CID + let verifiedFetch: VerifiedFetch + + beforeEach(async () => { + helia = await createHeliaHTTP() + const fs = unixfs(helia) + cid = await fs.addByteStream((async function * () { + yield uint8ArrayFromString('H4sICIlTHVIACw', 'base64') + })()) + }) + + afterEach(async () => { + await stop(verifiedFetch) + }) + + it('does not set content type if contentTypeParser is not passed', async () => { + verifiedFetch = new VerifiedFetch({ + helia + }) + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/octet-stream') + }) + + it('sets content type if contentTypeParser is passed', async () => { + verifiedFetch = new VerifiedFetch({ + helia + }, { + contentTypeParser: () => 'text/plain' + }) + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('text/plain') + }) + + it('supports @sgtpooki/file-type as a contentTypeParser', async () => { + verifiedFetch = new VerifiedFetch({ + helia + }, { + contentTypeParser: async (bytes) => { + const type = await fileTypeFromBuffer(bytes) + return type?.mime + } + }) + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/gzip') + }) + + it('supports magic-bytes.js as a contentTypeParser', async () => { + verifiedFetch = new VerifiedFetch({ + helia + }, { + contentTypeParser: (bytes) => { + return filetypemime(bytes)?.[0] + } + }) + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/gzip') + }) +}) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index b02be3cd5..1d20cf938 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -395,82 +395,4 @@ describe('@helia/verifed-fetch', () => { expect(data).to.equal('hello world') }) }) - - describe('with contentTypeParser', () => { - let pathWalkerStub: SinonStub, ReturnType> - let unixfsStub: ReturnType> - beforeEach(() => { - pathWalkerStub = sinon.stub, ReturnType>() - unixfsStub = stubInterface({ - cat: sinon.stub(), - stat: sinon.stub() - }) - }) - - it('does not set content type if contentTypeParser is not passed', async () => { - const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) - const verifiedFetch = new VerifiedFetch({ - helia: stubInterface({ - logger: defaultLogger() - }), - unixfs: unixfsStub - }) - pathWalkerStub.returns(Promise.resolve({ - ipfsRoots: [testCID], - terminalElement: { - cid: testCID, - size: BigInt(3), - depth: 1, - content: async function * () { yield finalRootFileContent }, - name: 'foobar.txt', - path: '', - type: 'raw', - node: finalRootFileContent - } - })) - unixfsStub.cat.returns({ - [Symbol.asyncIterator]: async function * () { - yield finalRootFileContent - } - }) - const resp = await verifiedFetch.fetch(testCID) - expect(resp.headers.get('content-type')).to.equal(null) - await verifiedFetch.stop() - }) - - it('sets content type if contentTypeParser is passed', async () => { - const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) - const contentTypeParser = sinon.stub().returns('text/plain') - pathWalkerStub.returns(Promise.resolve({ - ipfsRoots: [testCID], - terminalElement: { - cid: testCID, - size: BigInt(3), - depth: 1, - content: async function * () { yield finalRootFileContent }, - name: 'foobar.txt', - path: '', - type: 'raw', - node: finalRootFileContent - } - })) - unixfsStub.cat.returns({ - [Symbol.asyncIterator]: async function * () { - yield finalRootFileContent - } - }) - const verifiedFetch = new VerifiedFetch({ - helia: stubInterface({ - logger: defaultLogger() - }), - unixfs: unixfsStub - }, { - contentTypeParser - }) - const resp = await verifiedFetch.fetch(testCID) - expect(contentTypeParser.withArgs(finalRootFileContent).callCount).to.equal(1) - expect(resp.headers.get('content-type')).to.equal('text/plain') - await verifiedFetch.stop() - }) - }) })