feat: add explicit support for subdomain gateways

Author: 2color

packages/block-brokers/src/trustless-gateway/broker.ts

@@ -19,8 +19,8 @@ ProgressOptions<TrustlessGatewayGetBlockProgressEvents>
   constructor (components: TrustlessGatewayComponents, init: TrustlessGatewayBlockBrokerInit = {}) {
     this.log = components.logger.forComponent('helia:trustless-gateway-block-broker')
     this.gateways = (init.gateways ?? DEFAULT_TRUSTLESS_GATEWAYS)
-      .map((gatewayOrUrl) => {
-        return new TrustlessGateway(gatewayOrUrl)
+      .map((gw) => {
+        return new TrustlessGateway(gw.url, gw.isSubdomain)
       })
   }
 

packages/block-brokers/src/trustless-gateway/index.ts

@@ -3,22 +3,27 @@ import type { BlockRetriever } from '@helia/interface/src/blocks.js'
 import type { ComponentLogger } from '@libp2p/interface'
 import type { ProgressEvent } from 'progress-events'
 
-export const DEFAULT_TRUSTLESS_GATEWAYS = [
-  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
-  'https://trustless-gateway.link',
+export const DEFAULT_TRUSTLESS_GATEWAYS: TrustlessGatewayUrl[] = [
+  // 2024-02-20: IPNS and Block/CAR support from https://ipfs.github.io/public-gateway-checker/
+  { url: 'https://trustless-gateway.link', isSubdomain: false },
 
-  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
-  'https://cloudflare-ipfs.com',
+  // 2024-02-20: IPNS and Block/CAR support from https://ipfs.github.io/public-gateway-checker/
+  { url: 'https://cloudflare-ipfs.com', isSubdomain: false },
 
-  // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/
-  'https://4everland.io'
+  // 2024-02-20: IPNS, Origin, and Block/CAR support from https://ipfs.github.io/public-gateway-checker/
+  { url: 'https://4everland.io', isSubdomain: true },
 ]
 
+interface TrustlessGatewayUrl {
+  url: string | URL
+  isSubdomain: boolean
+}
+
 export type TrustlessGatewayGetBlockProgressEvents =
   ProgressEvent<'trustless-gateway:get-block:fetch', URL>
 
 export interface TrustlessGatewayBlockBrokerInit {
-  gateways?: Array<string | URL>
+  gateways?: Array<TrustlessGatewayUrl>
 }
 
 export interface TrustlessGatewayComponents {

packages/block-brokers/src/trustless-gateway/trustless-gateway.ts

@@ -1,4 +1,5 @@
 import type { CID } from 'multiformats/cid'
+import { base32 } from 'multiformats/bases/base32'
 
 /**
  * A `TrustlessGateway` keeps track of the number of attempts, errors, and
@@ -8,6 +9,12 @@ import type { CID } from 'multiformats/cid'
  */
 export class TrustlessGateway {
   public readonly url: URL
+
+  /**
+   * Whether this gateway is a subdomain resolution style gateway
+   */
+  public isSubdomain: boolean
+
   /**
    * The number of times this gateway has been attempted to be used to fetch a
    * block. This includes successful, errored, and aborted attempts. By counting
@@ -36,34 +43,36 @@ export class TrustlessGateway {
    */
   #successes = 0
 
-  constructor (url: URL | string) {
+  constructor(url: URL | string, isSubdomain: boolean = false) {
     this.url = url instanceof URL ? url : new URL(url)
+    this.isSubdomain = isSubdomain
   }
 
   /**
    * Fetch a raw block from `this.url` following the specification defined at
    * https://specs.ipfs.tech/http-gateways/trustless-gateway/
    */
-  async getRawBlock (cid: CID, signal?: AbortSignal): Promise<Uint8Array> {
-    const gwUrl = this.url
-    gwUrl.pathname = `/ipfs/${cid.toString()}`
+  async getRawBlock(cid: CID, signal?: AbortSignal): Promise<Uint8Array> {
+    const gwUrl = this.getGwUrl(cid)
 
     // necessary as not every gateway supports dag-cbor, but every should support
     // sending raw block as-is
     gwUrl.search = '?format=raw'
 
     if (signal?.aborted === true) {
-      throw new Error(`Signal to fetch raw block for CID ${cid} from gateway ${this.url} was aborted prior to fetch`)
+      throw new Error(
+        `Signal to fetch raw block for CID ${cid} from gateway ${this.url} was aborted prior to fetch`,
+      )
     }
 
     try {
       this.#attempts++
       const res = await fetch(gwUrl.toString(), {
         signal,
         headers: {
-        // also set header, just in case ?format= is filtered out by some
-        // reverse proxy
-          Accept: 'application/vnd.ipld.raw'
+          // also set header, just in case ?format= is filtered out by some
+          // reverse proxy
+          Accept: 'application/vnd.ipld.raw',
         },
         cache: 'force-cache'
       })
@@ -84,6 +93,20 @@ export class TrustlessGateway {
     }
   }
 
+  /**
+   * Construct the Gateway URL for a CID
+   */
+  getGwUrl(cid: CID): URL {
+    const gwUrl = new URL(this.url)
+
+    if (this.isSubdomain) {
+      gwUrl.hostname = `${cid.toString(base32)}.ipfs.${gwUrl.hostname}`
+    } else {
+      gwUrl.pathname = `/ipfs/${cid.toString()}`
+    }
+    return gwUrl
+  }
+
   /**
    * Encapsulate the logic for determining whether a gateway is considered
    * reliable, for prioritization. This is based on the number of successful attempts made

packages/block-brokers/test/trustless-gateway.spec.ts

@@ -37,7 +37,7 @@ describe('trustless-gateway-block-broker', () => {
 
     gateways = [
       stubConstructor(TrustlessGateway, 'http://localhost:8080'),
-      stubConstructor(TrustlessGateway, 'http://localhost:8081'),
+      stubConstructor(TrustlessGateway, 'http://localhost:8081', true),
       stubConstructor(TrustlessGateway, 'http://localhost:8082'),
       stubConstructor(TrustlessGateway, 'http://localhost:8083')
     ]
@@ -150,4 +150,17 @@ describe('trustless-gateway-block-broker', () => {
     expect(gateways[1].getRawBlock.calledWith(cid1, Sinon.match.any)).to.be.false()
     expect(gateways[2].getRawBlock.calledWith(cid1, Sinon.match.any)).to.be.false()
   })
+
+  it('constructs the gateway url for the cid for both path and subdomain gateways', async () => {
+    const pathGw = new TrustlessGateway('http://localhost:8080')
+    const subdomainGw = new TrustlessGateway('https://dweb.link', true)
+
+    expect(pathGw.getGwUrl(blocks[0].cid).hostname).to.equal(`localhost`)
+    expect(pathGw.getGwUrl(blocks[0].cid).toString()).to.equal(`http://localhost:8080/ipfs/bafkreiefnkxuhnq3536qo2i2w3tazvifek4mbbzb6zlq3ouhprjce5c3aq`)
+    expect(pathGw.getGwUrl(blocks[1].cid).toString()).to.equal(`http://localhost:8080/ipfs/${blocks[1].cid.toString()}`)
+
+    expect(subdomainGw.getGwUrl(blocks[0].cid).hostname).to.equal(`bafkreiefnkxuhnq3536qo2i2w3tazvifek4mbbzb6zlq3ouhprjce5c3aq.ipfs.dweb.link`)
+    expect(subdomainGw.getGwUrl(blocks[0].cid).toString()).to.equal(`https://bafkreiefnkxuhnq3536qo2i2w3tazvifek4mbbzb6zlq3ouhprjce5c3aq.ipfs.dweb.link/`)
+    expect(subdomainGw.getGwUrl(blocks[1].cid).toString()).to.equal(`https://${blocks[1].cid.toString()}.ipfs.dweb.link/`)
+  })
 })