Merge remote-tracking branch 'origin/main' into feat/handle-accept-header-for-raw-types

Author: achingbrain

packages/verified-fetch/src/types.ts

@@ -0,0 +1 @@
+export type RequestFormatShorthand = 'raw' | 'car' | 'tar' | 'ipns-record' | 'dag-json' | 'dag-cbor' | 'json' | 'cbor'

packages/verified-fetch/src/utils/get-e-tag.ts

@@ -0,0 +1,36 @@
+import type { RequestFormatShorthand } from '../types.js'
+import type { CID } from 'multiformats/cid'
+
+interface GetETagArg {
+  cid: CID
+  reqFormat?: RequestFormatShorthand
+  rangeStart?: number
+  rangeEnd?: number
+  /**
+   * Weak Etag is used when we can't guarantee byte-for-byte-determinism (generated, or mutable content).
+   * Some examples:
+   * - IPNS requests
+   * - CAR streamed with blocks in non-deterministic order
+   * - TAR streamed with files in non-deterministic order
+   */
+  weak?: boolean
+}
+
+/**
+ * etag
+ * you need to wrap cid  with ""
+ * we use strong Etags for immutable responses and weak one (prefixed with W/ ) for mutable/generated ones (ipns and generated HTML).
+ * block and car responses should have different etag than deserialized one, so you can add some prefix like we do in existing gateway
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
+ * @see https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header
+ */
+export function getETag ({ cid, reqFormat, weak, rangeStart, rangeEnd }: GetETagArg): string {
+  const prefix = weak === true ? 'W/' : ''
+  let suffix = reqFormat == null ? '' : `.${reqFormat}`
+  if (rangeStart != null || rangeEnd != null) {
+    suffix += `.${rangeStart ?? '0'}-${rangeEnd ?? 'N'}`
+  }
+
+  return `${prefix}"${cid.toString()}${suffix}"`
+}

packages/verified-fetch/src/utils/get-format.ts

@@ -1,18 +1,17 @@
 import { code as dagCborCode } from '@ipld/dag-cbor'
 import { code as dagJsonCode } from '@ipld/dag-json'
+import type { RequestFormatShorthand } from '../types.js'
 import type { CID } from 'multiformats/cid'
 
-export type FORMAT = 'raw' | 'car' | 'dag-json' | 'dag-cbor' | 'json' | 'cbor' | 'ipns-record' | 'tar'
-
 const FORMATS: string[] = [
   'raw', 'car', 'dag-json', 'dag-cbor', 'json', 'cbor', 'ipns-record', 'tar'
 ]
 
-function isSupportedFormat (format: string): format is FORMAT {
+function isSupportedFormat (format: string): format is RequestFormatShorthand {
   return FORMATS.includes(format)
 }
 
-const FORMAT_MAP: Record<string, FORMAT> = {
+const FORMAT_MAP: Record<string, RequestFormatShorthand> = {
   // https://www.iana.org/assignments/media-types/application/vnd.ipld.raw
   'application/vnd.ipld.raw': 'raw',
   'application/octet-stream': 'raw',
@@ -33,7 +32,7 @@ const FORMAT_MAP: Record<string, FORMAT> = {
   'application/x-tar': 'tar'
 }
 
-const MIME_TYPE_MAP: Record<FORMAT, string> = {
+const MIME_TYPE_MAP: Record<RequestFormatShorthand, string> = {
   raw: 'application/octet-stream',
   car: 'application/vnd.ipld.car',
   'dag-json': 'application/vnd.ipld.dag-json',
@@ -45,7 +44,7 @@ const MIME_TYPE_MAP: Record<FORMAT, string> = {
 }
 
 interface UserFormat {
-  format: FORMAT
+  format: RequestFormatShorthand
   mimeType: string
 }
 

packages/verified-fetch/src/utils/parse-url-string.ts

@@ -1,6 +1,7 @@
 import { peerIdFromString } from '@libp2p/peer-id'
 import { CID } from 'multiformats/cid'
 import { TLRU } from './tlru.js'
+import type { RequestFormatShorthand } from '../types.js'
 import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns'
 import type { ComponentLogger } from '@libp2p/interface'
 import type { ProgressOptions } from 'progress-events'
@@ -16,11 +17,15 @@ export interface ParseUrlStringOptions extends ProgressOptions<ResolveProgressEv
 
 }
 
+export interface ParsedUrlQuery extends Record<string, string | unknown> {
+  format?: RequestFormatShorthand
+}
+
 export interface ParsedUrlStringResults {
   protocol: string
   path: string
   cid: CID
-  query: Record<string, string>
+  query: ParsedUrlQuery
 }
 
 const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/$?]+)\/?(?<path>[^$?]*)\??(?<queryString>.*)$/

packages/verified-fetch/src/verified-fetch.ts

@@ -9,6 +9,7 @@ import { code as rawCode } from 'multiformats/codecs/raw'
 import { identity } from 'multiformats/hashes/identity'
 import { CustomProgressEvent } from 'progress-events'
 import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js'
+import { getETag } from './utils/get-e-tag.js'
 import { getFormat } from './utils/get-format.js'
 import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
 import { parseResource } from './utils/parse-resource.js'
@@ -383,7 +384,7 @@ export class VerifiedFetch {
       return notAcceptableResponse()
     }
 
-    response.headers.set('etag', cid.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header
+    response.headers.set('etag', getETag({ cid, reqFormat: format?.format, weak: false }))
     response.headers.set('cache-control', 'public, max-age=29030400, immutable')
     response.headers.set('X-Ipfs-Path', resource.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header
 

packages/verified-fetch/test/get-e-tag.spec.ts

@@ -0,0 +1,33 @@
+import { expect } from 'aegir/chai'
+import { CID } from 'multiformats/cid'
+import { getETag } from '../src/utils/get-e-tag.js'
+
+const cidString = 'QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'
+const testCID = CID.parse(cidString)
+
+describe('getETag', () => {
+  it('CID eTag', () => {
+    expect(getETag({ cid: testCID, weak: true })).to.equal(`W/"${cidString}"`)
+    expect(getETag({ cid: testCID, weak: false })).to.equal(`"${cidString}"`)
+  })
+
+  it('should return ETag with CID and format suffix', () => {
+    expect(getETag({ cid: testCID, reqFormat: 'raw' })).to.equal(`"${cidString}.raw"`)
+    expect(getETag({ cid: testCID, reqFormat: 'json' })).to.equal(`"${cidString}.json"`)
+  })
+
+  it('should return ETag with CID and range suffix', () => {
+    expect(getETag({ cid: testCID, weak: true, reqFormat: 'car', rangeStart: 10, rangeEnd: 20 })).to.equal(`W/"${cidString}.car.10-20"`)
+    expect(getETag({ cid: testCID, weak: false, reqFormat: 'car', rangeStart: 10, rangeEnd: 20 })).to.equal(`"${cidString}.car.10-20"`)
+  })
+
+  it('should return ETag with CID, format and range suffix', () => {
+    expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: 10, rangeEnd: 20 })).to.equal(`"${cidString}.raw.10-20"`)
+  })
+
+  it('should handle undefined rangeStart and rangeEnd', () => {
+    expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: undefined, rangeEnd: undefined })).to.equal(`"${cidString}.raw"`)
+    expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: 55, rangeEnd: undefined })).to.equal(`"${cidString}.raw.55-N"`)
+    expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: undefined, rangeEnd: 77 })).to.equal(`"${cidString}.raw.0-77"`)
+  })
+})