fix: verified-fetch etag header (#434)

Fixes the formatting for the etag header

---------

Co-authored-by: Marcin Rataj <lidel@lidel.org>
Co-authored-by: Alex Potsides <alex@achingbrain.net>

Author: 3 people

packages/verified-fetch/src/types.ts

@@ -0,0 +1 @@
+export type RequestFormatShorthand = 'raw' | 'car' | 'tar' | 'ipns-record' | 'dag-json' | 'dag-cbor' | 'json' | 'cbor'

packages/verified-fetch/src/utils/get-e-tag.ts

@@ -0,0 +1,36 @@
+import type { RequestFormatShorthand } from '../types.js'
+import type { CID } from 'multiformats/cid'
+
+interface GetETagArg {
+  cid: CID
+  reqFormat?: RequestFormatShorthand
+  rangeStart?: number
+  rangeEnd?: number
+  /**
+   * Weak Etag is used when we can't guarantee byte-for-byte-determinism (generated, or mutable content).
+   * Some examples:
+   * - IPNS requests
+   * - CAR streamed with blocks in non-deterministic order
+   * - TAR streamed with files in non-deterministic order
+   */
+  weak?: boolean
+}
+
+/**
+ * etag
+ * you need to wrap cid  with ""
+ * we use strong Etags for immutable responses and weak one (prefixed with W/ ) for mutable/generated ones (ipns and generated HTML).
+ * block and car responses should have different etag than deserialized one, so you can add some prefix like we do in existing gateway
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
+ * @see https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header
+ */
+export function getETag ({ cid, reqFormat, weak, rangeStart, rangeEnd }: GetETagArg): string {
+  const prefix = weak === true ? 'W/' : ''
+  let suffix = reqFormat == null ? '' : `.${reqFormat}`
+  if (rangeStart != null || rangeEnd != null) {
+    suffix += `.${rangeStart ?? '0'}-${rangeEnd ?? 'N'}`
+  }
+
+  return `${prefix}"${cid.toString()}${suffix}"`
+}

packages/verified-fetch/src/utils/parse-url-string.ts

@@ -1,6 +1,7 @@
 import { peerIdFromString } from '@libp2p/peer-id'
 import { CID } from 'multiformats/cid'
 import { TLRU } from './tlru.js'
+import type { RequestFormatShorthand } from '../types.js'
 import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns'
 import type { ComponentLogger } from '@libp2p/interface'
 import type { ProgressOptions } from 'progress-events'
@@ -16,11 +17,15 @@ export interface ParseUrlStringOptions extends ProgressOptions<ResolveProgressEv
 
 }
 
+export interface ParsedUrlQuery extends Record<string, string | unknown> {
+  format?: RequestFormatShorthand
+}
+
 export interface ParsedUrlStringResults {
   protocol: string
   path: string
   cid: CID
-  query: Record<string, string>
+  query: ParsedUrlQuery
 }
 
 const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/$?]+)\/?(?<path>[^$?]*)\??(?<queryString>.*)$/

packages/verified-fetch/src/verified-fetch.ts

@@ -9,10 +9,12 @@ import { code as rawCode } from 'multiformats/codecs/raw'
 import { identity } from 'multiformats/hashes/identity'
 import { CustomProgressEvent } from 'progress-events'
 import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js'
+import { getETag } from './utils/get-e-tag.js'
 import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
 import { parseResource } from './utils/parse-resource.js'
 import { walkPath, type PathWalkerFn } from './utils/walk-path.js'
 import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
+import type { RequestFormatShorthand } from './types.js'
 import type { Helia } from '@helia/interface'
 import type { AbortOptions, Logger } from '@libp2p/interface'
 import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
@@ -231,8 +233,8 @@ export class VerifiedFetch {
    * @see https://specs.ipfs.tech/http-gateways/path-gateway/#format-request-query-parameter
    * @default 'raw'
    */
-  private getFormat ({ headerFormat, queryFormat }: { headerFormat: string | null, queryFormat: string | null }): string | null {
-    const formatMap: Record<string, string> = {
+  private getFormat ({ headerFormat, queryFormat }: { headerFormat: string | null, queryFormat: RequestFormatShorthand | null }): RequestFormatShorthand | null {
+    const formatMap: Record<string, RequestFormatShorthand> = {
       'vnd.ipld.raw': 'raw',
       'vnd.ipld.car': 'car',
       'application/x-tar': 'tar',
@@ -323,7 +325,7 @@ export class VerifiedFetch {
       }
     }
 
-    response.headers.set('etag', cid.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header
+    response.headers.set('etag', getETag({ cid, reqFormat: format ?? undefined, weak: false }))
     response.headers.set('cache-control', 'public, max-age=29030400, immutable')
     response.headers.set('X-Ipfs-Path', resource.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header
 

packages/verified-fetch/test/get-e-tag.spec.ts

@@ -0,0 +1,33 @@
+import { expect } from 'aegir/chai'
+import { CID } from 'multiformats/cid'
+import { getETag } from '../src/utils/get-e-tag.js'
+
+const cidString = 'QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'
+const testCID = CID.parse(cidString)
+
+describe('getETag', () => {
+  it('CID eTag', () => {
+    expect(getETag({ cid: testCID, weak: true })).to.equal(`W/"${cidString}"`)
+    expect(getETag({ cid: testCID, weak: false })).to.equal(`"${cidString}"`)
+  })
+
+  it('should return ETag with CID and format suffix', () => {
+    expect(getETag({ cid: testCID, reqFormat: 'raw' })).to.equal(`"${cidString}.raw"`)
+    expect(getETag({ cid: testCID, reqFormat: 'json' })).to.equal(`"${cidString}.json"`)
+  })
+
+  it('should return ETag with CID and range suffix', () => {
+    expect(getETag({ cid: testCID, weak: true, reqFormat: 'car', rangeStart: 10, rangeEnd: 20 })).to.equal(`W/"${cidString}.car.10-20"`)
+    expect(getETag({ cid: testCID, weak: false, reqFormat: 'car', rangeStart: 10, rangeEnd: 20 })).to.equal(`"${cidString}.car.10-20"`)
+  })
+
+  it('should return ETag with CID, format and range suffix', () => {
+    expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: 10, rangeEnd: 20 })).to.equal(`"${cidString}.raw.10-20"`)
+  })
+
+  it('should handle undefined rangeStart and rangeEnd', () => {
+    expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: undefined, rangeEnd: undefined })).to.equal(`"${cidString}.raw"`)
+    expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: 55, rangeEnd: undefined })).to.equal(`"${cidString}.raw.55-N"`)
+    expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: undefined, rangeEnd: 77 })).to.equal(`"${cidString}.raw.0-77"`)
+  })
+})