From 261c202afebd76aed706623f7cc76ab0c5627344 Mon Sep 17 00:00:00 2001 From: Hiro DevOps <45208873+blockstack-devops@users.noreply.github.com> Date: Thu, 2 Oct 2025 13:18:30 -0400 Subject: [PATCH 1/2] [Snyk] Security upgrade undici from 5.28.3 to 5.29.0 (#307) fix: package.json & package-lock.json to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-UNDICI-10176064 Co-authored-by: snyk-bot --- package-lock.json | 18 +++++------------- package.json | 2 +- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index c466c1e..810b42c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,7 @@ "postgres": "^3.3.1", "sharp": "^0.33.3", "stacks-encoding-native-js": "^1.0.0", - "undici": "^5.12.0" + "undici": "^5.29.0" }, "devDependencies": { "@babel/core": "^7.23.6", @@ -12657,15 +12657,6 @@ "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==", "dev": true }, - "node_modules/redoc-cli/node_modules/@types/mkdirp": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.1.tgz", - "integrity": "sha512-HkGSK7CGAXncr8Qn/0VqNtExEE+PHMWb+qlR1faHMao7ng6P3tAaoWWBMdva0gL5h4zprjIO89GJOLXsMcDm1Q==", - "extraneous": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/redoc-cli/node_modules/@types/node": { "version": "15.12.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.2.tgz", @@ -17835,9 +17826,10 @@ } }, "node_modules/undici": { - "version": "5.28.3", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.3.tgz", - "integrity": "sha512-3ItfzbrhDlINjaP0duwnNsKpDQk3acHI3gVJ1z4fmwMK31k5G9OVIAMLSIaP6w4FaGkaAkN6zaQO9LUvZ1t7VA==", + "version": "5.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz", + "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==", + "license": "MIT", "dependencies": { "@fastify/busboy": "^2.0.0" }, diff --git a/package.json b/package.json index ee48a4b..10a2ee3 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,6 @@ "postgres": "^3.3.1", "sharp": "^0.33.3", "stacks-encoding-native-js": "^1.0.0", - "undici": "^5.12.0" + "undici": "^5.29.0" } } From d04012f9187a8885bab5046bb626010f681f57fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Thu, 2 Oct 2025 14:19:19 -0600 Subject: [PATCH 2/2] fix: replace hard coded IPFS gateways with configurable gateway (#319) * fix: replace hard coded public gateways * fix: make replace list configurable * fix: fetch headers * fix: tests --- src/env.ts | 20 ++++- src/token-processor/images/image-cache.ts | 38 ++++----- .../queue/job/process-token-job.ts | 4 +- src/token-processor/util/metadata-helpers.ts | 80 ++++++++++++++----- tests/token-queue/metadata-helpers.test.ts | 35 +++++++- 5 files changed, 128 insertions(+), 49 deletions(-) diff --git a/src/env.ts b/src/env.ts index 51a2a36..2b16d09 100644 --- a/src/env.ts +++ b/src/env.ts @@ -130,12 +130,26 @@ const schema = Type.Object({ METADATA_FETCH_MAX_REDIRECTIONS: Type.Number({ default: 5 }), /** - * Base URL for a public gateway which will provide access to all IPFS resources. Defaults to - * `https://cloudflare-ipfs.com`. + * Base URL for a public gateway which will provide access to all IPFS resources when metadata + * URLs use the `ipfs:` or `ipns:` protocol schemes. Defaults to `https://cloudflare-ipfs.com`. */ PUBLIC_GATEWAY_IPFS: Type.String({ default: 'https://cloudflare-ipfs.com' }), /** - * Base URL for a public gateway which will provide access to all Arweave resources. Defaults to + * Extra header key to add to the request when fetching metadata from the configured IPFS gateway, + * for example if authentication is required. Must be in the form 'Header: Value'. + */ + PUBLIC_GATEWAY_IPFS_EXTRA_HEADER: Type.Optional(Type.String()), + /** + * List of public IPFS gateways that will be replaced with the value of `PUBLIC_GATEWAY_IPFS` + * whenever a metadata URL has these gateways hard coded in `http:` or `https:` URLs. + */ + PUBLIC_GATEWAY_IPFS_REPLACED: Type.String({ + default: 'ipfs.io,dweb.link,gateway.pinata.cloud,cloudflare-ipfs.com,infura-ipfs.io', + }), + + /** + * Base URL for a public gateway which will provide access to all Arweave resources when metadata + * URLs use the `ar:` protocol scheme. Defaults to * `https://arweave.net`. */ PUBLIC_GATEWAY_ARWEAVE: Type.String({ default: 'https://arweave.net' }), diff --git a/src/token-processor/images/image-cache.ts b/src/token-processor/images/image-cache.ts index 1be8ed3..fbff160 100644 --- a/src/token-processor/images/image-cache.ts +++ b/src/token-processor/images/image-cache.ts @@ -1,5 +1,5 @@ import { ENV } from '../../env'; -import { parseDataUrl, getFetchableDecentralizedStorageUrl } from '../util/metadata-helpers'; +import { parseDataUrl, getFetchableMetadataUrl } from '../util/metadata-helpers'; import { logger } from '@hirosystems/api-toolkit'; import { PgStore } from '../../pg/pg-store'; import { Readable } from 'node:stream'; @@ -16,7 +16,6 @@ import { } from '../util/errors'; import { pipeline } from 'node:stream/promises'; import { Storage } from '@google-cloud/storage'; -import { RetryableJobError } from '../queue/errors'; /** Saves an image provided via a `data:` uri string to disk for processing. */ function convertDataImage(uri: string, tmpPath: string): string { @@ -33,10 +32,15 @@ function convertDataImage(uri: string, tmpPath: string): string { return filePath; } -async function downloadImage(imgUrl: string, tmpPath: string): Promise { +async function downloadImage( + imgUrl: string, + tmpPath: string, + headers?: Record +): Promise { return new Promise((resolve, reject) => { const filePath = `${tmpPath}/image`; fetch(imgUrl, { + headers, dispatcher: new Agent({ headersTimeout: ENV.METADATA_FETCH_TIMEOUT_MS, bodyTimeout: ENV.METADATA_FETCH_TIMEOUT_MS, @@ -109,11 +113,11 @@ async function transformImage(filePath: string, resize: boolean = false): Promis * For a list of configuration options, see `env.ts`. */ export async function processImageCache( - imgUrl: string, + rawImgUrl: string, contractPrincipal: string, tokenNumber: bigint ): Promise { - logger.info(`ImageCache processing token ${contractPrincipal} (${tokenNumber}) at ${imgUrl}`); + logger.info(`ImageCache processing token ${contractPrincipal} (${tokenNumber}) at ${rawImgUrl}`); try { const gcs = new Storage(); const gcsBucket = ENV.IMAGE_CACHE_GCS_BUCKET_NAME as string; @@ -121,10 +125,11 @@ export async function processImageCache( const tmpPath = `tmp/${contractPrincipal}_${tokenNumber}`; fs.mkdirSync(tmpPath, { recursive: true }); let original: string; - if (imgUrl.startsWith('data:')) { - original = convertDataImage(imgUrl, tmpPath); + if (rawImgUrl.startsWith('data:')) { + original = convertDataImage(rawImgUrl, tmpPath); } else { - original = await downloadImage(imgUrl, tmpPath); + const { url: httpUrl, fetchHeaders } = getFetchableMetadataUrl(rawImgUrl); + original = await downloadImage(httpUrl.toString(), tmpPath, fetchHeaders); } const image1 = await transformImage(original); @@ -152,10 +157,10 @@ export async function processImageCache( typeError.cause instanceof errors.BodyTimeoutError || typeError.cause instanceof errors.ConnectTimeoutError ) { - throw new ImageTimeoutError(new URL(imgUrl)); + throw new ImageTimeoutError(new URL(rawImgUrl)); } if (typeError.cause instanceof errors.ResponseExceededMaxSizeError) { - throw new ImageSizeExceededError(`ImageCache image too large: ${imgUrl}`); + throw new ImageSizeExceededError(`ImageCache image too large: ${rawImgUrl}`); } if ((typeError.cause as any).toString().includes('ECONNRESET')) { throw new ImageHttpError(`ImageCache server connection interrupted`, typeError); @@ -165,17 +170,6 @@ export async function processImageCache( } } -/** - * Converts a raw image URI from metadata into a fetchable URL. - * @param uri - Original image URI - * @returns Normalized URL string - */ -export function normalizeImageUri(uri: string): string { - if (uri.startsWith('data:')) return uri; - const fetchableUrl = getFetchableDecentralizedStorageUrl(uri); - return fetchableUrl.toString(); -} - export async function reprocessTokenImageCache( db: PgStore, contractPrincipal: string, @@ -186,7 +180,7 @@ export async function reprocessTokenImageCache( for (const token of imageUris) { try { const [cached, thumbnail] = await processImageCache( - getFetchableDecentralizedStorageUrl(token.image).toString(), + getFetchableMetadataUrl(token.image).toString(), contractPrincipal, BigInt(token.token_number) ); diff --git a/src/token-processor/queue/job/process-token-job.ts b/src/token-processor/queue/job/process-token-job.ts index a454f17..4d460fd 100644 --- a/src/token-processor/queue/job/process-token-job.ts +++ b/src/token-processor/queue/job/process-token-job.ts @@ -12,7 +12,7 @@ import { StacksNodeRpcClient } from '../../stacks-node/stacks-node-rpc-client'; import { SmartContractClarityError, TooManyRequestsHttpError } from '../../util/errors'; import { fetchAllMetadataLocalesFromBaseUri, - getFetchableDecentralizedStorageUrl, + getFetchableMetadataUrl, getTokenSpecificUri, } from '../../util/metadata-helpers'; import { RetryableJobError } from '../errors'; @@ -194,7 +194,7 @@ export class ProcessTokenJob extends Job { return; } // Before we return the uri, check if its fetchable hostname is not already rate limited. - const fetchable = getFetchableDecentralizedStorageUrl(uri); + const fetchable = getFetchableMetadataUrl(uri).url; const rateLimitedHost = await this.db.getRateLimitedHost({ hostname: fetchable.hostname }); if (rateLimitedHost) { const retryAfter = Date.parse(rateLimitedHost.retry_after); diff --git a/src/token-processor/util/metadata-helpers.ts b/src/token-processor/util/metadata-helpers.ts index 53a942a..b540937 100644 --- a/src/token-processor/util/metadata-helpers.ts +++ b/src/token-processor/util/metadata-helpers.ts @@ -19,7 +19,7 @@ import { UndiciCauseTypeError, } from './errors'; import { RetryableJobError } from '../queue/errors'; -import { normalizeImageUri, processImageCache } from '../images/image-cache'; +import { processImageCache } from '../images/image-cache'; import { RawMetadataLocale, RawMetadataLocalizationCType, @@ -40,6 +40,22 @@ const METADATA_FETCH_HTTP_AGENT = new Agent({ }, }); +/** + * A metadata URL that was analyzed and normalized into a fetchable URL. Specifies the URL, the + * gateway type, and any extra headers that may be required to fetch the metadata. + */ +export type FetchableMetadataUrl = { + url: URL; + gateway: 'ipfs' | 'arweave' | null; + fetchHeaders?: Record; +}; + +/** + * List of public IPFS gateways that will be replaced with the value of `ENV.PUBLIC_GATEWAY_IPFS` + * whenever a metadata URL has these gateways hard coded in `http:` or `https:` URLs. + */ +const PUBLIC_GATEWAY_IPFS_REPLACED = ENV.PUBLIC_GATEWAY_IPFS_REPLACED.split(','); + /** * Fetches all the localized metadata JSONs for a token. First, it downloads the default metadata * JSON and parses it looking for other localizations. If those are found, each of them is then @@ -172,9 +188,8 @@ async function parseMetadataForInsertion( let cachedImage: string | undefined; let cachedThumbnailImage: string | undefined; if (image && typeof image === 'string' && ENV.IMAGE_CACHE_PROCESSOR_ENABLED) { - const normalizedUrl = normalizeImageUri(image); [cachedImage, cachedThumbnailImage] = await processImageCache( - normalizedUrl, + image, contract.principal, token.token_number ); @@ -243,7 +258,8 @@ async function parseMetadataForInsertion( export async function fetchMetadata( httpUrl: URL, contract_principal: string, - token_number: bigint + token_number: bigint, + headers?: Record ): Promise { const url = httpUrl.toString(); try { @@ -251,6 +267,7 @@ export async function fetchMetadata( const result = await request(url, { method: 'GET', throwOnError: true, + headers, dispatcher: // Disable during tests so we can inject a global mock agent. process.env.NODE_ENV === 'test' ? undefined : METADATA_FETCH_HTTP_AGENT, @@ -304,10 +321,13 @@ export async function getMetadataFromUri( return parseJsonMetadata(token_uri, content); } - // Support HTTP/S URLs otherwise - const httpUrl = getFetchableDecentralizedStorageUrl(token_uri); + // Support HTTP/S URLs otherwise. + // Transform the URL to use a public gateway if necessary. + const { url: httpUrl, fetchHeaders } = getFetchableMetadataUrl(token_uri); const urlStr = httpUrl.toString(); - const content = await fetchMetadata(httpUrl, contract_principal, token_number); + + // Fetch the metadata. + const content = await fetchMetadata(httpUrl, contract_principal, token_number, fetchHeaders); return parseJsonMetadata(urlStr, content); } @@ -332,31 +352,55 @@ function parseJsonMetadata(url: string, content?: string): RawMetadata { /** * Helper method for creating http/s url for supported protocols. - * * URLs with `http` or `https` protocols are returned as-is. + * * URLs with `http` or `https` protocols are returned as-is. But if they are public IPFS gateways, + * they are replaced with `ENV.PUBLIC_GATEWAY_IPFS`. * * URLs with `ipfs` or `ipns` protocols are returned with as an `https` url using a public IPFS * gateway. * * URLs with `ar` protocols are returned as `https` using a public Arweave gateway. * @param uri - URL to convert * @returns Fetchable URL */ -export function getFetchableDecentralizedStorageUrl(uri: string): URL { +export function getFetchableMetadataUrl(uri: string): FetchableMetadataUrl { try { const parsedUri = new URL(uri); - if (parsedUri.protocol === 'http:' || parsedUri.protocol === 'https:') return parsedUri; - if (parsedUri.protocol === 'ipfs:') { + const result: FetchableMetadataUrl = { + url: parsedUri, + gateway: null, + fetchHeaders: undefined, + }; + if (parsedUri.protocol === 'http:' || parsedUri.protocol === 'https:') { + // If this is a known public IPFS gateway, replace it with `ENV.PUBLIC_GATEWAY_IPFS`. + if (PUBLIC_GATEWAY_IPFS_REPLACED.includes(parsedUri.hostname)) { + result.url = new URL(`${ENV.PUBLIC_GATEWAY_IPFS}${parsedUri.pathname}`); + result.gateway = 'ipfs'; + } else { + result.url = parsedUri; + } + } else if (parsedUri.protocol === 'ipfs:') { const host = parsedUri.host === 'ipfs' ? 'ipfs' : `ipfs/${parsedUri.host}`; - return new URL(`${ENV.PUBLIC_GATEWAY_IPFS}/${host}${parsedUri.pathname}`); - } - if (parsedUri.protocol === 'ipns:') { - return new URL(`${ENV.PUBLIC_GATEWAY_IPFS}/${parsedUri.host}${parsedUri.pathname}`); + result.url = new URL(`${ENV.PUBLIC_GATEWAY_IPFS}/${host}${parsedUri.pathname}`); + result.gateway = 'ipfs'; + } else if (parsedUri.protocol === 'ipns:') { + result.url = new URL(`${ENV.PUBLIC_GATEWAY_IPFS}/${parsedUri.host}${parsedUri.pathname}`); + result.gateway = 'ipfs'; + } else if (parsedUri.protocol === 'ar:') { + result.url = new URL(`${ENV.PUBLIC_GATEWAY_ARWEAVE}/${parsedUri.host}${parsedUri.pathname}`); + result.gateway = 'arweave'; + } else { + throw new MetadataParseError(`Unsupported uri protocol: ${uri}`); } - if (parsedUri.protocol === 'ar:') { - return new URL(`${ENV.PUBLIC_GATEWAY_ARWEAVE}/${parsedUri.host}${parsedUri.pathname}`); + + if (result.gateway === 'ipfs' && ENV.PUBLIC_GATEWAY_IPFS_EXTRA_HEADER) { + const [key, value] = ENV.PUBLIC_GATEWAY_IPFS_EXTRA_HEADER.split(':'); + result.fetchHeaders = { + [key.trim()]: value.trim(), + }; } + + return result; } catch (error) { throw new MetadataParseError(`Invalid uri: ${uri}`); } - throw new MetadataParseError(`Unsupported uri protocol: ${uri}`); } export function parseDataUrl( diff --git a/tests/token-queue/metadata-helpers.test.ts b/tests/token-queue/metadata-helpers.test.ts index 1eb8424..fb9c78c 100644 --- a/tests/token-queue/metadata-helpers.test.ts +++ b/tests/token-queue/metadata-helpers.test.ts @@ -2,7 +2,7 @@ import { MockAgent, setGlobalDispatcher } from 'undici'; import { ENV } from '../../src/env'; import { MetadataHttpError, MetadataParseError } from '../../src/token-processor/util/errors'; import { - getFetchableDecentralizedStorageUrl, + getFetchableMetadataUrl, getMetadataFromUri, getTokenSpecificUri, fetchMetadata, @@ -209,19 +209,46 @@ describe('Metadata Helpers', () => { test('get fetchable URLs', () => { ENV.PUBLIC_GATEWAY_IPFS = 'https://cloudflare-ipfs.com'; ENV.PUBLIC_GATEWAY_ARWEAVE = 'https://arweave.net'; + ENV.PUBLIC_GATEWAY_IPFS_EXTRA_HEADER = 'Authorization: Bearer 1234567890'; + const arweave = 'ar://II4z2ziYyqG7-kWDa98lWGfjxRdYOx9Zdld9P_I_kzE/9731.json'; - expect(getFetchableDecentralizedStorageUrl(arweave).toString()).toBe( + const fetch1 = getFetchableMetadataUrl(arweave); + expect(fetch1.url.toString()).toBe( 'https://arweave.net/II4z2ziYyqG7-kWDa98lWGfjxRdYOx9Zdld9P_I_kzE/9731.json' ); + expect(fetch1.gateway).toBe('arweave'); + expect(fetch1.fetchHeaders).toBeUndefined(); + const ipfs = 'ipfs://ipfs/bafybeifwoqwdhs5djtx6vopvuwfcdrqeuecayp5wzpzjylxycejnhtrhgu/vague_art_paintings/vague_art_paintings_6_metadata.json'; - expect(getFetchableDecentralizedStorageUrl(ipfs).toString()).toBe( + const fetch2 = getFetchableMetadataUrl(ipfs); + expect(fetch2.url.toString()).toBe( 'https://cloudflare-ipfs.com/ipfs/bafybeifwoqwdhs5djtx6vopvuwfcdrqeuecayp5wzpzjylxycejnhtrhgu/vague_art_paintings/vague_art_paintings_6_metadata.json' ); + expect(fetch2.gateway).toBe('ipfs'); + expect(fetch2.fetchHeaders).toEqual({ Authorization: 'Bearer 1234567890' }); + const ipfs2 = 'ipfs://QmYCnfeseno5cLpC75rmy6LQhsNYQCJabiuwqNUXMaA3Fo/1145.png'; - expect(getFetchableDecentralizedStorageUrl(ipfs2).toString()).toBe( + const fetch3 = getFetchableMetadataUrl(ipfs2); + expect(fetch3.url.toString()).toBe( 'https://cloudflare-ipfs.com/ipfs/QmYCnfeseno5cLpC75rmy6LQhsNYQCJabiuwqNUXMaA3Fo/1145.png' ); + expect(fetch3.gateway).toBe('ipfs'); + expect(fetch3.fetchHeaders).toEqual({ Authorization: 'Bearer 1234567890' }); + + const ipfs3 = 'https://ipfs.io/ipfs/QmYCnfeseno5cLpC75rmy6LQhsNYQCJabiuwqNUXMaA3Fo/1145.png'; + const fetch4 = getFetchableMetadataUrl(ipfs3); + expect(fetch4.url.toString()).toBe( + 'https://cloudflare-ipfs.com/ipfs/QmYCnfeseno5cLpC75rmy6LQhsNYQCJabiuwqNUXMaA3Fo/1145.png' + ); + expect(fetch4.gateway).toBe('ipfs'); + expect(fetch4.fetchHeaders).toEqual({ Authorization: 'Bearer 1234567890' }); + + const http = 'https://test.io/1.json'; + const fetch5 = getFetchableMetadataUrl(http); + expect(fetch5.url.toString()).toBe(http); + expect(fetch5.gateway).toBeNull(); + expect(fetch5.fetchHeaders).toBeUndefined(); }); test('replace URI string tokens', () => {