From f18b0a7d9c409dfa713a6eb314a56558606b028a Mon Sep 17 00:00:00 2001 From: dcs-soni <52232900+dcs-soni@users.noreply.github.com> Date: Mon, 29 Sep 2025 18:42:00 +0530 Subject: [PATCH 1/3] fix: improve image URL parsing robustness across providers --- core/llm/llms/Anthropic.ts | 9 ++++++++- core/llm/llms/Bedrock.ts | 7 ++++++- core/llm/llms/Gemini.ts | 9 ++++++++- core/llm/llms/Ollama.ts | 10 +++++++++- packages/openai-adapters/src/apis/Anthropic.ts | 9 ++++++++- packages/openai-adapters/src/apis/Bedrock.ts | 9 ++++++--- 6 files changed, 45 insertions(+), 8 deletions(-) diff --git a/core/llm/llms/Anthropic.ts b/core/llm/llms/Anthropic.ts index 62d963b99e3..833df4d7723 100644 --- a/core/llm/llms/Anthropic.ts +++ b/core/llm/llms/Anthropic.ts @@ -107,7 +107,14 @@ class Anthropic extends BaseLLM { source: { type: "base64", media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url), - data: part.imageUrl.url.split(",")[1], + data: (() => { + const urlParts = part.imageUrl.url.split(","); + if(urlParts.length < 2) { + throw new Error("Invalid data URL format: missing comma separator"); + } + const [...base64Parts] = urlParts; + return base64Parts.join(","); + })(), }, }; }); diff --git a/core/llm/llms/Bedrock.ts b/core/llm/llms/Bedrock.ts index ea47b63ae1f..55032abb58a 100644 --- a/core/llm/llms/Bedrock.ts +++ b/core/llm/llms/Bedrock.ts @@ -546,7 +546,12 @@ class Bedrock extends BaseLLM { blocks.push({ text: part.text }); } else if (part.type === "imageUrl" && part.imageUrl) { try { - const [mimeType, base64Data] = part.imageUrl.url.split(","); + const urlParts = part.imageUrl.url.split(","); + if(urlParts.length < 2) { + throw new Error("Invalid data URL format: missing comma separator"); + } + const [mimeType, ...base64Parts] = urlParts; + const base64Data = base64Parts.join(","); const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; if ( format === ImageFormat.JPEG || diff --git a/core/llm/llms/Gemini.ts b/core/llm/llms/Gemini.ts index 7caf2db7d75..791b882ec10 100644 --- a/core/llm/llms/Gemini.ts +++ b/core/llm/llms/Gemini.ts @@ -191,7 +191,14 @@ class Gemini extends BaseLLM { : { inlineData: { mimeType: "image/jpeg", - data: part.imageUrl?.url.split(",")[1], + data: (() => { + const urlParts = part.imageUrl.url.split(","); + if (urlParts.length < 2) { + throw new Error("Invalid data URL format: missing comma separator"); + } + const [, ...base64Parts] = urlParts; + return base64Parts.join(","); + })(), }, }; } diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index 0a239fce909..0ae06e7ce51 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -303,7 +303,15 @@ class Ollama extends BaseLLM implements ModelInstaller { const images: string[] = []; message.content.forEach((part) => { if (part.type === "imageUrl" && part.imageUrl) { - const image = part.imageUrl?.url.split(",").at(-1); + const image = (() => { + if (!part.imageUrl?.url) return undefined; + const urlParts = part.imageUrl.url.split(","); + if (urlParts.length < 2) { + throw new Error("Invalid data URL format: missing comma separator"); + } + const [, ...base64Parts] = urlParts; + return base64Parts.join(","); + })(); if (image) { images.push(image); } diff --git a/packages/openai-adapters/src/apis/Anthropic.ts b/packages/openai-adapters/src/apis/Anthropic.ts index b2663896f72..745c407ddd5 100644 --- a/packages/openai-adapters/src/apis/Anthropic.ts +++ b/packages/openai-adapters/src/apis/Anthropic.ts @@ -199,7 +199,14 @@ export class AnthropicApi implements BaseLlmApi { source: { type: "base64", media_type: getAnthropicMediaTypeFromDataUrl(dataUrl), - data: dataUrl.split(",")[1], + data: (() => { + const urlParts = dataUrl.split(","); + if (urlParts.length < 2) { + throw new Error("Invalid data URL format: missing comma separator"); + } + const [, ...base64Parts] = urlParts; + return base64Parts.join(","); + })(), }, }); } diff --git a/packages/openai-adapters/src/apis/Bedrock.ts b/packages/openai-adapters/src/apis/Bedrock.ts index cf5588d686d..c1699995f16 100644 --- a/packages/openai-adapters/src/apis/Bedrock.ts +++ b/packages/openai-adapters/src/apis/Bedrock.ts @@ -135,9 +135,12 @@ export class BedrockApi implements BaseLlmApi { case "image_url": default: try { - const [mimeType, base64Data] = ( - part as ChatCompletionContentPartImage - ).image_url.url.split(","); + const urlParts = (part as ChatCompletionContentPartImage).image_url.url.split(","); + if(urlParts.length < 2) { + throw new Error("Invalid data URL format: missing comma separator"); + } + const [mimeType, ...base64Parts] = urlParts; + const base64Data = base64Parts.join(","); const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; if ( format === ImageFormat.JPEG || From 3436b3ecbb3955239e5cda9d0ad7f0c7d0bcba09 Mon Sep 17 00:00:00 2001 From: dcs-soni <52232900+dcs-soni@users.noreply.github.com> Date: Thu, 2 Oct 2025 14:48:28 +0530 Subject: [PATCH 2/3] refactor: extract url parsing logic to utilities --- core/llm/llms/Anthropic.ts | 10 ++---- core/llm/llms/Bedrock.ts | 8 ++--- core/llm/llms/Gemini.ts | 10 ++---- core/llm/llms/Ollama.ts | 11 ++----- core/util/url.ts | 31 +++++++++++++++++++ .../openai-adapters/src/apis/Anthropic.ts | 10 ++---- packages/openai-adapters/src/apis/Bedrock.ts | 11 +++---- 7 files changed, 46 insertions(+), 45 deletions(-) diff --git a/core/llm/llms/Anthropic.ts b/core/llm/llms/Anthropic.ts index 833df4d7723..b7a3d8ec23b 100644 --- a/core/llm/llms/Anthropic.ts +++ b/core/llm/llms/Anthropic.ts @@ -28,6 +28,7 @@ import { } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { DEFAULT_REASONING_TOKENS } from "../constants.js"; import { BaseLLM } from "../index.js"; @@ -107,14 +108,7 @@ class Anthropic extends BaseLLM { source: { type: "base64", media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url), - data: (() => { - const urlParts = part.imageUrl.url.split(","); - if(urlParts.length < 2) { - throw new Error("Invalid data URL format: missing comma separator"); - } - const [...base64Parts] = urlParts; - return base64Parts.join(","); - })(), + data: extractBase64FromDataUrl(part.imageUrl.url), }, }; }); diff --git a/core/llm/llms/Bedrock.ts b/core/llm/llms/Bedrock.ts index 55032abb58a..d51093db174 100644 --- a/core/llm/llms/Bedrock.ts +++ b/core/llm/llms/Bedrock.ts @@ -21,6 +21,7 @@ import type { CompletionOptions } from "../../index.js"; import { ChatMessage, Chunk, LLMOptions, MessageContent } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { parseDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; import { PROVIDER_TOOL_SUPPORT } from "../toolSupport.js"; import { getSecureID } from "../utils/getSecureID.js"; @@ -546,12 +547,7 @@ class Bedrock extends BaseLLM { blocks.push({ text: part.text }); } else if (part.type === "imageUrl" && part.imageUrl) { try { - const urlParts = part.imageUrl.url.split(","); - if(urlParts.length < 2) { - throw new Error("Invalid data URL format: missing comma separator"); - } - const [mimeType, ...base64Parts] = urlParts; - const base64Data = base64Parts.join(","); + const { mimeType, base64Data } = parseDataUrl(part.imageUrl.url); const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; if ( format === ImageFormat.JPEG || diff --git a/core/llm/llms/Gemini.ts b/core/llm/llms/Gemini.ts index 791b882ec10..f2b19dbdcb9 100644 --- a/core/llm/llms/Gemini.ts +++ b/core/llm/llms/Gemini.ts @@ -11,6 +11,7 @@ import { } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; import { GeminiChatContent, @@ -191,14 +192,7 @@ class Gemini extends BaseLLM { : { inlineData: { mimeType: "image/jpeg", - data: (() => { - const urlParts = part.imageUrl.url.split(","); - if (urlParts.length < 2) { - throw new Error("Invalid data URL format: missing comma separator"); - } - const [, ...base64Parts] = urlParts; - return base64Parts.join(","); - })(), + data: part.imageUrl?.url ? extractBase64FromDataUrl(part.imageUrl.url) : "" }, }; } diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index 0ae06e7ce51..e29bef7147f 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -13,6 +13,7 @@ import { } from "../../index.js"; import { renderChatMessage } from "../../util/messageContent.js"; import { getRemoteModelInfo } from "../../util/ollamaHelper.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; type OllamaChatMessage = { @@ -303,15 +304,7 @@ class Ollama extends BaseLLM implements ModelInstaller { const images: string[] = []; message.content.forEach((part) => { if (part.type === "imageUrl" && part.imageUrl) { - const image = (() => { - if (!part.imageUrl?.url) return undefined; - const urlParts = part.imageUrl.url.split(","); - if (urlParts.length < 2) { - throw new Error("Invalid data URL format: missing comma separator"); - } - const [, ...base64Parts] = urlParts; - return base64Parts.join(","); - })(); + const image = part.imageUrl?.url ? extractBase64FromDataUrl(part.imageUrl.url) : undefined if (image) { images.push(image); } diff --git a/core/util/url.ts b/core/util/url.ts index 83e0edcba14..ae36ac1d6d4 100644 --- a/core/util/url.ts +++ b/core/util/url.ts @@ -9,3 +9,34 @@ export function canParseUrl(url: string): boolean { return false; } } + +export function parseDataUrl(dataUrl: string): { + mimeType: string; + base64Data: string +} { + const urlParts = dataUrl.split(","); + + if(urlParts.length < 2 ) { + throw new Error("Invalid data URL format: expected 'data:type;base64,data' format") + } + + const [mimeType, ...base64Parts] = urlParts; + const base64Data = base64Parts.join(","); + + return { mimeType, base64Data } +} + +export function extractBase64FromDataUrl(dataUrl: string): string { + return parseDataUrl(dataUrl).base64Data +} + +export function safeSplit(input: string, delimiter: string, expectedParts: number, errorContext: string = "input") : string[] { + + const parts = input.split(delimiter); + + if(parts.length !== expectedParts) { + throw new Error(`Invalid ${errorContext} format: expected ${expectedParts} parts separated by "${delimiter}", got ${parts.length}`) + } + + return parts; +} \ No newline at end of file diff --git a/packages/openai-adapters/src/apis/Anthropic.ts b/packages/openai-adapters/src/apis/Anthropic.ts index 745c407ddd5..84880e07dc7 100644 --- a/packages/openai-adapters/src/apis/Anthropic.ts +++ b/packages/openai-adapters/src/apis/Anthropic.ts @@ -33,6 +33,7 @@ import { usageChatChunk, } from "../util.js"; import { EMPTY_CHAT_COMPLETION } from "../util/emptyChatCompletion.js"; +import { extractBase64FromDataUrl } from "../../../../core/util/url.js" import { safeParseArgs } from "../util/parseArgs.js"; import { CACHING_STRATEGIES, @@ -199,14 +200,7 @@ export class AnthropicApi implements BaseLlmApi { source: { type: "base64", media_type: getAnthropicMediaTypeFromDataUrl(dataUrl), - data: (() => { - const urlParts = dataUrl.split(","); - if (urlParts.length < 2) { - throw new Error("Invalid data URL format: missing comma separator"); - } - const [, ...base64Parts] = urlParts; - return base64Parts.join(","); - })(), + data: extractBase64FromDataUrl(dataUrl); }, }); } diff --git a/packages/openai-adapters/src/apis/Bedrock.ts b/packages/openai-adapters/src/apis/Bedrock.ts index c1699995f16..d2c06c9ca7e 100644 --- a/packages/openai-adapters/src/apis/Bedrock.ts +++ b/packages/openai-adapters/src/apis/Bedrock.ts @@ -30,7 +30,9 @@ import { } from "openai/resources/index"; import { fromNodeProviderChain } from "@aws-sdk/credential-providers"; + import { fromStatic } from "@aws-sdk/token-providers"; +import { parseDataUrl } from "../../../../core/util/url.js"; import { BedrockConfig } from "../types.js"; import { chatChunk, chatChunkFromDelta, embedding, rerank } from "../util.js"; import { safeParseArgs } from "../util/parseArgs.js"; @@ -135,12 +137,9 @@ export class BedrockApi implements BaseLlmApi { case "image_url": default: try { - const urlParts = (part as ChatCompletionContentPartImage).image_url.url.split(","); - if(urlParts.length < 2) { - throw new Error("Invalid data URL format: missing comma separator"); - } - const [mimeType, ...base64Parts] = urlParts; - const base64Data = base64Parts.join(","); + const { mimeType, base64Data } = parseDataUrl( + (part as ChatCompletionContentPartImage).image_url.url, + ); const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; if ( format === ImageFormat.JPEG || From fb90ae7700f364c5cd2b6b3a3fa67405304a8cb2 Mon Sep 17 00:00:00 2001 From: dcs-soni <52232900+dcs-soni@users.noreply.github.com> Date: Tue, 7 Oct 2025 09:24:36 +0530 Subject: [PATCH 3/3] fix prettier ci checks --- core/llm/llms/Gemini.ts | 4 ++- core/llm/llms/Ollama.ts | 4 ++- core/util/url.ts | 30 ++++++++++++------- .../openai-adapters/src/apis/Anthropic.ts | 4 +-- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/core/llm/llms/Gemini.ts b/core/llm/llms/Gemini.ts index f2b19dbdcb9..d63bc5e9d36 100644 --- a/core/llm/llms/Gemini.ts +++ b/core/llm/llms/Gemini.ts @@ -192,7 +192,9 @@ class Gemini extends BaseLLM { : { inlineData: { mimeType: "image/jpeg", - data: part.imageUrl?.url ? extractBase64FromDataUrl(part.imageUrl.url) : "" + data: part.imageUrl?.url + ? extractBase64FromDataUrl(part.imageUrl.url) + : "", }, }; } diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index e29bef7147f..96f169af045 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -304,7 +304,9 @@ class Ollama extends BaseLLM implements ModelInstaller { const images: string[] = []; message.content.forEach((part) => { if (part.type === "imageUrl" && part.imageUrl) { - const image = part.imageUrl?.url ? extractBase64FromDataUrl(part.imageUrl.url) : undefined + const image = part.imageUrl?.url + ? extractBase64FromDataUrl(part.imageUrl.url) + : undefined; if (image) { images.push(image); } diff --git a/core/util/url.ts b/core/util/url.ts index ae36ac1d6d4..2c83e8f5999 100644 --- a/core/util/url.ts +++ b/core/util/url.ts @@ -12,31 +12,39 @@ export function canParseUrl(url: string): boolean { export function parseDataUrl(dataUrl: string): { mimeType: string; - base64Data: string + base64Data: string; } { const urlParts = dataUrl.split(","); - if(urlParts.length < 2 ) { - throw new Error("Invalid data URL format: expected 'data:type;base64,data' format") + if (urlParts.length < 2) { + throw new Error( + "Invalid data URL format: expected 'data:type;base64,data' format", + ); } const [mimeType, ...base64Parts] = urlParts; const base64Data = base64Parts.join(","); - - return { mimeType, base64Data } + + return { mimeType, base64Data }; } export function extractBase64FromDataUrl(dataUrl: string): string { - return parseDataUrl(dataUrl).base64Data + return parseDataUrl(dataUrl).base64Data; } -export function safeSplit(input: string, delimiter: string, expectedParts: number, errorContext: string = "input") : string[] { - +export function safeSplit( + input: string, + delimiter: string, + expectedParts: number, + errorContext: string = "input", +): string[] { const parts = input.split(delimiter); - if(parts.length !== expectedParts) { - throw new Error(`Invalid ${errorContext} format: expected ${expectedParts} parts separated by "${delimiter}", got ${parts.length}`) + if (parts.length !== expectedParts) { + throw new Error( + `Invalid ${errorContext} format: expected ${expectedParts} parts separated by "${delimiter}", got ${parts.length}`, + ); } return parts; -} \ No newline at end of file +} diff --git a/packages/openai-adapters/src/apis/Anthropic.ts b/packages/openai-adapters/src/apis/Anthropic.ts index 84880e07dc7..25a9c218faf 100644 --- a/packages/openai-adapters/src/apis/Anthropic.ts +++ b/packages/openai-adapters/src/apis/Anthropic.ts @@ -25,6 +25,7 @@ import { CompletionUsage, } from "openai/resources/index"; import { ChatCompletionCreateParams } from "openai/resources/index.js"; +import { extractBase64FromDataUrl } from "../../../../core/util/url.js"; import { AnthropicConfig } from "../types.js"; import { chatChunk, @@ -33,7 +34,6 @@ import { usageChatChunk, } from "../util.js"; import { EMPTY_CHAT_COMPLETION } from "../util/emptyChatCompletion.js"; -import { extractBase64FromDataUrl } from "../../../../core/util/url.js" import { safeParseArgs } from "../util/parseArgs.js"; import { CACHING_STRATEGIES, @@ -200,7 +200,7 @@ export class AnthropicApi implements BaseLlmApi { source: { type: "base64", media_type: getAnthropicMediaTypeFromDataUrl(dataUrl), - data: extractBase64FromDataUrl(dataUrl); + data: extractBase64FromDataUrl(dataUrl), }, }); }