From 64f3e127f96a33458f667792e8587581d5ba81af Mon Sep 17 00:00:00 2001 From: dcs-soni <52232900+dcs-soni@users.noreply.github.com> Date: Tue, 4 Nov 2025 14:53:32 +0530 Subject: [PATCH] fix: cubic fixes, graceful returns instead of throwing --- core/llm/llms/Anthropic.ts | 25 ++++++--- core/llm/llms/Bedrock.ts | 10 ++-- core/llm/llms/Gemini.ts | 36 ++++++++---- core/llm/llms/Ollama.ts | 10 +++- core/util/url.ts | 39 +++++++++++++ .../openai-adapters/src/apis/Anthropic.ts | 25 ++++++--- packages/openai-adapters/src/apis/Bedrock.ts | 55 ++++++++++--------- 7 files changed, 142 insertions(+), 58 deletions(-) diff --git a/core/llm/llms/Anthropic.ts b/core/llm/llms/Anthropic.ts index ff3f85e51eb..a1fdfbe3598 100644 --- a/core/llm/llms/Anthropic.ts +++ b/core/llm/llms/Anthropic.ts @@ -28,6 +28,7 @@ import { } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { DEFAULT_REASONING_TOKENS } from "../constants.js"; import { BaseLLM } from "../index.js"; @@ -105,14 +106,22 @@ class Anthropic extends BaseLLM { }); } } else { - parts.push({ - type: "image", - source: { - type: "base64", - media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url), - data: part.imageUrl.url.split(",")[1], - }, - }); + const base64Data = extractBase64FromDataUrl(part.imageUrl.url); + if (base64Data) { + parts.push({ + type: "image", + source: { + type: "base64", + media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url), + data: base64Data, + }, + }); + } else { + console.warn( + "Anthropic: skipping image with invalid data URL format", + part.imageUrl.url, + ); + } } } } diff --git a/core/llm/llms/Bedrock.ts b/core/llm/llms/Bedrock.ts index ea47b63ae1f..7b338e9df59 100644 --- a/core/llm/llms/Bedrock.ts +++ b/core/llm/llms/Bedrock.ts @@ -21,6 +21,7 @@ import type { CompletionOptions } from "../../index.js"; import { ChatMessage, Chunk, LLMOptions, MessageContent } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { parseDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; import { PROVIDER_TOOL_SUPPORT } from "../toolSupport.js"; import { getSecureID } from "../utils/getSecureID.js"; @@ -545,8 +546,9 @@ class Bedrock extends BaseLLM { if (part.type === "text") { blocks.push({ text: part.text }); } else if (part.type === "imageUrl" && part.imageUrl) { - try { - const [mimeType, base64Data] = part.imageUrl.url.split(","); + const parsed = parseDataUrl(part.imageUrl.url); + if (parsed) { + const { mimeType, base64Data } = parsed; const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; if ( format === ImageFormat.JPEG || @@ -568,8 +570,8 @@ class Bedrock extends BaseLLM { part, ); } - } catch (error) { - console.warn("Bedrock: failed to process image part", error, part); + } else { + console.warn("Bedrock: failed to process image part", part); } } } diff --git a/core/llm/llms/Gemini.ts b/core/llm/llms/Gemini.ts index 7caf2db7d75..0e0b9b243fe 100644 --- a/core/llm/llms/Gemini.ts +++ b/core/llm/llms/Gemini.ts @@ -11,6 +11,7 @@ import { } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; import { GeminiChatContent, @@ -184,16 +185,31 @@ class Gemini extends BaseLLM { } continuePartToGeminiPart(part: MessagePart): GeminiChatContentPart { - return part.type === "text" - ? { - text: part.text, - } - : { - inlineData: { - mimeType: "image/jpeg", - data: part.imageUrl?.url.split(",")[1], - }, - }; + if (part.type === "text") { + return { + text: part.text, + }; + } + + let data = ""; + if (part.imageUrl?.url) { + const extracted = extractBase64FromDataUrl(part.imageUrl.url); + if (extracted) { + data = extracted; + } else { + console.warn( + "Gemini: skipping image with invalid data URL format", + part.imageUrl.url, + ); + } + } + + return { + inlineData: { + mimeType: "image/jpeg", + data, + }, + }; } public prepareBody( diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index 0a239fce909..7e36ac83647 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -13,6 +13,7 @@ import { } from "../../index.js"; import { renderChatMessage } from "../../util/messageContent.js"; import { getRemoteModelInfo } from "../../util/ollamaHelper.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; type OllamaChatMessage = { @@ -303,9 +304,16 @@ class Ollama extends BaseLLM implements ModelInstaller { const images: string[] = []; message.content.forEach((part) => { if (part.type === "imageUrl" && part.imageUrl) { - const image = part.imageUrl?.url.split(",").at(-1); + const image = part.imageUrl?.url + ? extractBase64FromDataUrl(part.imageUrl.url) + : undefined; if (image) { images.push(image); + } else if (part.imageUrl?.url) { + console.warn( + "Ollama: skipping image with invalid data URL format", + part.imageUrl.url, + ); } } }); diff --git a/core/util/url.ts b/core/util/url.ts index 83e0edcba14..d31bc775e5e 100644 --- a/core/util/url.ts +++ b/core/util/url.ts @@ -9,3 +9,42 @@ export function canParseUrl(url: string): boolean { return false; } } + +export function parseDataUrl(dataUrl: string): + | { + mimeType: string; + base64Data: string; + } + | undefined { + const urlParts = dataUrl.split(","); + + if (urlParts.length < 2) { + return undefined; + } + + const [mimeType, ...base64Parts] = urlParts; + const base64Data = base64Parts.join(","); + + return { mimeType, base64Data }; +} + +export function extractBase64FromDataUrl(dataUrl: string): string | undefined { + return parseDataUrl(dataUrl)?.base64Data; +} + +export function safeSplit( + input: string, + delimiter: string, + expectedParts: number, + errorContext: string = "input", +): string[] { + const parts = input.split(delimiter); + + if (parts.length !== expectedParts) { + throw new Error( + `Invalid ${errorContext} format: expected ${expectedParts} parts separated by "${delimiter}", got ${parts.length}`, + ); + } + + return parts; +} diff --git a/packages/openai-adapters/src/apis/Anthropic.ts b/packages/openai-adapters/src/apis/Anthropic.ts index 0a7bc573cc5..661089efaa1 100644 --- a/packages/openai-adapters/src/apis/Anthropic.ts +++ b/packages/openai-adapters/src/apis/Anthropic.ts @@ -25,6 +25,7 @@ import { CompletionUsage, } from "openai/resources/index"; import { ChatCompletionCreateParams } from "openai/resources/index.js"; +import { extractBase64FromDataUrl } from "../../../../core/util/url.js"; import { AnthropicConfig } from "../types.js"; import { chatChunk, @@ -194,14 +195,22 @@ export class AnthropicApi implements BaseLlmApi { if (part.type === "image_url") { const dataUrl = part.image_url.url; if (dataUrl?.startsWith("data:")) { - blocks.push({ - type: "image", - source: { - type: "base64", - media_type: getAnthropicMediaTypeFromDataUrl(dataUrl), - data: dataUrl.split(",")[1], - }, - }); + const base64Data = extractBase64FromDataUrl(dataUrl); + if (base64Data) { + blocks.push({ + type: "image", + source: { + type: "base64", + media_type: getAnthropicMediaTypeFromDataUrl(dataUrl), + data: base64Data, + }, + }); + } else { + console.warn( + "Anthropic: skipping image with invalid data URL format", + dataUrl, + ); + } } } else { const text = part.type === "text" ? part.text : part.refusal; diff --git a/packages/openai-adapters/src/apis/Bedrock.ts b/packages/openai-adapters/src/apis/Bedrock.ts index cf5588d686d..7b332399980 100644 --- a/packages/openai-adapters/src/apis/Bedrock.ts +++ b/packages/openai-adapters/src/apis/Bedrock.ts @@ -31,6 +31,7 @@ import { import { fromNodeProviderChain } from "@aws-sdk/credential-providers"; import { fromStatic } from "@aws-sdk/token-providers"; +import { parseDataUrl } from "../../../../core/util/url.js"; import { BedrockConfig } from "../types.js"; import { chatChunk, chatChunkFromDelta, embedding, rerank } from "../util.js"; import { safeParseArgs } from "../util/parseArgs.js"; @@ -134,35 +135,35 @@ export class BedrockApi implements BaseLlmApi { throw new Error("Unsupported part type: input_audio"); case "image_url": default: - try { - const [mimeType, base64Data] = ( - part as ChatCompletionContentPartImage - ).image_url.url.split(","); - const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; - if ( - format === ImageFormat.JPEG || - format === ImageFormat.PNG || - format === ImageFormat.WEBP || - format === ImageFormat.GIF - ) { - return { - image: { - format, - source: { - bytes: Uint8Array.from(Buffer.from(base64Data, "base64")), - }, - }, - }; - } else { - console.warn( - `Bedrock: skipping unsupported image part format: ${format}`, - ); - return { text: "[Unsupported image format]" }; - } - } catch (error) { - console.warn("Bedrock: failed to process image part", error); + const parsed = parseDataUrl( + (part as ChatCompletionContentPartImage).image_url.url, + ); + if (!parsed) { + console.warn("Bedrock: failed to process image part - invalid URL"); return { text: "[Failed to process image]" }; } + const { mimeType, base64Data } = parsed; + const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; + if ( + format === ImageFormat.JPEG || + format === ImageFormat.PNG || + format === ImageFormat.WEBP || + format === ImageFormat.GIF + ) { + return { + image: { + format, + source: { + bytes: Uint8Array.from(Buffer.from(base64Data, "base64")), + }, + }, + }; + } else { + console.warn( + `Bedrock: skipping unsupported image part format: ${format}`, + ); + return { text: "[Unsupported image format]" }; + } } }