From b562c94537e2d05c99fcec2b367d4cf21cf26964 Mon Sep 17 00:00:00 2001 From: ppranay20 Date: Tue, 18 Nov 2025 23:27:22 +0530 Subject: [PATCH 1/4] feat: added lm-studio provider --- apps/web/.env.example | 2 ++ apps/web/env.ts | 4 ++++ apps/web/package.json | 1 + apps/web/utils/llms/config.ts | 2 ++ apps/web/utils/llms/model.ts | 21 +++++++++++++++++++++ 5 files changed, 30 insertions(+) diff --git a/apps/web/.env.example b/apps/web/.env.example index 2fbabc90d1..4a05211e92 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -39,6 +39,8 @@ DEFAULT_LLM_MODEL= # BEDROCK_REGION=us-west-2 # OLLAMA_BASE_URL=http://localhost:11434/api # NEXT_PUBLIC_OLLAMA_MODEL=phi3 +# LM_STUDIO_BASE_URL=http://localhost:1234/v1 +# NEXT_PUBLIC_LM_STUDIO_MODEL=qwen/qwen3-vl-4b # Economy LLM configuration (for large context windows where cost efficiency matters) ECONOMY_LLM_PROVIDER= diff --git a/apps/web/env.ts b/apps/web/env.ts index fc35a293c8..9649ecfd85 100644 --- a/apps/web/env.ts +++ b/apps/web/env.ts @@ -11,6 +11,7 @@ const llmProviderEnum = z.enum([ "groq", "aigateway", "ollama", + "lmstudio", ]); export const env = createEnv({ @@ -57,6 +58,7 @@ export const env = createEnv({ OPENROUTER_API_KEY: z.string().optional(), AI_GATEWAY_API_KEY: z.string().optional(), OLLAMA_BASE_URL: z.string().optional(), + LM_STUDIO_BASE_URL: z.string().optional(), UPSTASH_REDIS_URL: z.string().optional(), UPSTASH_REDIS_TOKEN: z.string().optional(), @@ -177,6 +179,7 @@ export const env = createEnv({ .string() .default("us.anthropic.claude-3-7-sonnet-20250219-v1:0"), NEXT_PUBLIC_OLLAMA_MODEL: z.string().optional(), + NEXT_PUBLIC_LM_STUDIO_MODEL: z.string().optional(), NEXT_PUBLIC_APP_HOME_PATH: z.string().default("/setup"), NEXT_PUBLIC_DUB_REFER_DOMAIN: z.string().optional(), NEXT_PUBLIC_DISABLE_REFERRAL_SIGNATURE: z.coerce @@ -237,6 +240,7 @@ export const env = createEnv({ NEXT_PUBLIC_BEDROCK_SONNET_MODEL: process.env.NEXT_PUBLIC_BEDROCK_SONNET_MODEL, NEXT_PUBLIC_OLLAMA_MODEL: process.env.NEXT_PUBLIC_OLLAMA_MODEL, + NEXT_PUBLIC_LM_STUDIO_MODEL: process.env.NEXT_PUBLIC_LM_STUDIO_MODEL, NEXT_PUBLIC_APP_HOME_PATH: process.env.NEXT_PUBLIC_APP_HOME_PATH, NEXT_PUBLIC_DUB_REFER_DOMAIN: process.env.NEXT_PUBLIC_DUB_REFER_DOMAIN, NEXT_PUBLIC_DISABLE_REFERRAL_SIGNATURE: diff --git a/apps/web/package.json b/apps/web/package.json index d20e2d81f1..ea71a7e487 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -20,6 +20,7 @@ "@ai-sdk/google": "2.0.23", "@ai-sdk/groq": "2.0.24", "@ai-sdk/openai": "2.0.53", + "@ai-sdk/openai-compatible": "^1.0.27", "@ai-sdk/provider": "2.0.0", "@ai-sdk/react": "2.0.76", "@asteasolutions/zod-to-openapi": "8.1.0", diff --git a/apps/web/utils/llms/config.ts b/apps/web/utils/llms/config.ts index 58af32a3d9..a5e96e3597 100644 --- a/apps/web/utils/llms/config.ts +++ b/apps/web/utils/llms/config.ts @@ -11,6 +11,7 @@ export const Provider = { GROQ: "groq", OPENROUTER: "openrouter", AI_GATEWAY: "aigateway", + LM_STUDIO: "lmstudio", ...(supportsOllama ? { OLLAMA: "ollama" } : {}), }; @@ -31,6 +32,7 @@ export const Model = { GEMINI_2_5_PRO_OPENROUTER: "google/gemini-2.5-pro", GROQ_LLAMA_3_3_70B: "llama-3.3-70b-versatile", KIMI_K2_OPENROUTER: "moonshotai/kimi-k2", + LM_STUDIO: env.NEXT_PUBLIC_LM_STUDIO_MODEL, ...(supportsOllama ? { OLLAMA: env.NEXT_PUBLIC_OLLAMA_MODEL } : {}), }; diff --git a/apps/web/utils/llms/model.ts b/apps/web/utils/llms/model.ts index 859da9cd2f..cdb9f8bbe3 100644 --- a/apps/web/utils/llms/model.ts +++ b/apps/web/utils/llms/model.ts @@ -6,6 +6,7 @@ import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { createGroq } from "@ai-sdk/groq"; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { createGateway } from "@ai-sdk/gateway"; +import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; // import { createOllama } from "ollama-ai-provider"; import { env } from "@/env"; import { Model, Provider } from "@/utils/llms/config"; @@ -139,6 +140,26 @@ function selectModel( // model: createOllama({ baseURL: env.OLLAMA_BASE_URL })(model), // }; } + case Provider.LM_STUDIO: { + const modelName = aiModel || Model.LM_STUDIO; + + if (!modelName) { + throw new Error("For LM_STUDIO, 'LLM_MODEL' must be set."); + } + + const lmstudio = createOpenAICompatible({ + name: Provider.LM_STUDIO, + baseURL: env.LM_STUDIO_BASE_URL!, + supportsStructuredOutputs: true, + }); + + return { + provider: Provider.LM_STUDIO, + modelName, + model: lmstudio(modelName), + backupModel: null, + }; + } // this is messy. better to have two providers. one for bedrock and one for anthropic case Provider.ANTHROPIC: { From 25d19139ab56262759e7aae75a2d62cad92e1278 Mon Sep 17 00:00:00 2001 From: ppranay20 Date: Wed, 19 Nov 2025 00:45:52 +0530 Subject: [PATCH 2/4] test(llm): added test for lm studio --- apps/web/utils/llms/model.test.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/apps/web/utils/llms/model.test.ts b/apps/web/utils/llms/model.test.ts index af3ffa280f..ae6fbadac1 100644 --- a/apps/web/utils/llms/model.test.ts +++ b/apps/web/utils/llms/model.test.ts @@ -3,6 +3,7 @@ import { getModel } from "./model"; import { Provider, Model } from "./config"; import { env } from "@/env"; import type { UserAIFields } from "./types"; +import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; // Mock AI provider imports vi.mock("@ai-sdk/openai", () => ({ @@ -35,6 +36,10 @@ vi.mock("ollama-ai-provider", () => ({ createOllama: vi.fn(() => (model: string) => ({ model })), })); +vi.mock("@ai-sdk/openai-compatible", () => ({ + createOpenAICompatible: vi.fn(() => (model: string) => ({ model })), +})); + vi.mock("@/env", () => ({ env: { DEFAULT_LLM_PROVIDER: "openai", @@ -56,6 +61,8 @@ vi.mock("@/env", () => ({ BEDROCK_ACCESS_KEY: "", BEDROCK_SECRET_KEY: "", NEXT_PUBLIC_BEDROCK_SONNET_MODEL: "anthropic.claude-3-sonnet-20240229-v1:0", + LM_STUDIO_BASE_URL: "http://localhost:1234/v1", + NEXT_PUBLIC_LM_STUDIO_MODEL: "qwen/qwen3-vl-4b", }, })); @@ -164,6 +171,27 @@ describe("Models", () => { // expect(result.model).toBeDefined(); // }); + it("should configure LM Studio model correctly", () => { + const userAi: UserAIFields = { + aiApiKey: "user-api-key", + aiProvider: Provider.LM_STUDIO, + aiModel: Model.LM_STUDIO!, + }; + + const result = getModel(userAi); + + expect(result.provider).toBe(Provider.LM_STUDIO); + expect(result.modelName).toBe(Model.LM_STUDIO); + + expect(createOpenAICompatible).toHaveBeenCalledWith({ + name: Provider.LM_STUDIO, + baseURL: env.LM_STUDIO_BASE_URL!, + supportsStructuredOutputs: true, + }); + + expect(result.model).toBeDefined(); + }); + it("should configure Anthropic model correctly without Bedrock credentials", () => { const userAi: UserAIFields = { aiApiKey: "user-api-key", From 1cf1aabec41092d5b30e8b99779491261c40e76b Mon Sep 17 00:00:00 2001 From: ppranay20 Date: Wed, 19 Nov 2025 17:45:47 +0530 Subject: [PATCH 3/4] fix: add return for LM Studio in chat/economy selection --- apps/web/utils/llms/model.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/apps/web/utils/llms/model.ts b/apps/web/utils/llms/model.ts index cdb9f8bbe3..2b381735b0 100644 --- a/apps/web/utils/llms/model.ts +++ b/apps/web/utils/llms/model.ts @@ -232,7 +232,17 @@ function createOpenRouterProviderOptions( */ function selectEconomyModel(userAi: UserAIFields): SelectModel { if (env.ECONOMY_LLM_PROVIDER && env.ECONOMY_LLM_MODEL) { + const isLMStudio = env.ECONOMY_LLM_PROVIDER === Provider.LM_STUDIO; + if (isLMStudio) { + return selectModel({ + aiProvider: Provider.LM_STUDIO, + aiModel: env.ECONOMY_LLM_MODEL, + aiApiKey: null, + }); + } + const apiKey = getProviderApiKey(env.ECONOMY_LLM_PROVIDER); + if (!apiKey) { logger.warn("Economy LLM provider configured but API key not found", { provider: env.ECONOMY_LLM_PROVIDER, @@ -269,6 +279,15 @@ function selectEconomyModel(userAi: UserAIFields): SelectModel { */ function selectChatModel(userAi: UserAIFields): SelectModel { if (env.CHAT_LLM_PROVIDER && env.CHAT_LLM_MODEL) { + const isLMStudio = env.CHAT_LLM_PROVIDER === Provider.LM_STUDIO; + if (isLMStudio) { + return selectModel({ + aiProvider: Provider.LM_STUDIO, + aiModel: env.CHAT_LLM_MODEL, + aiApiKey: null, + }); + } + const apiKey = getProviderApiKey(env.CHAT_LLM_PROVIDER); if (!apiKey) { logger.warn("Chat LLM provider configured but API key not found", { @@ -322,7 +341,6 @@ function selectDefaultModel(userAi: UserAIFields): SelectModel { const openRouterOptions = createOpenRouterProviderOptions( env.DEFAULT_OPENROUTER_PROVIDERS || "", ); - // Preserve any custom options set earlier; always ensure reasoning exists. const existingOpenRouterOptions = providerOptions.openrouter || {}; providerOptions.openrouter = { From 6996fd7020cf28da1aeffb16595ed57bc955557c Mon Sep 17 00:00:00 2001 From: ppranay20 Date: Wed, 19 Nov 2025 18:25:07 +0530 Subject: [PATCH 4/4] fix(code rabbit): throw error for LM_STUDIO_BASE_URL --- apps/web/utils/llms/model.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/web/utils/llms/model.ts b/apps/web/utils/llms/model.ts index 2b381735b0..cf2c7be1d8 100644 --- a/apps/web/utils/llms/model.ts +++ b/apps/web/utils/llms/model.ts @@ -144,12 +144,20 @@ function selectModel( const modelName = aiModel || Model.LM_STUDIO; if (!modelName) { - throw new Error("For LM_STUDIO, 'LLM_MODEL' must be set."); + throw new Error( + "LM Studio model not configured. Please set NEXT_PUBLIC_LM_STUDIO_MODEL environment variable.", + ); + } + + if (!env.LM_STUDIO_BASE_URL) { + throw new Error( + "LM Studio base URL not configured. Please set LM_STUDIO_BASE_URL environment variable.", + ); } const lmstudio = createOpenAICompatible({ name: Provider.LM_STUDIO, - baseURL: env.LM_STUDIO_BASE_URL!, + baseURL: env.LM_STUDIO_BASE_URL, supportsStructuredOutputs: true, });